Unity - extracting camera pixel array is incredibly slow

Question

I'm using the below lines of code to extract the pixel array from a camera at every frame, saving it as jpg and then running a python process on the jpg. Although it works, it is incredibly slow. The bottleneck seems to be reading the pixels in unity. The python process itself only lasts 0.02 seconds.

Can anyone suggest relatively easy ways I can speed up this process?

I've seen this , but it's too high-level for me to understand how to adapt it to my use-case.

public override void Initialize()
{
    renderTexture = new RenderTexture(84, 84, 24);
    rawByteData = new byte[84 * 84 * bytesPerPixel];
    texture2D = new Texture2D(84, 84, TextureFormat.RGB24, false);
    rect = new Rect(0, 0, 84, 84);
    cam.targetTexture = renderTexture;

}
private List<float> run_cmd()
{
    // Setup a camera, texture and render texture
    cam.targetTexture = renderTexture;
    cam.Render();
    // Read pixels to texture
    RenderTexture.active = renderTexture;
    texture2D.ReadPixels(rect, 0, 0);
    rawByteData = ImageConversion.EncodeToJPG(texture2D);
    // Assign random temporary filename to jpg
    string fileName = "/media/home/tmp/" + Guid.NewGuid().ToString() + ".jpg";
    File.WriteAllBytes(fileName, rawByteData); // Requires System.IO

     // Start Python process
     ProcessStartInfo start = new ProcessStartInfo();
     start.FileName = "/media/home/path/to/python/exe";
     start.Arguments = string.Format(
        "/media/home/path/to/pythonfile.py photo {0}", fileName);
     start.UseShellExecute = false;
     start.RedirectStandardOutput = true;
     start.RedirectStandardError = true;
     string stdout;
     using(Process process = Process.Start(start))
     {
         using(StreamReader reader = process.StandardOutput)
         {
             stdout = reader.ReadToEnd();
         }
     }

    string[] tokens = stdout.Split(',');
    List<float> result = tokens.Select(x => float.Parse(x)).ToList();
    System.IO.File.Delete(fileName);

    return result;


}

Answer 1

In Unity 2018.1 was added a new system for async read data from GPU - https://docs.unity3d.com/ScriptReference/Rendering.AsyncGPUReadback.html, and in this not necessary use camera for getting array of textures, I wrote a simple example which good work and pretty fast:

using System.Collections;
using System.IO;
using System.Threading.Tasks;
using UnityEngine;
using UnityEngine.Experimental.Rendering;
using UnityEngine.Rendering;

public class CameraTextureTest : MonoBehaviour
{
    private GraphicsFormat format;
    
    private int frameIndex = 0;
    
    IEnumerator Start()
    {
        yield return new WaitForSeconds(1);

        while (true)
        {
            yield return new WaitForSeconds(0.032f);
            yield return new WaitForEndOfFrame();

            var rt = RenderTexture.GetTemporary(Screen.width, Screen.height, 32);

            format = rt.graphicsFormat;

            ScreenCapture.CaptureScreenshotIntoRenderTexture(rt);
            
            AsyncGPUReadback.Request(rt, 0, TextureFormat.RGBA32, OnCompleteReadback);
            
            RenderTexture.ReleaseTemporary(rt);
        }
    }


    void OnCompleteReadback(AsyncGPUReadbackRequest request)
    {
        if (request.hasError)
        {
            Debug.Log("GPU readback error detected.");
            return;
        }

        byte[] array = request.GetData<byte>().ToArray();
        
        Task.Run(() =>
        {
            File.WriteAllBytes($"D:/Screenshots/Screenshot{frameIndex}.png",ImageConversion.EncodeArrayToPNG(array, format, (uint) Screen.width, (uint) Screen.height));
        });

        frameIndex++;
    }
}

You can adapt this example to your task.