What is the correct way to get Preview Frames using Android NDK Camera2

Question

Based on NDK camera sample texture-view, I want to create an ImageReader to get preview frames.

What I've done

Create the ImageReader and the camera session:

yuvReader_ = new ImageReader(&compatibleCameraRes_, AIMAGE_FORMAT_YUV_420_888);
camera_->CreateSession(ANativeWindow_fromSurface(env_, surface), yuvReader_->GetNativeWindow());

void NDKCamera::CreateSession(ANativeWindow* previewWindow, ANativeWindow* yuvWindow) {
    // Create output from this app's ANativeWindow, and add into output container
    requests_[PREVIEW_REQUEST_IDX].outputNativeWindow_ = previewWindow;
    requests_[PREVIEW_REQUEST_IDX].template_ = TEMPLATE_PREVIEW;
    requests_[YUV_REQUEST_IDX].outputNativeWindow_ = yuvWindow;
    requests_[YUV_REQUEST_IDX].template_ = TEMPLATE_PREVIEW;

    CALL_CONTAINER(create(&outputContainer_));
    for (auto& req : requests_) {
        if (!req.outputNativeWindow_) continue;

        ANativeWindow_acquire(req.outputNativeWindow_);
        CALL_OUTPUT(create(req.outputNativeWindow_, &req.sessionOutput_));
        CALL_CONTAINER(add(outputContainer_, req.sessionOutput_));
        CALL_TARGET(create(req.outputNativeWindow_, &req.target_));
        CALL_DEV(createCaptureRequest(cameras_[activeCameraId_].device_,
                                      req.template_, &req.request_));
        CALL_REQUEST(addTarget(req.request_, req.target_));
    }

    // Create a capture session for the given preview request
    captureSessionState_ = CaptureSessionState::READY;
    CALL_DEV(createCaptureSession(cameras_[activeCameraId_].device_,
                                  outputContainer_, GetSessionListener(),
                                  &captureSession_));
}

Then start the preview:

void NDKCamera::StartPreview(bool start) {
  if (start) {
    ACaptureRequest* requests[] = { requests_[PREVIEW_REQUEST_IDX].request_, requests_[YUV_REQUEST_IDX].request_};
    CALL_SESSION(setRepeatingRequest(captureSession_, nullptr, 2,
                                     requests,
                                     nullptr));
  } else if (!start && captureSessionState_ == CaptureSessionState::ACTIVE) {
    ACameraCaptureSession_stopRepeating(captureSession_);
  }
}

I set two requests in setRepeatingRequest. One for TextureView display, and the other for receiving the preview frames in C++.

Now, the problem is after setting two outputs, the preview performance goes down (looks like playing slides), which doesn't occur in Java:

mCameraDevice.createCaptureSession(Arrays.asList(surface, mImageReader.getSurface()),
                    new CameraCaptureSession.StateCallback() {

                        @Override
                        public void onConfigured(@NonNull CameraCaptureSession cameraCaptureSession) {
                            // The camera is already closed
                            if (null == mCameraDevice) {
                                return;
                            }

                            mCaptureSession = cameraCaptureSession;
                            startPreview();
                        }

                        @Override
                        public void onConfigureFailed(
                                @NonNull CameraCaptureSession cameraCaptureSession) {
                            showToast("Failed");
                        }
                    }, null
            );

I also tried one request with two output targets. But the code caused screen frozen:

void NDKCamera::CreateSession(ANativeWindow* textureViewWindow, ANativeWindow* imgReaderWindow) {
    auto& req = requests_[PREVIEW_REQUEST_IDX];
    req.outputNativeWindow_ = textureViewWindow;
    req.yuvWindow = imgReaderWindow;
    req.template_ = TEMPLATE_PREVIEW;

    ACaptureSessionOutputContainer_create(&outputContainer_);
    CALL_DEV(createCaptureRequest(cameras_[activeCameraId_].device_,
                                          req.template_, &req.request_));
    
    // Add the texture view surface to the container
    ANativeWindow_acquire(req.outputNativeWindow_);
    CALL_OUTPUT(create(req.outputNativeWindow_, &req.sessionOutput_));
    CALL_CONTAINER(add(outputContainer_, req.sessionOutput_));
    CALL_TARGET(create(req.outputNativeWindow_, &req.target_));
    CALL_REQUEST(addTarget(req.request_, req.target_));

    // Add the image reader surface to the container
    ANativeWindow_acquire(req.yuvWindow);
    CALL_OUTPUT(create(req.yuvWindow, &req.yuvOutput));
    CALL_CONTAINER(add(outputContainer_, req.yuvOutput));
    CALL_TARGET(create(req.yuvWindow, &req.yuvTarget));
    CALL_REQUEST(addTarget(req.request_, req.yuvTarget));

    captureSessionState_ = CaptureSessionState::READY;
    ACameraDevice_createCaptureSession(cameras_[activeCameraId_].device_,
                                  outputContainer_, GetSessionListener(),
                                  &captureSession_);
}

void NDKCamera::StartPreview(bool start) {
  if (start) {
    ACaptureRequest* requests[] = { requests_[PREVIEW_REQUEST_IDX].request_};
    ACameraCaptureSession_setRepeatingRequest(captureSession_, nullptr, 1,
                                     requests,
                                     nullptr);
  } else if (!start && captureSessionState_ == CaptureSessionState::ACTIVE) {
    ACameraCaptureSession_stopRepeating(captureSession_);
  }
}

Here is the log:

2021-12-14 08:42:20.316 24536-24556/com.sample.textureview D/ACameraDevice: Device error received, code 3, frame number 13, request ID 0, subseq ID 0
2021-12-14 08:42:21.319 24536-24556/com.sample.textureview D/ACameraDevice: Device error received, code 3, frame number 14, request ID 0, subseq ID 0
2021-12-14 08:42:22.321 24536-24584/com.sample.textureview D/ACameraDevice: Device error received, code 3, frame number 15, request ID 0, subseq ID 0
2021-12-14 08:42:23.323 24536-24584/com.sample.textureview D/ACameraDevice: Device error received, code 3, frame number 16, request ID 0, subseq ID 0
2021-12-14 08:42:24.325 24536-24556/com.sample.textureview D/ACameraDevice: Device error received, code 3, frame number 17, request ID 0, subseq ID 0
2021-12-14 08:42:25.328 24536-24584/com.sample.textureview D/ACameraDevice: Device error received, code 3, frame number 18, request ID 0, subseq ID 0
2021-12-14 08:42:26.330 24536-24584/com.sample.textureview D/ACameraDevice: Device error received, code 3, frame number 19, request ID 0, subseq ID 0

Anybody knows why? Thanks!

Answer 1

I don't know how you've set up your Java code by comparison, but what you're doing in the NDK code will drop your frame rate by half. If you want to get both preview frames and frames to the native ImageReader at 30fps, you need to include both targets in a single capture request, not alternate between two capture requests that each target just one output. The latter will get you 15fps to each output at best.

So just create one request, and call addTarget twice on it with both the preview and the YUV windows. There are limits on how many targets you can add to a single request, but generally that's equal to the number of targets you can configure in a single session, which depends on the hardware capability of the device, and the resolution of each output.

2 streams, one preview and one app-bound YUV, should always work, however.