How to get depth data from camera images

Question

I am writing an app to get the depth data and disparity data from pictures taken from the camera. I can get the disparity data but not the depth data it always returns nil. I need to get the depth information and save it as a jpg

I have tried the below code where user can switch between front and back camera and take pictures then the picture we took will be the process

import UIKit
import AVFoundation

class ViewController: UIViewController {

    @IBOutlet weak var ImageView: UIView!

    var img:UIImage?
    var rgbImage:UIImage?

    var captureSession: AVCaptureSession?
    var videoPreviewLayer: AVCaptureVideoPreviewLayer?
    var backCamera = AVCaptureDevice.default(.builtInDualCamera, for: .video, position: .back)
    var frontCamera = AVCaptureDevice.default(.builtInTrueDepthCamera, for: .video, position: .front)
    var capturePhotoOut : AVCapturePhotoOutput?

    override func viewDidLoad() {
        super.viewDidLoad()

        if #available(iOS 10.2, *){
            let captureDevice = AVCaptureDevice.default(.builtInDualCamera, for: .video, position: .back)
            do{
                let input = try  AVCaptureDeviceInput(device: captureDevice!)
                captureSession = AVCaptureSession()
                captureSession?.addInput(input)
                videoPreviewLayer = AVCaptureVideoPreviewLayer(session: captureSession!)
                videoPreviewLayer?.frame = view.layer.bounds
                ImageView.layer.addSublayer(videoPreviewLayer!)
                captureSession?.startRunning()
            }catch{
                print("error")
            }
        }

        capturePhotoOut = AVCapturePhotoOutput()
        capturePhotoOut?.isHighResolutionCaptureEnabled = true
        captureSession?.sessionPreset = .photo
        captureSession?.addOutput(capturePhotoOut!)
        capturePhotoOut!.isDepthDataDeliveryEnabled = capturePhotoOut!.isDepthDataDeliverySupported
        capturePhotoOut!.isPortraitEffectsMatteDeliveryEnabled = capturePhotoOut!.isPortraitEffectsMatteDeliverySupported
    }


    @IBAction func imageCapture(_ sender: Any) {
        guard let capturePhotoOutput = self.capturePhotoOut else {return}
        let photoSettings = AVCapturePhotoSettings()
        photoSettings.isAutoStillImageStabilizationEnabled = true
        photoSettings.isHighResolutionPhotoEnabled = true
        photoSettings.isDepthDataDeliveryEnabled = true
        photoSettings.isPortraitEffectsMatteDeliveryEnabled = true
        capturePhotoOut?.capturePhoto(with: photoSettings, delegate: self)
    }

    override func prepare(for segue: UIStoryboardSegue, sender: Any?) {
        let vc = segue.destination   as! DepthImageView
        vc.img = img
        vc.rgbImg = rgbImage
    }


extension ViewController : AVCapturePhotoCaptureDelegate{
    public func photoOutput(_ output: AVCapturePhotoOutput, didFinishProcessingPhoto photo: AVCapturePhoto, error: Error?) {
        guard error == nil else{return}
        guard let imageData = photo.fileDataRepresentation() else {return}


        let detailImage = UIImage.init(data: imageData,scale: 1.0)
        rgbImage = detailImage

        let nsData = imageData as NSData
        let ptr = nsData.bytes.assumingMemoryBound(to: UInt8.self)

        let cfDataset = CFDataCreate(nil,ptr,imageData.count)


        guard let source = CGImageSourceCreateWithData(cfDataset!,nil) else {return}

        guard let auxDataInfo = CGImageSourceCopyAuxiliaryDataInfoAtIndex(source, 0, kCGImageAuxiliaryDataTypeDepth) as? [String : AnyObject] else {
            return
        }

        var depthData: AVDepthData

        do {
            depthData = try AVDepthData(fromDictionaryRepresentation: auxDataInfo)

            if depthData.depthDataType != kCVPixelFormatType_DepthFloat32 {
                depthData = depthData.converting(toDepthDataType: kCVPixelFormatType_DepthFloat32)
            }

            let depthDataMap = depthData.depthDataMap

            let ciImage = CIImage(cvPixelBuffer: depthDataMap)

            let depthDataMapImage = UIImage(ciImage: ciImage,scale: 1.0,orientation: .down)
            img = depthDataMapImage
            self.performSegue(withIdentifier: "ImageViewScreen", sender: self)

        } catch {
            print("Error")
        }
    }
}

I always get nil at auxDataInfo guard

Answer 1

AVCapturePhoto containing information about AVDepthData. Try to get depth Data from photo

let depthData = photo.depthData