CI Face Detector stopping audio from recording with AVAssetWriter

Question

I'm wanting to detect a face in real time video using apples CI Face Detector and then I want to record the video to file using AVAssetWriter.

I thought I had it working but the audio is being temperamental. Sometimes it will record properly with the video, other times it will start recording but then go mute, other times it's out of sync with the video, and sometimes it won't work at all.

With a print statement I can see that the audio sample buffer is there. It must have something to do with the face detection as when I comment out that code the recording works fine.

Here's my code:

// MARK: AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAudioOutputSampleBufferDelegate
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {

    let writable = canWrite()
    if writable {
        print("Writable")
    }

    if writable,
        sessionAtSourceTime == nil {
        // Start Writing
        sessionAtSourceTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
        videoWriter.startSession(atSourceTime: sessionAtSourceTime!)
        print("session started")
    }


    // processing on the images, not audio
    if output == videoDataOutput {
        connection.videoOrientation = .portrait
        if connection.isVideoMirroringSupported {
            connection.isVideoMirrored = true
        }

        // convert current frame to CIImage
        let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer)
        let attachments = CMCopyDictionaryOfAttachments(kCFAllocatorDefault, pixelBuffer!, CMAttachmentMode(kCMAttachmentMode_ShouldPropagate)) as? [String: Any]
        let ciImage = CIImage(cvImageBuffer: pixelBuffer!, options: attachments)

        // Detects faces based on your ciimage
        let features = faceDetector?.features(in: ciImage, options: [CIDetectorSmile : true,
                                                                     CIDetectorEyeBlink : true,
                                                                     ]).compactMap({ $0 as? CIFaceFeature })

        // Retreive frame of your buffer
        let desc = CMSampleBufferGetFormatDescription(sampleBuffer)
        let bufferFrame = CMVideoFormatDescriptionGetCleanAperture(desc!, false)

        // Draw face masks
        DispatchQueue.main.async { [weak self] in
            UIView.animate(withDuration: 0.2) {
            self?.drawFaceMasksFor(features: features!, bufferFrame: bufferFrame)
            }
        }
    }

    if writable,
        output == videoDataOutput,
        (videoWriterInput.isReadyForMoreMediaData) {
        // write video buffer
        videoWriterInput.append(sampleBuffer)
        print("video buffering")
    } else if writable,
        output == audioDataOutput,
        (audioWriterInput.isReadyForMoreMediaData) {
        // write audio buffer
        audioWriterInput?.append(sampleBuffer)
        print("audio buffering")
    }

}