Metal core image kernel with sampler

Question

I am trying to use a CIColorKernel or CIBlendKernel with sampler arguments but the program crashes. Here is my shader code which compiles successfully.

extern "C" float4 wipeLinear(coreimage::sampler t1, coreimage::sampler t2, float time) {
    float2 coord1 = t1.coord();
    float2 coord2 = t2.coord();

    float4 innerRect = t2.extent();

    float minX = innerRect.x + time*innerRect.z;
    float minY = innerRect.y + time*innerRect.w;
    float cropWidth = (1 - time) * innerRect.w;
    float cropHeight = (1 - time) * innerRect.z;

    float4 s1 = t1.sample(coord1);
    float4 s2 = t2.sample(coord2);

   if ( coord1.x > minX && coord1.x < minX + cropWidth && coord1.y > minY && coord1.y <= minY + cropHeight) {
       return s1;
   } else {
      return s2;
   }
}

And it crashes on initialization.

class CIWipeRenderer: CIFilter {
var backgroundImage:CIImage?
var foregroundImage:CIImage?
var  inputTime: Float = 0.0

static var kernel:CIColorKernel = { () -> CIColorKernel in

    let url = Bundle.main.url(forResource: "AppCIKernels", withExtension: "ci.metallib")!
    let data = try! Data(contentsOf: url)
    return try! CIColorKernel(functionName: "wipeLinear", fromMetalLibraryData: data) //Crashes here!!!!
    
}()

override var outputImage: CIImage? {
    guard let backgroundImage = backgroundImage else {
        return nil
    }
    
    guard let foregroundImage = foregroundImage else {
        return nil
    }
    
    return CIWipeRenderer.kernel.apply(extent: backgroundImage.extent, arguments: [backgroundImage, foregroundImage, inputTime])
}

}

It crashes in the try line with the following error:

 Fatal error: 'try!' expression unexpectedly raised an error: Foundation._GenericObjCError.nilError

If I replace the kernel code with the following, it works like a charm:

  extern "C" float4 wipeLinear(coreimage::sample_t s1, coreimage::sample_t s2, float time)
{
     return mix(s1, s2, time);
}

So there are no obvious errors in the code, such as passing incorrect function name or so.

Answer 1

Yes, you can't use samplers in CIColorKernel or CIBlendKernel. Those kernels are optimized for the use case where you have a 1:1 mapping from input pixel to output pixel. This allows Core Image to execute multiple of these kernels in one command buffer since they don't require any intermediate buffer writes.
A sampler would allow you to sample the input at arbitrary coordinates, which is not allowed in this case.

You can simply use a CIKernel instead. It's meant to be used when you need to sample the input more freely.

To initialize the kernel, you need to adapt the code like this:

static var kernel: CIKernel = {
    let url = Bundle.main.url(forResource: "AppCIKernels", withExtension: "ci.metallib")!
let data = try! Data(contentsOf: URL)
    return try! CIKernel(functionName: "wipeLinear", fromMetalLibraryData: data)
}()

When calling the kernel, you now need to also provide a ROI callback, like this:

let roiCallback: CIKernelROICallback = { index, rect -> CGRect in
    return rect // you need the same region from the input as the output

}
// or even shorter
let roiCallback: CIKernelROICallback = { $1 }
return CIWipeRenderer.kernel.apply(extent: backgroundImage.extent, roiCallback: roiCallback, arguments: [backgroundImage, foregroundImage, inputTime])

Answer 2

For your use case, you actually can use a CIColorKernel. You just have to pass the extent of your render destination to the kernel as well, then you don't need the sampler to access it.

The kernel would look like this:

extern "C" float4 wipeLinear(coreimage::sample_t t1, coreimage::sample_t t2, float4 destinationExtent, float time, coreimage::destination destination) {
    float minX = destinationExtent.x + time * destinationExtent.z;
    float minY = destinationExtent.y + time * destinationExtent.w;
    float cropWidth = (1.0 - time) * destinationExtent.w;
    float cropHeight = (1.0 - time) * destinationExtent.z;

    float2 destCoord = destination.coord();

   if ( destCoord.x > minX && destCoord.x < minX + cropWidth && destCoord.y > minY && destCoord.y <= minY + cropHeight) {
       return t1;
   } else {
      return t2;
   }
}

And you call it like this:

let destinationExtent = CIVector(cgRect: backgroundImage.extent)
return CIWipeRenderer.kernel.apply(extent: backgroundImage.extent, arguments: [backgroundImage, foregroundImage, destinationExtent, inputTime])

Note that the last destination parameter in the kernel is passed automatically by Core Image. You don't need to pass it with the arguments.

Answer 3

Bonus answer:

For this blending effect, you actually don't need any kernel at all. You can achieve all that with simple cropping and compositing:

class CIWipeRenderer: CIFilter {
    var backgroundImage:CIImage?
    var foregroundImage:CIImage?
    var inputTime: CGFloat = 0.0

    override var outputImage: CIImage? {
        guard let backgroundImage = backgroundImage else { return nil }
        guard let foregroundImage = foregroundImage else { return nil }

        // crop the foreground based on time
        var foregroundCrop = foregroundImage.extent
        foregroundCrop.size.width *= inputTime
        foregroundCrop.size.height *= inputTime

        return foregroundImage.cropped(to: foregroundCrop).composited(over: backgroundImage)
    }
}