How to feather the chroma mask from edges

Question

This is the shader i am using to do chroma key , the shader works well but i need to feather the edges of the chroma mask.

How can i do that ?

#version 430 core

uniform sampler2D u_tex;     
vec4 keyRGBA = vec4(86.0 / 255.0 , 194.0 / 255.0, 46.0 / 255.0 , 1.0);    // key color as rgba
vec2 keyCC;      // the CC part of YCC color model of key color
uniform vec2 rangeSpill = vec2(0.1, .52);      // the smoothstep range for spill detection
uniform vec2 range = vec2(0.05, 0.21);      // the smoothstep range for chroma detection
in vec2 texCoord;
out vec4 FragColor;

vec2 RGBToCC(vec4 rgba) {
    float Y = 0.299 * rgba.r + 0.587 * rgba.g + 0.114 * rgba.b;
    return vec2((rgba.b - Y) * 0.565, (rgba.r - Y) * 0.713);
}       

vec2 RGBAToCC (float r, float g, float b) {
    
    float y = 0.299 * r + 0.587 * g + 0.114 * b;    
    return vec2((b - y) * 0.565, (r - y) * 0.713);
}


vec3 RGBToYCC( vec3 col )
{
    float y = 0.299 * col.r + 0.587 * col.g + 0.114 * col.b;    
    return vec3( y ,(col.b - y) * 0.565, (col.r - y) * 0.713);

}


vec3 YCCToRGB( vec3 col )
{
   float R  = col.x + (col.z - 128) *  1.40200;
   float G  = col.x + (col.y - 128) * -0.34414 + (col.z - 128) * -0.71414;
   float B  = col.x + (col.y - 128) *  1.77200;
   return vec3( R , G , B);

}


vec3 hueShift( vec3 color, float hueAdjust ){ 
     vec3  kRGBToYPrime = vec3 (0.299, 0.587, 0.114);
     vec3  kRGBToI      = vec3 (0.596, -0.275, -0.321);
     vec3  kRGBToQ      = vec3 (0.212, -0.523, 0.311);
     vec3  kYIQToR     = vec3 (1.0, 0.956, 0.621);
     vec3  kYIQToG     = vec3 (1.0, -0.272, -0.647);
     vec3  kYIQToB     = vec3 (1.0, -1.107, 1.704);
     float   YPrime  = dot (color, kRGBToYPrime);
     float   I       = dot (color, kRGBToI);
     float   Q       = dot (color, kRGBToQ);
     float   hue     = atan (Q, I);
     float   chroma  = sqrt (I * I + Q * Q);
     hue += hueAdjust;
     Q = chroma * sin (hue);
     I = chroma * cos (hue);
     vec3    yIQ   = vec3 (YPrime, I, Q);
     return vec3( dot (yIQ, kYIQToR), dot (yIQ, kYIQToG), dot (yIQ, kYIQToB) );
}


float GetYComponent( vec3 color){ 
     vec3  kRGBToYPrime = vec3 (0.299, 0.587, 0.114);
     vec3  kRGBToI      = vec3 (0.596, -0.275, -0.321);
     vec3  kRGBToQ      = vec3 (0.212, -0.523, 0.311);
     vec3  kYIQToR     = vec3 (1.0, 0.956, 0.621);
     vec3  kYIQToG     = vec3 (1.0, -0.272, -0.647);
     vec3  kYIQToB     = vec3 (1.0, -1.107, 1.704);
     float   YPrime  = dot (color, kRGBToYPrime);
     return YPrime;
}

        
void main() {
    vec4 src1Color = texture2D(u_tex,  texCoord);   
    keyCC = RGBAToCC( keyRGBA.r , keyRGBA.g , keyRGBA.b  );
    vec2 CC = RGBToCC(src1Color);   
    float mask = sqrt(pow(keyCC.x - CC.x, 2.0) + pow(keyCC.y - CC.y, 2.0));     
    mask = smoothstep(rangeSpill.x + 0.5, rangeSpill.y, mask);
    if (mask > 0.0 && mask < .8)
    {
      src1Color = vec4( hueShift(src1Color.rgb , 1.8 ) , src1Color.a );  // spill remover           
    }   
    
    // Now the spill is removed do the chroma
     vec2 CC2 = RGBToCC(src1Color);
     float mask2 = sqrt(pow(keyCC.x - CC2.x, 2.0) + pow(keyCC.y - CC2.y, 2.0));
     mask2 = smoothstep(range.x, range.y, mask2);       
    if (mask2 == 0.0) { discard; }
            else if (mask2 == 1.0)
        {   
        FragColor = vec4(src1Color.rgb ,  mask2);
        }
        else 
        {
            vec4 col = max(src1Color - (1.0 - mask2) * keyRGBA, 0.0);               
            FragColor = vec4(hueShift(col.rgb , 0.3 ) , col.a); // do color correction      
        }
        
}

This is the base image

This is the result after chroma keying.

Also there is not much information avaliable for chroma keying if someone could also give some information about adding more details in the shader.

Answer 1

Effectively, you need to extrude the areas where the Chroma key matched. While you could just sample in a pattern (instead of a single point) in a single render pass, that's not quite efficient.

Instead you should rather write the mask to a 1bit (or as much as you would like for transparency) mask texture first. Then you can run a simple 1D shader in X and Y direction over that mask to extrude the already excluded areas by a fixed amount. You need a temporary texture for playing ping-pong either way, and splitting X and Y dimensions requires far less samples in total.

E.g. the minimum opacity in a range of 5px, or a Gaussian blur with a scaler / clamp to keep already full transparent pixels still transparent.

Ultimately, combine your final mask with the source image as usual.