How does FFMPEG 4.4 convert NV12 format to YUV420 on Windows?

Question

I want to use FFmpeg to convert the input NV12 format to YUV420P.
I tried to use sws_scale conversion, but the colors are all green.
I have successfully converted YUYV422 to 420P using sws, but FAILED to convert NV12 to YUV420.
What should I change?

Here is my code：

QStringList inList = int_put_res.split("x");

m_in_width = inList[0].toInt();

m_in_hieght = inList[1].toInt();

QStringList outList = out_put_res.split("x");

m_out_width = outList[0].toInt();

m_out_hieght = outList[1].toInt();

int VSize = m_in_width * m_in_hieght;

AVPacket *packet;

AVCodecContext *enc_ctx = NULL;

AVFrame *image_inFrame= nullptr;

packet = av_packet_alloc();

int ret = 0;

int base = 0; 

image_inFrame = av_frame_alloc();

open_encoder(m_out_width,m_out_hieght, &enc_ctx);

switch (AVPixelFormat(m_fmt_ctx->streams[m_videoindex]->codecpar->format))
{
case AV_PIX_FMT_YUV420P:
     m_yuv420p_convert_ctx = initSWS(image_inFrame,NULL,m_in_width,m_in_hieght,AV_PIX_FMT_YUV420P,AV_PIX_FMT_YUV420P);
    break;
case AV_PIX_FMT_YUYV422:
      m_yuv420p_convert_ctx = initSWS(image_inFrame,NULL,m_in_width,m_in_hieght,AV_PIX_FMT_YUYV422,AV_PIX_FMT_YUV420P);
    break;
case AV_PIX_FMT_NV12:
      m_yuv420p_convert_ctx = initSWS(image_inFrame,NULL,m_in_width,m_in_hieght,AV_PIX_FMT_NV12,AV_PIX_FMT_YUV420P);
    break;

}


m_frame = create_frame(m_out_width, m_out_hieght);

AVPacket *newpkt = av_packet_alloc();
if (!newpkt)
{
    spdlog::error("failed to alloc avpacket!");
    goto __ERROR;
}
if (!m_fmt_ctx)
{

    goto __ERROR;
}
while (m_isStop != true)
{
    if(av_read_frame(m_fmt_ctx, packet) != 0) {
        goto __ERROR;
    }

    getSWS(m_yuv420p_convert_ctx,packet,image_inFrame,m_frame);
    av_packet_unref(packet);


    m_frame->pts = base++;
    
}

---

SwsContext *FFSDK::initSWS(AVFrame *scrFrame, AVFrame *dstFrame, int 
image_width, int image_height,enum AVPixelFormat scrFmt, enum 
AVPixelFormat dstFmt)

if(scrFrame != NULL){
    av_image_alloc(scrFrame->data, scrFrame->linesize,image_width, image_height, scrFmt, 1);
}

if(dstFrame != NULL){
    av_image_alloc(dstFrame->data, dstFrame->linesize, m_out_width, m_out_hieght, dstFmt, 1);
}

return  sws_getContext(image_width,image_height,
                       scrFmt,
                       m_out_width, m_out_hieght, dstFmt,
                       SWS_BICUBIC,
                       nullptr,
                       nullptr,
                       nullptr);

}

void FFSDK::getSWS(SwsContext *context,AVPacket *packet,AVFrame *scrFrame, AVFrame *dstFrame)
{

    scrFrame->data[0] = packet->data;
    sws_scale(context,
              scrFrame->data,
              scrFrame->linesize, 0, m_in_hieght, dstFrame->data,
              dstFrame->linesize);
}

Answer 1

I can't see the problem from the pieces of code that you have posted.

Green colors are usually result of zero data (all elements of Y, U and V are zeros).
The reason for the zero content is probably because nothing is written to the destination (or source) buffers of the video frame.

---

I have created a "self contained" code sample that demonstrates the conversion from NV12 to YUV420 using sws_scale.

• Start by building synthetic input frame using FFmpeg (command line tool).
  The command creates 320x240 video frame in raw NV12 format:

    ffmpeg -y -f lavfi -i testsrc=size=320x240:rate=1 -vcodec rawvideo -pix_fmt nv12 -frames 1 -f rawvideo nv12_image.bin
  

---

The next code sample applies the following stages:

• Allocate memory for the source frame (in NV12 format).
• Read NV12 data from binary file (for testing).
• Allocate memory for the destination frame (in YUV420 format).
• Apply color space conversion (using sws_scale).
• Write the converted YUV420 data to binary file (for testing).

Here is the complete code:

//Use extern "C", because the code is built as C++ (cpp file) and not C.
extern "C"
{
    #include <libswscale/swscale.h>
    #include <libavformat/avformat.h>
    #include <libswresample/swresample.h>
    #include <libavutil/pixdesc.h>
    #include <libavutil/imgutils.h>
}


int main()
{
    int width = 320;
    int height = 240;   //The code sample assumes height is even.
    int align = 0;
    AVPixelFormat srcPxlFormat = AV_PIX_FMT_NV12;
    AVPixelFormat dstPxlFormat = AV_PIX_FMT_YUV420P;
    int sts;

    //Source frame allocation
    ////////////////////////////////////////////////////////////////////////////
    AVFrame* pNV12Frame = av_frame_alloc();   

    pNV12Frame->format = srcPxlFormat;
    pNV12Frame->width = width;
    pNV12Frame->height = height;

    sts = av_frame_get_buffer(pNV12Frame, align);

    if (sts < 0)
    {
        return -1;  //Error!
    }
    ////////////////////////////////////////////////////////////////////////////


    //Read NV12 data from binary file (for testing)
    ////////////////////////////////////////////////////////////////////////////
    //Use FFmpeg for building raw NV12 image (used as input).
    //ffmpeg -y -f lavfi -i testsrc=size=320x240:rate=1 -vcodec rawvideo -pix_fmt nv12 -frames 1 -f rawvideo nv12_image.bin

    FILE *f = fopen("nv12_image.bin", "rb");

    if (f == NULL)
    {
        return -1;  //Error!
    }

    //Read Y channel from nv12_image.bin (Y channel size is width x height).
    //Reading row by row is required in rare cases when pNV12Frame->linesize[0] != width
    uint8_t *Y = pNV12Frame->data[0];   //Pointer to Y color channel of the NV12 frame.
    for (int row = 0; row < height; row++)
    {
        fread(Y + (uintptr_t)row * pNV12Frame->linesize[0], 1, width, f); //Read row (width pixels) to Y0.
    }

    //Read UV channel from nv12_image.bin (UV channel size is width x height/2).
    uint8_t* UV = pNV12Frame->data[1];   //Pointer to UV color channels of the NV12 frame (ordered as UVUVUVUV...).
    for (int row = 0; row < height/2; row++)
    {
        fread(UV + (uintptr_t)row * pNV12Frame->linesize[1], 1, width, f); //Read row (width pixels) to UV0.
    }

    fclose(f);
    ////////////////////////////////////////////////////////////////////////////



    //Destination frame allocation
    ////////////////////////////////////////////////////////////////////////////
    AVFrame *pYUV420Frame = av_frame_alloc();

    pYUV420Frame->format = dstPxlFormat;
    pYUV420Frame->width = width;
    pYUV420Frame->height = height;

    sts = av_frame_get_buffer(pYUV420Frame, align);

    if (sts < 0)
    {
        return -1;  //Error!
    }
    ////////////////////////////////////////////////////////////////////////////


    //Color space conversion
    ////////////////////////////////////////////////////////////////////////////
    SwsContext *sws_context = sws_getContext(width,
                                             height,
                                             srcPxlFormat,
                                             width,
                                             height,
                                             dstPxlFormat,
                                             SWS_FAST_BILINEAR,
                                             NULL,
                                             NULL,
                                             NULL);

    if (sws_context == NULL)
    {
        return -1;  //Error!
    }

    sts = sws_scale(sws_context,             //struct SwsContext* c,
                    pNV12Frame->data,        //const uint8_t* const srcSlice[],
                    pNV12Frame->linesize,    //const int srcStride[],
                    0,                       //int srcSliceY, 
                    pNV12Frame->height,      //int srcSliceH,
                    pYUV420Frame->data,      //uint8_t* const dst[], 
                    pYUV420Frame->linesize); //const int dstStride[]);    

    if (sts != pYUV420Frame->height)
    {
        return -1;  //Error!
    }
    ////////////////////////////////////////////////////////////////////////////


    //Write YUV420 data to binary file (for testing)
    ////////////////////////////////////////////////////////////////////////////
    //Use FFmpeg for converting the binary image to PNG after saving the data.
    //ffmpeg -y -f rawvideo -video_size 320x240 -pixel_format yuv420p -i yuv420_image.bin -pix_fmt rgb24 rgb_image.png

    f = fopen("yuv420_image.bin", "wb");

    if (f == NULL)
    {
        return -1;  //Error!
    }

    //Write Y channel to yuv420_image.bin (Y channel size is width x height).
    //Writing row by row is required in rare cases when pYUV420Frame->linesize[0] != width
    Y = pYUV420Frame->data[0];   //Pointer to Y color channel of the YUV420 frame.
    for (int row = 0; row < height; row++)
    {
        fwrite(Y + (uintptr_t)row * pYUV420Frame->linesize[0], 1, width, f); //Write row (width pixels) to file.
    }

    //Write U channel to yuv420_image.bin (U channel size is width/2 x height/2).
    uint8_t* U = pYUV420Frame->data[1];   //Pointer to U color channels of the YUV420 frame.
    for (int row = 0; row < height/2; row++)
    {
        fwrite(U + (uintptr_t)row * pYUV420Frame->linesize[1], 1, width/2, f); //Write row (width/2 pixels) to file.
    }

    //Write V channel to yuv420_image.bin (V channel size is width/2 x height/2).
    uint8_t* V = pYUV420Frame->data[2];   //Pointer to V color channels of the YUV420 frame.
    for (int row = 0; row < height/2; row++)
    {
        fwrite(V + (uintptr_t)row * pYUV420Frame->linesize[2], 1, width/2, f); //Write row (width/2 pixels) to file.
    }

    fclose(f);
    ////////////////////////////////////////////////////////////////////////////


    //Cleanup
    ////////////////////////////////////////////////////////////////////////////
    sws_freeContext(sws_context);
    av_frame_free(&pYUV420Frame);
    av_frame_free(&pNV12Frame);
    ////////////////////////////////////////////////////////////////////////////

    return 0;
}

---

For validating the output:

• After executing the code, execute FFmpeg (command line tool).
  The following command converts the raw binary frame (in YUV420 format) to PNG (in RGB format).

    ffmpeg -y -f rawvideo -video_size 320x240 -pixel_format yuv420p -i yuv420_image.bin -pix_fmt rgb24 rgb_image.png
  

---

Sample output (after converting from YUV420 to PNG image file format):

---

I hope it helps you find the problem...