Issue decoding with openh264 a libvlc broadcast sent over UDP

Question

With the following code, I get the same data over UDP however DecodeFrameNoDelay returns an error about Sequence Parameter Sets.

Is there something I'm missing in libvlc config or openh264 usage ?

Broadcasting part :

// broadcaster.h
class Broadcaster
{
    ...

    std::vector<uint8_t> audioBuffer_;
    std::vector<uint8_t> videoBuffer_;
};

// broadcaster.cpp
void cbVideoPrerender(void* p_video_data, uint8_t** pp_pixel_buffer, int size)
{
    Broadcaster::instance()->videoPrerender(p_video_data, pp_pixel_buffer, size);
}

void cbVideoPostrender(
    void* p_video_data, uint8_t* p_pixel_buffer, int width, int height, int pixel_pitch, int size, int64_t pts)
{
    Broadcaster::instance()->videoPostrender(p_video_data, p_pixel_buffer, width, height, pixel_pitch, size, pts);
}

void Broadcaster::videoPrerender(void* p_video_data, uint8_t** pp_pixel_buffer, int size)
{
    spdlog::debug("videoPrerender");
    videoBuffer_.resize(size);
    *pp_pixel_buffer = videoBuffer_.data();
}

void Broadcaster::videoPostrender(
    void* p_video_data, uint8_t* p_pixel_buffer, int width, int height, int pixel_pitch, int size, int64_t pts)
{
    spdlog::debug("videoPostrender");

    std::vector<unsigned char> frame;
    frame.resize(size);
    memcpy(frame.data(), p_pixel_buffer, size);
    spdlog::debug("size: {}", size);
    // sending data over UDP
}

Broadcaster::Broadcaster()
{
    std::ostringstream stream;
    stream << "#transcode{vcodec=h264,venc=x264{profile=baseline,level=3.0},vb=800,scale=1,acodec=mp3,ab=128,channels=2,samplerate=44100}:smem{";
    stream << "video-prerender-callback=" << (long long int)(intptr_t)(void*)&cbVideoPrerender << ",";
    stream << "video-postrender-callback=" << (long long int)(intptr_t)(void*)&cbVideoPostrender << ",";
    stream << "audio-prerender-callback=" << (long long int)(intptr_t)(void*)&cbAudioPrerender << ",";
    stream << "audio-postrender-callback=" << (long long int)(intptr_t)(void*)&cbAudioPostrender << ",";
    stream << "audio-data=" << (long long int)0 << ",";
    stream << "video-data=" << (long long int)0 << "";
    stream << "},";

    inst_ = libvlc_new(0, NULL);

    std::vector<const char*> params = {
        "sout", "sout-all", "screen-top=0", "screen-left=0", "screen-width=1920", "screen-height=1080", "screen-fps=10"};

    libvlc_vlm_add_broadcast(inst_, "stream", "screen://", stream.str().c_str(), params.size(), params.data(), 1, 0);
}

Decoding part (based on https://github.com/cisco/openh264/wiki/UsageExampleForDecoder):

// streamwindow.h
class StreamWindow
{
    ...

    ISVCDecoder*   pSvcDecoder_;
    unsigned char* pData_[3];
    SDecodingParam sDecParam_ = {0};
};

// streamwindow.cpp
static spdlog::level::level_enum openh264_to_spdlog(int level)
{
    switch (level)
    {
    case WELS_LOG_QUIET:
        return spdlog::level::level_enum::trace;
    case WELS_LOG_ERROR:
        return spdlog::level::level_enum::err;
    case WELS_LOG_WARNING:
        return spdlog::level::level_enum::warn;
    case WELS_LOG_INFO:
        return spdlog::level::level_enum::info;
    case WELS_LOG_DEBUG:
        return spdlog::level::level_enum::debug;
    case WELS_LOG_DETAIL:
        return spdlog::level::level_enum::info;
    default:
        return spdlog::level::level_enum::debug;
    }
}

static void openh264_spdlog(void* context, int level, const char* message)
{
    spdlog::log(openh264_to_spdlog(level), message);
}

StreamWindow::StreamWindow()
{
    WelsCreateDecoder(&pSvcDecoder_);
    sDecParam_.sVideoProperty.eVideoBsType = VIDEO_BITSTREAM_SVC;
    pSvcDecoder_->Initialize(&sDecParam_);

    int               log_level = WELS_LOG_DETAIL;
    WelsTraceCallback callback_function;
    callback_function = openh264_spdlog;
    pSvcDecoder_->SetOption(DECODER_OPTION_TRACE_LEVEL, &log_level);
    pSvcDecoder_->SetOption(DECODER_OPTION_TRACE_CALLBACK, (void*)&callback_function);

    ...
}

// called on received udp data
void StreamWindow::decodeStreamData(unsigned char* data, int size)
{
    SBufferInfo sDstBufInfo;
    memset(&sDstBufInfo, 0, sizeof(SBufferInfo));
    DECODING_STATE iRet = pSvcDecoder_->DecodeFrameNoDelay(data, size, pData_, &sDstBufInfo);
    if (iRet != 0)
    {
        spdlog::warn("DecodeFrameNoDelay failed: 0x{0:x}", iRet);
    }

    ...
}

EDIT:

libVLC logs

[00007f2f440011f0] x264 encoder: using cpu capabilities: MMX2 SSE2Fast SSSE3 SSE4.2 AVX
[00007f2f440011f0] x264 encoder: profile Constrained Baseline, level 3.0, 4:2:0, 8-bit
[00007f2f440011f0] x264 encoder: final ratefactor: 38.99
[00007f2f440011f0] x264 encoder: using SAR=1/1
[00007f2f440011f0] x264 encoder: using cpu capabilities: MMX2 SSE2Fast SSSE3 SSE4.2 AVX
[00007f2f440011f0] x264 encoder: profile Constrained Baseline, level 3.0, 4:2:0, 8-bit

decodeStreamData logs

[2020-09-07 21:52:38.463] [debug] decodeStreamData, size: 47022
[2020-09-07 21:52:38.464] [warning] [OpenH264] this = 0x0x5630b8e202b0, Warning:parse_nal(), no exist Sequence Parameter Sets ahead of sequence when try to decode NAL(type:5).
[2020-09-07 21:52:38.464] [info] [OpenH264] this = 0x0x5630b8e202b0, Info:decode failed, failure type:16 

[2020-09-07 21:52:38.464] [warning] DecodeFrameNoDelay failed: 0x10

Answer 1

The VLC logs report the stream is high profile. Open264 only supports baseline profile.