Create a VideoFrame from Canvas

89 Views Asked by At

In the video call using Livekit, need to do the video frame-processing before sending them. So, I need to create a VideoFrame from a Canvas. However, I am getting just a black still image on the other end, not the actual Canvas content.

Can somebody point out what's missing or wrong here to get the correct Canvas with a red rectangle drawn on it?

override fun onFrameCaptured(frame: VideoFrame?) {
    if (frame != null) {
        val sourceBuffer = frame.buffer?.toI420()!!
        val width = sourceBuffer.width
        val height = sourceBuffer.height

        val start = System.nanoTime()
        val textures = IntArray(1)
        GLES20.glGenTextures(1, textures, 0)

        val yuvConverter = YuvConverter()
        val buffer = TextureBufferImpl(
            width, height, VideoFrame.TextureBuffer.Type.RGB,
            textures[0], Matrix(), surfaceTextureHelper.handler, yuvConverter, null
        )
        val bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888)

        val canvas = Canvas(bitmap)
        canvas.drawRGB(255, 255, 255)
        val paint = Paint()
        paint.color = Color.RED
        paint.alpha = 0xff
        canvas.drawRect(100f, 100f, 200f, 200f, paint)
        surfaceTextureHelper.handler.post {
            GLES20.glTexParameteri(
                GLES20.GL_TEXTURE_2D,
                GLES20.GL_TEXTURE_MIN_FILTER,
                GLES20.GL_NEAREST
            )
            GLES20.glTexParameteri(
                GLES20.GL_TEXTURE_2D,
                GLES20.GL_TEXTURE_MAG_FILTER,
                GLES20.GL_NEAREST
            )
            GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, 0, bitmap, 0)
            val i420Buf = yuvConverter.convert(buffer)
            val frameTime: Long = System.nanoTime() - start
            
            val videoFrame = VideoFrame(i420Buf, frame.rotation, frameTime)
            
            sink?.onFrame(videoFrame)
            videoFrame.release()
        }
    }
}
0

There are 0 best solutions below