In the video call using Livekit, need to do the video frame-processing before sending them. So, I need to create a VideoFrame from a Canvas. However, I am getting just a black still image on the other end, not the actual Canvas content.
Can somebody point out what's missing or wrong here to get the correct Canvas with a red rectangle drawn on it?
override fun onFrameCaptured(frame: VideoFrame?) {
if (frame != null) {
val sourceBuffer = frame.buffer?.toI420()!!
val width = sourceBuffer.width
val height = sourceBuffer.height
val start = System.nanoTime()
val textures = IntArray(1)
GLES20.glGenTextures(1, textures, 0)
val yuvConverter = YuvConverter()
val buffer = TextureBufferImpl(
width, height, VideoFrame.TextureBuffer.Type.RGB,
textures[0], Matrix(), surfaceTextureHelper.handler, yuvConverter, null
)
val bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888)
val canvas = Canvas(bitmap)
canvas.drawRGB(255, 255, 255)
val paint = Paint()
paint.color = Color.RED
paint.alpha = 0xff
canvas.drawRect(100f, 100f, 200f, 200f, paint)
surfaceTextureHelper.handler.post {
GLES20.glTexParameteri(
GLES20.GL_TEXTURE_2D,
GLES20.GL_TEXTURE_MIN_FILTER,
GLES20.GL_NEAREST
)
GLES20.glTexParameteri(
GLES20.GL_TEXTURE_2D,
GLES20.GL_TEXTURE_MAG_FILTER,
GLES20.GL_NEAREST
)
GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, 0, bitmap, 0)
val i420Buf = yuvConverter.convert(buffer)
val frameTime: Long = System.nanoTime() - start
val videoFrame = VideoFrame(i420Buf, frame.rotation, frameTime)
sink?.onFrame(videoFrame)
videoFrame.release()
}
}
}