WebGL retrieve arbitrary meta data from shaders

Question

I have a shader that renders stuff.

During that rendering process I want to write certain data into a buffer for each pixel - be that for picking objects, depth values, whatever.

It seems that WebGL is extremely limited in what data can be written and read.

In my specific usecase, I want to write 3 signed values into a buffer for each pixel and then read a specific value for a pixel of my choice.

I know you can render data into a texture but if that data is color data then you are bound to clamp the output to float values between 0 and 1. What if I want to have integers as output? What if I want to have signed values as the output? I want to write arbitrary data with a shader and READ that data.

readPixels() seems to only read values from the COLOR_ATTACHMENT0 buffer which needs to be an RGBA texture or renderbuffer.

In the following example we have a fragment-shader that should simply write the X,Y and Z coordinate of a rendered voxel into the output so I can get what voxel was rendered to what pixel later in the program:

#version 300 es

precision highp float;

uniform sampler2D texture_color;

layout(std140) uniform Voxel {
  ivec4 voxel_pos;
};

in vec2 fs_texture_coordinate;
layout (location=0) out vec4 out_color;
layout (location=1) out ivec4 out_voxel;

void main(){
    out_color = texture(texture_color, fs_texture_coordinate);
    out_voxel = voxel_pos;
}

How the fs_voxel_coord gets its value doesn't matter - it can be a uniform, something interpolated, whatever.

I want to be able to read that value in the main program with readPixels or any other function from COLOR_ATTACHMENT1. How? I have been hanging on this simple task to get some metadata as feedback from the shaders for hours and have been running in circles, making progress and then getting thrown back again by limitations of the representable data.

My current, slightly working, flow is to write into a framebuffer with a texture at COLOR_ATTACHMENT0 the coordinates that I pass on as uniform data BUT because the values are RGBA I can only get values back between 0 and 255. I can of course use some bias but that limits the amount of things I can represent into a very tiny range where I want to have arbitrary data passed from the shader to the outside.

This is the framebuffer setup code:

function setupFramebuffer(GL) {
  const w = 1200;
  const h = 800;

  offscreenColorTexture = GL.createTexture();
  GL.bindTexture(GL.TEXTURE_2D, offscreenColorTexture);

  const level = 0;
  const border = 0;

  GL.texImage2D(
    GL.TEXTURE_2D,
    level,
    GL.RGBA,
    w,
    h,
    border,
    GL.RGBA,
    GL.UNSIGNED_BYTE,
    null
  );
  GL.texParameteri(GL.TEXTURE_2D, GL.TEXTURE_MIN_FILTER, GL.NEAREST);
  GL.texParameteri(GL.TEXTURE_2D, GL.TEXTURE_MAG_FILTER, GL.NEAREST);
  GL.texParameteri(GL.TEXTURE_2D, GL.TEXTURE_WRAP_S, GL.CLAMP_TO_EDGE);
  GL.texParameteri(GL.TEXTURE_2D, GL.TEXTURE_WRAP_T, GL.CLAMP_TO_EDGE);

  offscreenCoordTexture = GL.createTexture();
  GL.bindTexture(GL.TEXTURE_2D, offscreenCoordTexture);

  GL.texImage2D(
    GL.TEXTURE_2D,
    level,
    GL.RGBA32I,
    w,
    h,
    border,
    GL.RGBA_INTEGER,
    GL.INT,
    null
  );

  GL.texParameteri(GL.TEXTURE_2D, GL.TEXTURE_MIN_FILTER, GL.NEAREST);
  GL.texParameteri(GL.TEXTURE_2D, GL.TEXTURE_MAG_FILTER, GL.NEAREST);
  GL.texParameteri(GL.TEXTURE_2D, GL.TEXTURE_WRAP_S, GL.CLAMP_TO_EDGE);
  GL.texParameteri(GL.TEXTURE_2D, GL.TEXTURE_WRAP_T, GL.CLAMP_TO_EDGE);

  const depthBuffer = GL.createRenderbuffer();
  GL.bindRenderbuffer(GL.RENDERBUFFER, depthBuffer);
  GL.renderbufferStorage(GL.RENDERBUFFER, GL.DEPTH_COMPONENT16, w, h);

  const fb = GL.createFramebuffer();
  GL.bindFramebuffer(GL.FRAMEBUFFER, fb);
  GL.bindTexture(GL.TEXTURE_2D, offscreenColorTexture);
  GL.framebufferTexture2D(
    GL.FRAMEBUFFER,
    GL.COLOR_ATTACHMENT0,
    GL.TEXTURE_2D,
    offscreenColorTexture,
    0
  );

  GL.bindTexture(GL.TEXTURE_2D, offscreenCoordTexture);
  GL.framebufferTexture2D(
    GL.FRAMEBUFFER,
    GL.COLOR_ATTACHMENT1,
    GL.TEXTURE_2D,
    offscreenCoordTexture,
    0
  );

  GL.framebufferRenderbuffer(
    GL.FRAMEBUFFER,
    GL.DEPTH_ATTACHMENT,
    GL.RENDERBUFFER,
    depthBuffer,
    0
  );

  GL.bindFramebuffer(GL.FRAMEBUFFER, null);
  offscreenFramebuffer = fb;
}

I want to be able to read the integers from the COLOR_ATTACHMENT1 by pixel for object detection / picking.

Code Snippet to read out the values:

  GL.readBuffer(GL.COLOR_ATTACHMENT1);
  const pixel = new Int32Array(4);
  GL.readPixels(mouse.x, 800 - mouse.y, 1, 1, GL.RGBA_INTEGER, GL.INT, pixel);
  console.log(pixel); //[0,0,0,0] everywhere

The shader and uniform init code (using some helper classes of my own but it should be clear what happens):

function setupShaders(GL) {
  const vert3DShader = new Shader(GL, GL.VERTEX_SHADER, vertexShader3DSource);
  const frag3DShad = new Shader(GL, GL.FRAGMENT_SHADER, fragmentShader3DSource);
  shaderPrograms.main = new Program(GL, vert3DShader, frag3DShad);

  const matrixIndex = shaderPrograms.main.getUniformBlockIndex(GL, "Matrices");
  const voxelIndex = shaderPrograms.main.getUniformBlockIndex(GL, "Voxel");

  shaderPrograms.main.bindIndex(GL, matrixIndex, 2);
  shaderPrograms.main.bindIndex(GL, voxelIndex, 3);
}

function initializeUniforms(GL) {
  uniformBuffers.matrices = new Buffer(GL);
  uniformBuffers.voxel = new Buffer(GL);

  GL.bindBuffer(GL.UNIFORM_BUFFER, uniformBuffers.matrices.id);
  GL.bufferData(GL.UNIFORM_BUFFER, 3 * 16 * 4, GL.DYNAMIC_DRAW);
  GL.bindBufferBase(GL.UNIFORM_BUFFER, 2, uniformBuffers.matrices.id);

  GL.bindBuffer(GL.UNIFORM_BUFFER, uniformBuffers.voxel.id);
  GL.bindBufferBase(GL.UNIFORM_BUFFER, 3, uniformBuffers.voxel.id);
}

function drawVoxel(GL, position) {
  const modelMatrix = Matrix.translate(position.x, position.y, position.z);

  uniformBuffers.matrices.bufferSubData(
    GL,
    GL.UNIFORM_BUFFER,
    modelMatrix.m,
    32 * Float32Array.BYTES_PER_ELEMENT
  );

  uniformBuffers.voxel.bufferData(
    GL,
    GL.UNIFORM_BUFFER,
    [Math.round(position.x), Math.round(position.y), Math.round(position.z), 0],
    Int32Array
  );

  models.cube.render(GL);
}