How to get ECS task to communicate with cloudwatch agent?

Question

I can't get the agent to work.

    resource "aws_ecs_task_definition" "task_definition" {
    family = "my-service${var.suffix}_service"
    container_definitions = jsonencode([
        {
            name = "cloudwatch-agent"
            image = "public.ecr.aws/cloudwatch-agent/cloudwatch-agent:latest"
            memory = 256
            cpu = 256
            portMappings = [
                {
                    protocol = "tcp"
                    containerPort = 25888
                }
            ]
            logConfiguration = {
                logDriver = "awslogs"
                options = {
                    awslogs-create-group = "true"
                    awslogs-group = "/aws/ecs/my-log-group"
                    awslogs-region = "us-west-2"
                    awslogs-stream-prefix = "ecs"
                }
            }
            Environment = [
                {"name": "CW_CONFIG_CONTENT", "value": "{\"logs\": { \"metrics_collected\": { \"emf\": { }}}}"}
            ]
        },
        {
            name = "my-service${var.suffix}container"
            cpu = 1024
            memory = 2048
            image = "${var.image-name}@${var.image-hash}"
            portMappings = [
                {
                    name = "scanrunner-http"
                    containerPort = 80
                    hostPort = 80
                    protocol = "http"
                }
            ]
            logConfiguration = {
                logDriver = "awslogs"
                options = {
                    awslogs-create-group = "true"
                    awslogs-group = "/aws/ecs/my-log-group"
                    awslogs-region = "us-west-2"
                    awslogs-stream-prefix = "ecs"
                }
            }
            health_check = {

            }
            Environment = [
                [...]
                {"name": "CLOUDWATCH_LOG_GROUP", "value": aws_cloudwatch_log_group.metrics.name},   
                {"name": "AWS_EMF_AGENT_ENDPOINT", "value": "tcp://127.0.0.1:25888"}
            ]
        }
    ])
    cpu = 2048
    execution_role_arn = var.service_role
    task_role_arn = var.service_role
    memory = 4096
    network_mode = "awsvpc"
    requires_compatibilities = ["FARGATE"]
    runtime_platform {
        cpu_architecture = "ARM64"
        operating_system_family = "LINUX"
    }
}

Setup in code

Amazon.CloudWatch.EMF.Config.EnvironmentConfigurationProvider.Config =
        new Amazon.CloudWatch.EMF.Config.Configuration
        {
            ServiceName = "Athena-ScanRunner",
            ServiceType = "WebApi",
            LogGroupName = Environment.GetEnvironmentVariable("CLOUDWATCH_LOG_GROUP"),
            EnvironmentOverride = Amazon.CloudWatch.EMF.Environment.Environments.ECS
        };

    builder.Services.AddEmf();

Everything starts but I don't see anything in the logs for either that indicates metrics are flowing. nothing ends up my metrics log group. I've made sure to setup the VPC endpoint for logs, and I think I've added all the needed IAM permissions to the tasks role.