AWS ECS Fargate Logging: Facing issue with Firelens side-car container approach to push log files present inside application container to Cloudwatch

Question

I want to push log files which are present in a specific folder inside a container to Cloudwatch. For this, I tried Firelens logdriver but had no luck.

As mentioned here : https://github.com/aws-samples/amazon-ecs-firelens-examples/tree/mainline/examples/fluent-bit/config-file-type-file

I created a custom docker image for fluentbit and deployed it as a side car container in task definition:

FROM public.ecr.aws/aws-observability/aws-for-fluent-bit:stable
COPY fluentbit.conf /extra.conf

fluentbit.conf (I tried exec for debugging purpose, aim is to use tail):

[INPUT]
    Name          exec
    Tag           exec_ls
    Command       ls /opt/apache-tomcat-8.5.72/my-app/logs
    Interval_Sec  3
    Interval_NSec 0
    Buf_Size      8mb

[OUTPUT]
    Name cloudwatch
    Match   *
    region ap-south-1
    log_group_name fluent-bit-adapter-logs
    log_stream_prefix from-fluent-bit-
    auto_create_group true
    log_key log

This is my ECS Fargate task definition :

{
  "ipcMode": null,
  "executionRoleArn": "<role>",
  "containerDefinitions": [
    {
      "dnsSearchDomains": null,
      "environmentFiles": null,
      "logConfiguration": {
        "logDriver": "awsfirelens",
        "secretOptions": null,
        "options": {
          "log_group_name": "/aws/ecs/containerinsights/$(ecs_cluster)/application",
          "auto_create_group": "true",
          "log_key": "log",
          "log_stream_prefix": "log_stream_name",
          "region": "ap-south-1",
          "Name": "cloudwatch"
        }
      },
      "entryPoint": null,
      "portMappings": [
        {
          "hostPort": 8080,
          "protocol": "tcp",
          "containerPort": 8080
        }
      ],
      "command": null,
      "linuxParameters": null,
      "cpu": 0,
      "environment": [],
      "resourceRequirements": null,
      "ulimits": null,
      "dnsServers": null,
      "mountPoints": [],
      "workingDirectory": null,
      "secrets": null,
      "dockerSecurityOptions": null,
      "memory": null,
      "memoryReservation": null,
      "volumesFrom": [],
      "stopTimeout": null,
      "image": "<image-url>",
      "startTimeout": null,
      "firelensConfiguration": null,
      "dependsOn": null,
      "disableNetworking": null,
      "interactive": null,
      "healthCheck": null,
      "essential": true,
      "links": null,
      "hostname": null,
      "extraHosts": null,
      "pseudoTerminal": null,
      "user": null,
      "readonlyRootFilesystem": null,
      "dockerLabels": null,
      "systemControls": null,
      "privileged": null,
      "name": "my-app"
    },
    {
      "dnsSearchDomains": null,
      "environmentFiles": null,
      "logConfiguration": {
        "logDriver": "awslogs",
        "secretOptions": null,
        "options": {
          "awslogs-group": "/ecs/log_router",
          "awslogs-region": "ap-south-1",
          "awslogs-create-group": "true",
          "awslogs-stream-prefix": "firelens"
        }
      },
      "entryPoint": null,
      "portMappings": [],
      "command": null,
      "linuxParameters": null,
      "cpu": 0,
      "environment": [],
      "resourceRequirements": null,
      "ulimits": null,
      "dnsServers": null,
      "mountPoints": [],
      "workingDirectory": null,
      "secrets": null,
      "dockerSecurityOptions": null,
      "memory": null,
      "memoryReservation": 50,
      "volumesFrom": [],
      "stopTimeout": null,
      "image": "<image-url>",
      "startTimeout": null,
      "firelensConfiguration": {
        "type": "fluentbit",
        "options": {
          "config-file-type": "file",
          "config-file-value": "/extra.conf"
        }
      },
      "dependsOn": null,
      "disableNetworking": null,
      "interactive": null,
      "healthCheck": null,
      "essential": true,
      "links": null,
      "hostname": null,
      "extraHosts": null,
      "pseudoTerminal": null,
      "user": "0",
      "readonlyRootFilesystem": null,
      "dockerLabels": null,
      "systemControls": null,
      "privileged": null,
      "name": "log_router"
    }
  ],
  "placementConstraints": [],
  "memory": "8192",
  "taskRoleArn": "<role>",
  "compatibilities": [
    "EC2",
    "FARGATE"
  ],
  "taskDefinitionArn": "<my-app arn>",
  "family": "equbemi",
  "requiresAttributes": [
    {
      "targetId": null,
      "targetType": null,
      "value": null,
      "name": "ecs.capability.execution-role-awslogs"
    },
    {
      "targetId": null,
      "targetType": null,
      "value": null,
      "name": "com.amazonaws.ecs.capability.ecr-auth"
    },
    {
      "targetId": null,
      "targetType": null,
      "value": null,
      "name": "ecs.capability.firelens.options.config.file"
    },
    {
      "targetId": null,
      "targetType": null,
      "value": null,
      "name": "com.amazonaws.ecs.capability.docker-remote-api.1.17"
    },
    {
      "targetId": null,
      "targetType": null,
      "value": null,
      "name": "com.amazonaws.ecs.capability.docker-remote-api.1.21"
    },
    {
      "targetId": null,
      "targetType": null,
      "value": null,
      "name": "com.amazonaws.ecs.capability.logging-driver.awsfirelens"
    },
    {
      "targetId": null,
      "targetType": null,
      "value": null,
      "name": "com.amazonaws.ecs.capability.task-iam-role"
    },
    {
      "targetId": null,
      "targetType": null,
      "value": null,
      "name": "ecs.capability.execution-role-ecr-pull"
    },
    {
      "targetId": null,
      "targetType": null,
      "value": null,
      "name": "com.amazonaws.ecs.capability.docker-remote-api.1.18"
    },
    {
      "targetId": null,
      "targetType": null,
      "value": null,
      "name": "ecs.capability.task-eni"
    },
    {
      "targetId": null,
      "targetType": null,
      "value": null,
      "name": "com.amazonaws.ecs.capability.docker-remote-api.1.29"
    },
    {
      "targetId": null,
      "targetType": null,
      "value": null,
      "name": "com.amazonaws.ecs.capability.logging-driver.awslogs"
    },
    {
      "targetId": null,
      "targetType": null,
      "value": null,
      "name": "com.amazonaws.ecs.capability.docker-remote-api.1.19"
    },
    {
      "targetId": null,
      "targetType": null,
      "value": null,
      "name": "ecs.capability.firelens.fluentbit"
    }
  ],
  "pidMode": null,
  "requiresCompatibilities": [
    "FARGATE"
  ],
  "networkMode": "awsvpc",
  "cpu": "4096",
  "revision": 19,
  "status": "ACTIVE",
  "inferenceAccelerators": null,
  "proxyConfiguration": null,
  "volumes": []
}

For my application container, I've given logconfiguration as firelens and deployed a side-car container as mentioned in documentation. I tried the tail command in firelens config but didn't work. So just to troubleshoot, I tried exec and found out in firelens container logs that it is giving "File not found" exception. I assume it is trying to find the path in its own container(the side-car one) and not in the application container. I'm not sure how to make the firelens container access the application container. Am I missing anything here?

*

Answer 1

Here, the problem is that the fluent-bit container does not have access to the application container file system. You need to configure a volume so that the log path will be shared between both of these containers.

https://docs.aws.amazon.com/AmazonECS/latest/developerguide/using_data_volumes.html

Add a volume at the parent level and add this volume as a mount point in both of the container definitions:

{
...
"volumes": [
    {
        "fsxWindowsFileServerVolumeConfiguration": null,
        "efsVolumeConfiguration": null,
        "name": "log_path",
        "host": {
            "sourcePath": null
        },
        "dockerVolumeConfiguration": null
    }
],
...
"containerDefinitions": [
    {
        ...
        "mountPoints": [
            {
                "readOnly": null,
                "containerPath": "/var/log",
                "sourceVolume": "log_path"
            }
        ],
        ...
    },
    {
        ...
        "mountPoints": [
            {
                "readOnly": null,
                "containerPath": "/var/log",
                "sourceVolume": "log_path"
            }
        ],
        ...
    }
]
...
}