Boto3: no logStreamName from AWS batch

32 Views Asked by At

I'm using boto3 to launch and monitor an AWS batch job. [This documentation][1] would suggest that, once the job is running, a call to describe_jobs on that job should get me a logStreamName field as part of the dictionary in the response this should be true once the job has reached the RUNNING state. However, it's actually appearing once the job is in the STARTING state - but if I try to access that log stream I get a "does not exist" error. Is this to be expected?

Here's the code, slightly modified to remove proprietary information:

import boto3, time

REGION = "us-west-2"
LOG_GROUP = "/aws/batch/job"
client = boto3.client('batch', region_name=REGION)

def startJob():
    response = client.submit_job(
        jobName='kens_boto3_test',  
        jobQueue='blast-queue', 
        jobDefinition='<PROPRIETARY REFERENCE REMOVED>', 
        containerOverrides={
            'environment': [
                {
                    'name': 'FILENAME_LIST_S3_URL',
                    'value': '',
                },
                {
                    'name': 'FILENAME',
                    'value': 'Seq1.fasta',
                },
            ],
        },
    )
    return response


counter = 0

def getUpdate(job):
    global counter
    why = ""
    logName = ""
    
    response2 = client.describe_jobs(jobs=[job])
    jobInfo = response2["jobs"][0]
    name = jobInfo["jobName"]
    id = jobInfo["jobId"]
    status = jobInfo["status"]
    if "statusReason" in jobInfo:
        why = jobInfo["statusReason"]
    attempts = len(jobInfo["attempts"])
    
    container = jobInfo["container"]
    command = container["command"]
    if "logName" in jobInfo:
        logName = container["logStreamName"]
    counter += 1
    print(f'[{counter}] ', name)
    if counter == 1:
        print("command:", command)
        print("id:", id)
    print("status:", status, " :", why)
    # print("attempts:", attempts)
    print("logName:", logName)
    print("")

    if logName:
        do_log_events(LOG_GROUP, logName)
    return status


next_token = None

def do_log_events(log_group_name, log_stream_name, start_time=None):
    global next_token
    client = boto3.client('logs')

    if start_time is None:
        start_time = 0  # Start from the beginning of the log stream
    if next_token:
        response = client.get_log_events(
            logGroupName=log_group_name,
            logStreamName=log_stream_name,
            nextToken=next_token
        )
    else:
        response = client.get_log_events(
            logGroupName=log_group_name,
            logStreamName=log_stream_name,
            startTime=start_time
        )

    if 'nextForwardToken' in response:
        next_token = response['nextForwardToken']
    events = response['events']
    for event in events:
        print(event['message'])
    print(len(events), "events")
    
    return response


def main():
    response = startJob()
    job = response["jobArn"]
    status = ""
    
    while not status in ["SUCCEEDED", "FAILED"]:
        status = getUpdate(job)
        time.sleep(5)

main()
0

There are 0 best solutions below