I'm using boto3 to launch and monitor an AWS batch job. [This documentation][1] would suggest that, once the job is running, a call to describe_jobs on that job should get me a logStreamName field as part of the dictionary in the response this should be true once the job has reached the RUNNING state. However, it's actually appearing once the job is in the STARTING state - but if I try to access that log stream I get a "does not exist" error. Is this to be expected?
Here's the code, slightly modified to remove proprietary information:
import boto3, time
REGION = "us-west-2"
LOG_GROUP = "/aws/batch/job"
client = boto3.client('batch', region_name=REGION)
def startJob():
response = client.submit_job(
jobName='kens_boto3_test',
jobQueue='blast-queue',
jobDefinition='<PROPRIETARY REFERENCE REMOVED>',
containerOverrides={
'environment': [
{
'name': 'FILENAME_LIST_S3_URL',
'value': '',
},
{
'name': 'FILENAME',
'value': 'Seq1.fasta',
},
],
},
)
return response
counter = 0
def getUpdate(job):
global counter
why = ""
logName = ""
response2 = client.describe_jobs(jobs=[job])
jobInfo = response2["jobs"][0]
name = jobInfo["jobName"]
id = jobInfo["jobId"]
status = jobInfo["status"]
if "statusReason" in jobInfo:
why = jobInfo["statusReason"]
attempts = len(jobInfo["attempts"])
container = jobInfo["container"]
command = container["command"]
if "logName" in jobInfo:
logName = container["logStreamName"]
counter += 1
print(f'[{counter}] ', name)
if counter == 1:
print("command:", command)
print("id:", id)
print("status:", status, " :", why)
# print("attempts:", attempts)
print("logName:", logName)
print("")
if logName:
do_log_events(LOG_GROUP, logName)
return status
next_token = None
def do_log_events(log_group_name, log_stream_name, start_time=None):
global next_token
client = boto3.client('logs')
if start_time is None:
start_time = 0 # Start from the beginning of the log stream
if next_token:
response = client.get_log_events(
logGroupName=log_group_name,
logStreamName=log_stream_name,
nextToken=next_token
)
else:
response = client.get_log_events(
logGroupName=log_group_name,
logStreamName=log_stream_name,
startTime=start_time
)
if 'nextForwardToken' in response:
next_token = response['nextForwardToken']
events = response['events']
for event in events:
print(event['message'])
print(len(events), "events")
return response
def main():
response = startJob()
job = response["jobArn"]
status = ""
while not status in ["SUCCEEDED", "FAILED"]:
status = getUpdate(job)
time.sleep(5)
main()