How to find where an S3 multipart upload is failing in Python?

Question

I am implementing a cron job that will upload a large daily backup file to an S3 Bucket. It works most of the time, but every once in a while, I will check the bucket, and the file size is significantly smaller than the actual size.

It should be roughly 50GB, but the last time it happened, it showed 34GB. My main problem is that I am unsure of what error to try/catch.

I am still learning Python as I go, so bare with me.

from progress import ProgressPercentage  # class file progress.py
from slack import *  # function file for Slack notifications
import random
import glob
import os
import boto3
import botocore
from boto3.s3.transfer import TransferConfig

bucket = "my-s3-backup"
s3 = boto3.resource('s3')

# Grabbing the last file, and removing the full path from the string
pattern = "/path/to/backup/file/xb_*"
files = list(filter(os.path.isfile, glob.glob(pattern)))
files.sort(key=lambda x: os.path.getmtime(x))
file_to_upload = files[-1]
file_name = file_to_upload.replace('/path/to/backup/file/', '')
key_path = 'physical_db_backups/' + file_name

# Multipart upload function
def multi_part_upload():
    config = TransferConfig(multipart_threshold=1024 * 25,
                            max_concurrency=10,
                            multipart_chunksize=1024 * 25,
                            use_threads=True)

    try:
        s3.meta.client.upload_file(file_to_upload, bucket, key_path, Config=config,
                                   Callback=ProgressPercentage(file_to_upload))

        # Custom Slack notification to inform completion
        sendslacksuccess("Physical Backup to S3 Complete:\n" + file_name)
    except botocore.exceptions.ClientError as error:
        
        # Custom Slack notification to inform of failure
        sendslackerror("Physical Backup to S3 Failed:\n" + file_name + "\nError: " + error)


if __name__ == '__main__':
    multi_part_upload()

If the script is not "failing," but it's not uploading the complete file size, what exception am I trying to catch here? Should I log output somewhere?

I'm looking through the Botocore Exceptions documentation. I'm just unsure of what to try/catch with this.

For reference, here is the file size difference:

aws s3 ls --summarize --human-readable --recursive s3://my-s3-backup/physical_db_backups/

2022-05-07 14:31:28   50.7 GiB physical_db_backups/xb_202205070101.xb.zst
2022-05-08 12:48:07   50.8 GiB physical_db_backups/xb_202205080101.xb.zst
2022-05-09 01:30:04   34.2 GiB physical_db_backups/xb_202205090101.xb.zst <--- WRONG

Answer 1

Alright, since I was an idiot and didn't realize the file had not completed yet, I made a couple of changes.

1. I edited the cron to start later.

2. I have created logic to determine if the backup script is running.

3. I may incorporate additional checks to make sure the file exists, but for now this is a working POC that has been tested.

    from progress import ProgressPercentage  # class file progress.py
    from slack import *  # function file for Slack notifications
    import random
    from time import sleep
    import psutil
    import glob
    import os
    import boto3
    import botocore
    from boto3.s3.transfer import TransferConfig
    import logging
   
    bucket = "fsn-s3-backup"
    s3 = boto3.resource('s3')
    pattern = "/path/to/backup/file/xb_*"
    files = list(filter(os.path.isfile, glob.glob(pattern)))
    files.sort(key=lambda x: os.path.getmtime(x))
    file_to_upload = files[-1]
    file_name = file_to_upload.replace('/path/to/backup/file/', '')
    key_path = 'physical_db_backups/' + file_name
   
    logging.basicConfig(filename='/var/log/s3-backup.log', format='%(asctime)s - %(levelname)s - %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p', filemode='a')
    logger = logging.getLogger()
    logger.setLevel(logging.INFO)
   
   
    def multi_part_upload():
        config = TransferConfig(multipart_threshold=1024 * 25,
                                max_concurrency=10,
                                multipart_chunksize=1024 * 25,
                                use_threads=True)
   
        try:
            s3.meta.client.upload_file(file_to_upload, bucket, key_path, Config=config,
                                       Callback=ProgressPercentage(file_to_upload),
                                       ExtraArgs={'ContentType': 'application/zstd'})
            logger.info("Physical Backup to S3 Complete")
            sendslacksuccess("Physical Backup to S3 Complete:\n" + file_name)
        except botocore.exceptions.ClientError as error:
            logger.error("Physical Backup to S3 Failed: " + error)
            sendslackerror("Physical Backup to S3 Failed:\n" + file_name + "\nError: " + error)
   
   
    def checkIfProcessRunning(processName):
        for proc in psutil.process_iter():
            cmdline = proc.cmdline()
            if processName in cmdline:
                return True
        return False
   
   
    if __name__ == '__main__':
        backuprunning = True
        while backuprunning:
            logger.info("Checking if backup shell script is running")
            if checkIfProcessRunning('/path/to/physical_backup.sh'):
                logger.info("Backup shell script still running. Sleeping for 60s")
                sleep(60)
            else:
                backuprunning = False
                logger.info("Beginning multipart upload")
                multi_part_upload()