Resumable upload to Google Photos (Python) - why isn't resuming working?

Question

I wrote the code below to upload files to my Google Photos account. Because uploads of very large files tended to fail, I used the resumable upload REST protocol using the "single request" approach recommended at https://developers.google.com/photos/library/guides/resumable-uploads . However, when I run it, the upload either completes successfully with the initial put request or fails with the error

error HTTPSConnectionPool(host='photoslibrary.googleapis.com', port=443): Max 
retries exceeded with url: /v1/uploads?upload_id=...&upload_protocol=resumable 
(Caused by SSLError(SSLEOFError(8, 'EOF occurred in violation of protocol 
(_ssl.c:2393)')))

so the "resumable" loop is never entered. What am I misunderstanding?

    # Create resumable session
    headers = {
        'Authorization': f'Bearer {creds.token}',
        'X-Goog-Upload-Content-Type': content_type,
        'X-Goog-Upload-Protocol': 'resumable',
        'X-Goog-Upload-Command': 'start',
        'X-Goog-Upload-Raw-Size': str(file_size)
        }
    response = requests.post('https://photoslibrary.googleapis.com/v1/uploads', 
                             headers=headers)
    session_url = response.headers['X-Goog-Upload-URL']
    printstr(f"created session ... ")

    # Perform the upload
    with open(file_path, 'rb') as f:
        # Initiate the upload
        headers = {
            'Authorization': 'Bearer ' + creds.token,
            'Content-Length': str(file_size),
            'X-Goog-Upload-Content-Type': content_type,
            'X-Goog-Upload-Command': 'upload, finalize',
            'X-Goog-Upload-Offset': '0'
            }
        response = requests.post(session_url, data=f, headers=headers)
        printstr(f"uploading ...")

        # Keep resuming as necessary
        while response.status_code != 200:  # THIS LOOP IS NEVER ENTERED - WHY?
            # Determine how much has been uploaded
            query_headers = { 
                'Authorization': 'Bearer ' + creds.token,
                'Content-Length': str(0),
                'X-Goog-Upload-Command': 'query'
                }
            printstr(f"put broke status {response.status_code} ... ")
            query_response = requests.post(session_url, headers=query_headers)
            if query_response.status_code != 200:
                raise RuntimeError(f"Media upload query failed status {response.status_code}")
            offset = int(response.headers['X-Goog-Upload-Size-Received'])
            printstr("resuming at {offset} ... ")
            # Resume upload where it left off
            f.seek(offset)
            headers['X-Goog-Upload-Offset'] = str(offset)
            response = requests.put(session_url, headers=headers, data=f.read(file_size - offset))

Answer 1

Well, no answers here, but after a lot of experimentation I figured it out. Some resumable errors throw an exception but others do not. A solution is to check for a non-200 exit code for errors that don't throw exceptions and then throw an exception directly, and handle both cases in the except: section.

response = requests.post(session_url, data=data, headers=headers)
                    if response.status_code != 200:
                        raise RuntimeError(f"non-200 response {response.status_code}")

Here's my full code that others might find useful.

import os
import pickle
import requests
import argparse  
from google_auth_oauthlib.flow import InstalledAppFlow
from google.auth.transport.requests import Request
import mimetypes
import sys

DryRun = False
Debug = False


def printstr(s):
    print(s, end='')
    sys.stdout.flush()


def authenticate():
    creds = None
    SCOPES = ['https://www.googleapis.com/auth/photoslibrary.sharing']
    os.makedirs(os.path.expanduser('~/.credentials/gphotos'), exist_ok=True)
    CLIENT_SECRET_FILE = os.path.expanduser('~/.credentials/gphotos/client_secret.json')
    TOKEN_FILE = os.path.expanduser('~/.credentials/gphotos/token.pickle')
    if os.path.exists(TOKEN_FILE):
        with open(TOKEN_FILE, 'rb') as token:
            creds = pickle.load(token)
    if not creds or not creds.valid:
        if creds and creds.expired and creds.refresh_token:
            creds.refresh(Request())
        else:
            flow = InstalledAppFlow.from_client_secrets_file(CLIENT_SECRET_FILE, SCOPES)
            creds = flow.run_local_server(port=0)
        with open(TOKEN_FILE, 'wb') as token:
            pickle.dump(creds, token)
    return creds

def create_album(creds, album_name, collab):
    if DryRun:
        print(f"Dry run created album {album_name}")
        return { 'id':0, 'shareableURL': 'NO_URL' }
    try:
        headers = { 
            'Authorization': 'Bearer ' + creds.token,
            }
        json_body = {
                "album": {
                    "title": album_name
                }
            }
        response = (requests.post(
            'https://photoslibrary.googleapis.com/v1/albums',   
            json=json_body, headers=headers)).json()
        printstr(f"Created album {album_name} ... ")
        albumID = response['id']
        json_body = {
                "sharedAlbumOptions": {
                    "isCollaborative": "true" if collab else "false",
                    "isCommentable": "true"
                }  
            }      
        response = (requests.post(
            'https://photoslibrary.googleapis.com/v1/albums/' + albumID + ':share',
            json=json_body, headers=headers)).json()
        print("sharing options set")
        shareableURL = response['shareInfo']['shareableUrl']
        return {'id':albumID, 'shareableURL': shareableURL}
    except Exception as error:
        print(f"An error occurred trying to create album {album_name}: {error}")
        raise RuntimeError('Create album failed') from error

def is_media(file_path):
    # Return mime type if is an image or video, otherwise False
    (type, _) = mimetypes.guess_type(file_path)
    if type and (str(type).startswith('image/') or str(type).startswith('video/')):
        return type
    else:
        return False

def upload_photos(creds, folder_paths, collab, chunk_factor):
    failures = []
    albums = []
    for folder_path in folder_paths:
        if not os.path.isdir(folder_path):
            print(f"Bad argument {folder_path}, it is not a folder")
            continue   
        for root, dirs, files in os.walk(folder_path):
            # sort dirs in place, so at next level they are visited alphabetically
            dirs.sort()
            # determine if root has media files and so will become an album
            mediaFiles = [ f for f in files if is_media(f) ]
            if len(mediaFiles)>0:
                # create an album corresponding to root
                mediaFiles.sort()
                album_name = os.path.basename(root)
                album_path = root
                try:
                    album = create_album(creds, album_name, collab)
                    album_id = album['id']
                    albums.append([album_name,album['shareableURL']])
                except Exception:
                    print(f"Continuing on to next album")
                    failures.append(('Album creation failure:', 
                                     album_name, 'from', album_path))
                    continue
                # upload the media into the album
                for file in mediaFiles:
                    file_path = os.path.join(album_path, file)
                    try:
                        upload_photo(creds, file_path, file, album_id, album_name, chunk_factor)
                    except Exception:
                        failures.append(['Media upload failure:', file, 'in', 
                                         album_name, 'from', file_path])
                        print("Continuing to next file")
    print(f"Number of failures: {len(failures)}")
    for failure in failures:
        print(' '.join(failure))
    print("HTML Index")
    albums.sort(reverse=True)
    for (name, url) in albums:
        print(f'<p><a target="_blank" href="{url}">{name}</a></p>')

def upload_photo(creds, file_path, file_name, album_id, album_name, chunk_factor):
    if DryRun:
        print(f"Dry run upload {file_name}")
        return
    try:
        file_size = os.path.getsize(file_path)
        content_type = is_media(file_name)
        printstr(f"Uploading {file_name} of type {content_type} of length {file_size} ... ")
 
        # Create resumable session
        headers = {
            'Authorization': f'Bearer {creds.token}',
            'X-Goog-Upload-Content-Type': content_type,
            'X-Goog-Upload-Protocol': 'resumable',
            'X-Goog-Upload-Command': 'start',
            'X-Goog-Upload-Raw-Size': str(file_size)
            }
        
        response = requests.post('https://photoslibrary.googleapis.com/v1/uploads', 
                                 headers=headers)
        session_url = response.headers['X-Goog-Upload-URL']

        chunk_size = int(response.headers['X-Goog-Upload-Chunk-Granularity']) * chunk_factor

        printstr(f"created session with chunk size {chunk_size} ... ")
        offset = 0
        final = False

        # Perform the upload
        exception_count = 0
        with open(file_path, 'rb') as f:
            chunk_number = 0
            while not final:
                chunk_number += 1
                data_size = min(chunk_size, file_size - offset)
                if offset + data_size >= file_size:
                    command = 'upload, finalize'
                else:
                    command = 'upload'
                f.seek(offset)
                if Debug and chunk_number == 2:
                    # Force an unexpected EOF error on second chunk
                    data = f.read(data_size - 100)
                else:
                    data = f.read(data_size)
                headers = {
                    'Authorization': 'Bearer ' + creds.token,
                    'Content-Length': str(data_size),
                    'X-Goog-Upload-Content-Type': content_type,
                    'X-Goog-Upload-Command': command,
                    'X-Goog-Upload-Offset': str(offset)
                    }
                
                try:
                    response = requests.post(session_url, data=data, headers=headers)
                    if response.status_code != 200:
                        raise RuntimeError(f"non-200 response {response.status_code}")
                    
                    printstr(f"uploaded chunk {chunk_number} ... ")
                    offset += data_size
                    if offset >= file_size:
                        final = True
                    continue
    
                except (RuntimeError, OSError, requests.exceptions.RequestException) as e:
                    printstr(f"error uploading chunk {chunk_number} ... ")
                    exception_count += 1
                    if exception_count > 4:
                        raise RuntimeError(f"Too many exceptions")
                    
                    query_headers = { 
                        'Authorization': 'Bearer ' + creds.token,
                        'Content-Length': "0",
                        'X-Goog-Upload-Command': 'query'
                        }
                    
                    query_response = requests.post(session_url, headers=query_headers)
                    if query_response.status_code != 200:
                        raise RuntimeError(f"Query failed with code {response.status_code}")
                                        
                    offset = int(query_response.headers['X-Goog-Upload-Size-Received'])
                    printstr(f"resuming at {offset} ... ")
        uploadToken = response.text
        
        # add the uploaded media to the album
        headers = {'Authorization': 'Bearer ' + creds.token}
        json_body = {
                'albumId':album_id, 
                'newMediaItems':[{
                    'description':"",
                    'simpleMediaItem':{'uploadToken':uploadToken}
                }]
            }
        response = requests.post(
            'https://photoslibrary.googleapis.com/v1/mediaItems:batchCreate', 
            json=json_body, headers=headers)
        print(f"added to album {album_name}")

    except Exception as error:
        print(f"\nCould not upload file {file_name} due to error {error}")
        raise RuntimeError('Media upload failed') from error

def main():
    parser = argparse.ArgumentParser(description='Upload photo albums to Google Photos.')
    parser.add_argument('-dryrun', action='store_true')
    parser.add_argument('-collab', action='store_true')
    parser.add_argument('-debug', action='store_true')
    parser.add_argument('-chunk', type=int, default=20)
    parser.add_argument('folderpaths', metavar='dirs', nargs='+', 
        type=os.path.abspath,
        help='One or more folders containing photos and/or subfolders')
    args = parser.parse_args()
    global Debug
    Debug = args.debug
    global DryRun
    DryRun = args.dryrun
    if DryRun:
        print(f"Dry run protocol")
    creds = authenticate()
    upload_photos(creds, args.folderpaths, args.collab, args.chunk )

if __name__ == '__main__':
    main()