File rotation and deletion on json files

Question

How can I implement fileRotation on json files. I know there exists a fileRotation handler in logging module of python but that is used to rotate and delete log files. My program is reading and writing using json module, although I can also use logging module to write in json form into the files but I don't think that is a good practice. Is there any other library I can use or will I have to write my own script to do rotation and deletion?

Answer 1

You can use the logging and json modules to achieve this as follows:

import logging
import json
import os
from logging.handlers import RotatingFileHandler

def rotate_json_file(file_path, max_bytes, backup_count):
    logger = logging.getLogger(__name__)
    logger.setLevel(logging.INFO)
    
    file_handler = RotatingFileHandler(file_path, mode='a', maxBytes=max_bytes, backupCount=backup_count)
    file_handler.setFormatter(logging.Formatter('%(message)s'))

    logger.addHandler(file_handler)
    
    json_data = {'key': 'value'}
    logger.info(json.dumps(json_data))

    logger.removeHandler(file_handler)
    file_handler.close()

    record = logging.makeLogRecord(json_data)
    
    if file_handler.shouldRollover(record):
        # Get the rotated file names
        rotated_files = [file_path] + [file_path + '.{}'.format(i) for i in range(1, backup_count + 1)]
        
        os.remove(rotated_files[-1])

        for i in range(len(rotated_files) - 1, 0, -1):
            os.rename(rotated_files[i - 1], rotated_files[i])

        open(file_path, 'a').close()

If you would like it to be Time based:

import logging
import json
import os
from logging.handlers import TimedRotatingFileHandler

def rotate_json_file(file_path, when, interval, backup_count):
    logger = logging.getLogger(__name__)
    logger.setLevel(logging.INFO)
    
    file_handler = TimedRotatingFileHandler(file_path, when=when, interval=interval, backupCount=backup_count)
    file_handler.setFormatter(logging.Formatter('%(message)s'))

    logger.addHandler(file_handler)
    
    json_data = {'key': 'value'}
    logger.info(json.dumps(json_data))

    logger.removeHandler(file_handler)
    file_handler.close()

    record = logging.makeLogRecord(json_data)
    
    if file_handler.shouldRollover(record):
        # Get the rotated file names
        rotated_files = [file_handler.baseFilename] + file_handler.getFilesToDelete()

        os.remove(rotated_files[-1])

        for i in range(len(rotated_files) - 1, 0, -1):
            os.rename(rotated_files[i - 1], rotated_files[i])

        open(file_handler.baseFilename, 'a').close()

Where we use the "TimedRotatingFileHandler" instead.