Cloud Run Jobs container exits before all the logging is complete

Question

When viewing the logs for a specific task for cloud run job:

For multiple tasks I see: Container called exit(0). which indicates the job ran successfully. However, it is missing bunch of logs before it. They are not visible even in log explorer, so it seems that the container exited without flushing out all the logs.

Here is the snippet from script for reference, which has not made any difference to ensure all the logs are displayed:

    try:
        logger.info("TESTS")
        task_params = get_task_params()
        run_task(TASK_INDEX, TASK_ATTEMPT, task_params)
        logger.info("Waiting for 5 seconds before exiting to allow logs to be flushed.")
        logging_config.flush_logs()
        time.sleep(5)  # Wait 5 seconds before exiting to allow logs to be flushed
    except Exception as err:
        message = f"Task #{TASK_INDEX}, Attempt #{TASK_ATTEMPT} failed: {str(err)}"
        logger.info(message)
        sys.exit(1)  # Non-zero exit code for failure

Only time I ever see logger.info("Waiting for 5 seconds before exiting to allow logs to be flushed.") is for very short running tasks.

Looking for suggestions on how to fix the logging.

EDIT: Details on how logging is setup in config.py

def setup_logging(config):
    # Instantiates a client
    client = google.cloud.logging.Client()
    # Retrieves a Cloud Logging handler based on the environment
    # you're running in and integrates the handler with the
    # Python logging module. By default this captures all logs
    # at INFO level and higher
    client.setup_logging()

    if not os.path.exists(os.path.dirname(config.LOG_FILE)):
        os.makedirs(os.path.dirname(config.LOG_FILE))

    log_formatter = logging.Formatter(
        "%(asctime)s %(levelname)s: %(message)s [in %(pathname)s:%(lineno)d]"
    )

    # Configure the file handler
    file_handler = RotatingFileHandler(
        config.LOG_FILE, maxBytes=1024 * 1024 * 10, backupCount=10
    )
    file_handler.setFormatter(log_formatter)
    file_handler.setLevel(getattr(logging, config.LOG_LEVEL.upper()))

    # Configure the stream handler
    stream_handler = logging.StreamHandler()
    stream_handler.setFormatter(log_formatter)
    stream_handler.setLevel(getattr(logging, config.LOG_LEVEL.upper()))

    # Set the log level and handlers for the logger
    logger = logging.getLogger()  # Define logger at the module level
    logger.setLevel(getattr(logging, config.LOG_LEVEL.upper()))
    logger.addHandler(file_handler)
    logger.addHandler(stream_handler)

    # Log the initialization
    logger.info("Logging is set up")


def flush_logs(config=configurations["default"]):
    logger = logging.getLogger()
    handlers = logger.handlers[:]
    for handler in handlers:
        handler.flush()
        handler.close()
        logger.removeHandler(handler)

    logger.info("All logging handlers have been flushed and closed.")

    logging.shutdown()


# Initialize logging with the desired configuration
setup_logging(configurations["default"])