Python – Upload latest via S3

The following python script uploads a file and directory via S3. The file and directory must match a certain pattern.

If the file or folder do not exist, an error is returned.

import os
import datetime
import re
import subprocess

def get_newest_in_current_day(path):
    current_day_files = []
    current_day_dirs = []

    current_day = datetime.datetime.now()
    pattern = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z(\.pbm\.json)?$")

    for name in os.listdir(path):
        # Skip names that do not match the pattern
        if not pattern.match(name):
            continue

        full_path = os.path.join(path, name)
        timestamp_str = name.split('T')[0]  # Extract the date part from the name
        timestamp = datetime.datetime.strptime(timestamp_str, "%Y-%m-%d")

        # Check if it's the current day
        if timestamp.date() == current_day.date():
            if os.path.isfile(full_path):
                current_day_files.append((timestamp, name))
            elif os.path.isdir(full_path):
                current_day_dirs.append((timestamp, name))

    # Sort by timestamp (newest first) and get the first name
    current_day_files.sort(reverse=True)
    current_day_dirs.sort(reverse=True)

    newest_file = current_day_files[0][1] if current_day_files else None
    newest_dir = current_day_dirs[0][1] if current_day_dirs else None

    # If both the file and directory exist, upload them
    if newest_file and newest_dir:
        s3_target_path = "/weekly-prd/database/"
        s3_config_path = "./s3config.txt"
        subprocess.call(["s3cmd", "put", "--config", s3_config_path, os.path.join(path, newest_file), s3_target_path])
        subprocess.call(["s3cmd", "put", "--recursive", "--config", s3_config_path, os.path.join(path, newest_dir), s3_target_path])
        print("File and directory were uploaded.")
    else:
        print("Nothing was uploaded.")

    return newest_file, newest_dir


path = "/mnt/backups/0300/rs"
get_newest_in_current_day(path)