{"id":55,"date":"2023-05-16T15:14:45","date_gmt":"2023-05-16T15:14:45","guid":{"rendered":"http:\/\/77interactive.com\/?p=55"},"modified":"2023-10-06T18:08:53","modified_gmt":"2023-10-06T18:08:53","slug":"python-upload-sunday-files-via-s3","status":"publish","type":"post","link":"http:\/\/77interactive.com\/?p=55","title":{"rendered":"Python &#8211; Upload Sunday Files via S3"},"content":{"rendered":"\n<h2 class=\"wp-block-heading\">Upload Python Script<\/h2>\n\n\n\n<p>The following code uploads files and directories via S3. This version gets files and directories created on a Sunday<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>import os\nimport datetime\nimport re\nimport subprocess\nimport argparse\n\ndef get_newest_in_sunday(path, s3_target_path):\n    \"\"\"\n    This function scans a given directory for files and directories that were created on the last Sunday.\n    It then uploads the newest file and directory to a specified S3 target path using the s3cmd tool.\n\n    Args:\n        path (str): The path of the directory to scan.\n        s3_target_path (str): The S3 path to upload the file and directory to.\n\n    Returns:\n        Tuple&#91;str, str]: The names of the newest file and directory, if they exist.\n    \"\"\"\n    sunday_files = &#91;]\n    sunday_dirs = &#91;]\n\n    today = datetime.datetime.now()\n\n    # Calculate the last Sunday\n    days_since_sunday = (today.weekday() - 6) % 7\n    last_sunday = today - datetime.timedelta(days=days_since_sunday)\n\n    pattern = re.compile(r\"\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z(\\.pbm\\.json)?$\")\n\n    for name in os.listdir(path):\n        # Skip names that do not match the pattern\n        if not pattern.match(name):\n            continue\n\n        full_path = os.path.join(path, name)\n        timestamp_str = name.split('T')&#91;0]  # Extract the date part from the name\n        timestamp = datetime.datetime.strptime(timestamp_str, \"%Y-%m-%d\")\n\n        # Check if it's the last Sunday\n        if timestamp.date() == last_sunday.date():\n            if os.path.isfile(full_path):\n                sunday_files.append((timestamp, name))\n            elif os.path.isdir(full_path):\n                sunday_dirs.append((timestamp, name))\n\n    # Sort by timestamp (newest first) and get the first name\n    sunday_files.sort(reverse=True)\n    sunday_dirs.sort(reverse=True)\n\n    newest_file = sunday_files&#91;0]&#91;1] if sunday_files else None\n    newest_dir = sunday_dirs&#91;0]&#91;1] if sunday_dirs else None\n\n    # If both the file and directory exist, upload them\n    if newest_file and newest_dir:\n        s3_config_path = \".\/s3config.txt\"\n        subprocess.call(&#91;\"s3cmd\", \"put\", \"--config\", s3_config_path, os.path.join(path, newest_file), s3_target_path])\n        subprocess.call(&#91;\"s3cmd\", \"put\", \"--recursive\", \"--config\", s3_config_path, os.path.join(path, newest_dir), s3_target_path])\n        print(f\"Uploaded file: {newest_file}\")\n        print(f\"Uploaded directory: {newest_dir}\")\n    else:\n        print(\"Nothing was uploaded.\")\n\n    return newest_file, newest_dir\n\ndef main():\n    parser = argparse.ArgumentParser(description='Process some paths.')\n    parser.add_argument('path', type=str, help='The path of the directory to scan')\n    parser.add_argument('s3_target_path', type=str, help='The S3 path to upload the file and directory to')\n\n    args = parser.parse_args()\n\n    get_newest_in_sunday(args.path, args.s3_target_path)\n\n\nif __name__ == \"__main__\":\n    main()\n<\/code><\/pre>\n\n\n\n<h2 class=\"wp-block-heading\">How to call from Command line<\/h2>\n\n\n\n<p>We can then call the function from the command line like this:<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>python3 scan_upload.py \/mnt\/backups\/server\/rs \/weekly-backup-folder\/\n\n<\/code><\/pre>\n\n\n\n<h2 class=\"wp-block-heading\">First of the month version<\/h2>\n\n\n\n<p>The following version returns values for the first of the month.<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>import os\r\nimport datetime\r\nimport re\r\nimport subprocess\r\nimport argparse\r\n\r\ndef get_newest_from_first_of_month(path, s3_target_path):\r\n    \"\"\"\r\n    This function scans a given directory for files and directories that were created on the first day of the current month.\r\n    It then uploads the newest file and directory to a specified S3 target path using the s3cmd tool.\r\n\r\n    Args:\r\n        path (str): The path of the directory to scan.\r\n        s3_target_path (str): The S3 path to upload the file and directory to.\r\n\r\n    Returns:\r\n        Tuple&#91;str, str]: The names of the newest file and directory, if they exist.\r\n    \"\"\"\r\n    monthly_files = &#91;]\r\n    monthly_dirs = &#91;]\r\n\r\n    today = datetime.datetime.now()\r\n\r\n    # Calculate the first day of the current month\r\n    first_of_month = today.replace(day=1)\r\n\r\n    pattern = re.compile(r\"\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z(\\.pbm\\.json)?$\")\r\n\r\n    for name in os.listdir(path):\r\n        # Skip names that do not match the pattern\r\n        if not pattern.match(name):\r\n            continue\r\n\r\n        full_path = os.path.join(path, name)\r\n        timestamp_str = name.split('T')&#91;0]  # Extract the date part from the name\r\n        timestamp = datetime.datetime.strptime(timestamp_str, \"%Y-%m-%d\")\r\n\r\n        # Check if it's the first day of the current month\r\n        if timestamp.date() == first_of_month.date():\r\n            if os.path.isfile(full_path):\r\n                monthly_files.append((timestamp, name))\r\n            elif os.path.isdir(full_path):\r\n                monthly_dirs.append((timestamp, name))\r\n\r\n    # Sort by timestamp (newest first) and get the first name\r\n    monthly_files.sort(reverse=True)\r\n    monthly_dirs.sort(reverse=True)\r\n\r\n    newest_file = monthly_files&#91;0]&#91;1] if monthly_files else None\r\n    newest_dir = monthly_dirs&#91;0]&#91;1] if monthly_dirs else None\r\n\r\n    # If both the file and directory exist, upload them\r\n    if newest_file and newest_dir:\r\n        s3_config_path = \".\/s3config.txt\"\r\n        subprocess.call(&#91;\"s3cmd\", \"put\", \"--config\", s3_config_path, os.path.join(path, newest_file), s3_target_path])\r\n        subprocess.call(&#91;\"s3cmd\", \"put\", \"--recursive\", \"--config\", s3_config_path, os.path.join(path, newest_dir), s3_target_path])\r\n        print(f\"Uploaded file: {newest_file}\")\r\n        print(f\"Uploaded directory: {newest_dir}\")\r\n    else:\r\n        print(\"Nothing was uploaded.\")\r\n\r\n    return newest_file, newest_dir\r\n\r\ndef main():\r\n    parser = argparse.ArgumentParser(description='Process some paths.')\r\n    parser.add_argument('path', type=str, help='The path of the directory to scan')\r\n    parser.add_argument('s3_target_path', type=str, help='The S3 path to upload the file and directory to')\r\n\r\n    args = parser.parse_args()\r\n\r\n    get_newest_from_first_of_month(args.path, args.s3_target_path)\r\n\r\nif __name__ == \"__main__\":\r\n    main()\r\n<\/code><\/pre>\n","protected":false},"excerpt":{"rendered":"<p>Upload Python Script The following code uploads files and directories via S3. This version gets files and directories created on a Sunday How to call from Command line We can then call the function from the command line like this: First of the month version The following version returns values for the first of the [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"closed","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[2],"tags":[10,15,3],"class_list":["post-55","post","type-post","status-publish","format-standard","hentry","category-python","tag-backup","tag-bash","tag-python"],"yoast_head":"<!-- This site is optimized with the Yoast SEO plugin v26.9 - https:\/\/yoast.com\/product\/yoast-seo-wordpress\/ -->\n<title>Python - Upload Sunday Files via S3 - 77 Interactive<\/title>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"http:\/\/77interactive.com\/?p=55\" \/>\n<meta property=\"og:locale\" content=\"en_US\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"Python - Upload Sunday Files via S3 - 77 Interactive\" \/>\n<meta property=\"og:description\" content=\"Upload Python Script The following code uploads files and directories via S3. This version gets files and directories created on a Sunday How to call from Command line We can then call the function from the command line like this: First of the month version The following version returns values for the first of the [&hellip;]\" \/>\n<meta property=\"og:url\" content=\"http:\/\/77interactive.com\/?p=55\" \/>\n<meta property=\"og:site_name\" content=\"77 Interactive\" \/>\n<meta property=\"article:published_time\" content=\"2023-05-16T15:14:45+00:00\" \/>\n<meta property=\"article:modified_time\" content=\"2023-10-06T18:08:53+00:00\" \/>\n<meta name=\"author\" content=\"Rudy\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:label1\" content=\"Written by\" \/>\n\t<meta name=\"twitter:data1\" content=\"Rudy\" \/>\n\t<meta name=\"twitter:label2\" content=\"Est. reading time\" \/>\n\t<meta name=\"twitter:data2\" content=\"4 minutes\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"Article\",\"@id\":\"http:\/\/77interactive.com\/?p=55#article\",\"isPartOf\":{\"@id\":\"http:\/\/77interactive.com\/?p=55\"},\"author\":{\"name\":\"Rudy\",\"@id\":\"http:\/\/77interactive.com\/#\/schema\/person\/0e61d2a984b8304618026b207e6121e9\"},\"headline\":\"Python &#8211; Upload Sunday Files via S3\",\"datePublished\":\"2023-05-16T15:14:45+00:00\",\"dateModified\":\"2023-10-06T18:08:53+00:00\",\"mainEntityOfPage\":{\"@id\":\"http:\/\/77interactive.com\/?p=55\"},\"wordCount\":63,\"keywords\":[\"backup\",\"bash\",\"python\"],\"articleSection\":[\"Python\"],\"inLanguage\":\"en-US\"},{\"@type\":\"WebPage\",\"@id\":\"http:\/\/77interactive.com\/?p=55\",\"url\":\"http:\/\/77interactive.com\/?p=55\",\"name\":\"Python - Upload Sunday Files via S3 - 77 Interactive\",\"isPartOf\":{\"@id\":\"http:\/\/77interactive.com\/#website\"},\"datePublished\":\"2023-05-16T15:14:45+00:00\",\"dateModified\":\"2023-10-06T18:08:53+00:00\",\"author\":{\"@id\":\"http:\/\/77interactive.com\/#\/schema\/person\/0e61d2a984b8304618026b207e6121e9\"},\"breadcrumb\":{\"@id\":\"http:\/\/77interactive.com\/?p=55#breadcrumb\"},\"inLanguage\":\"en-US\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"http:\/\/77interactive.com\/?p=55\"]}]},{\"@type\":\"BreadcrumbList\",\"@id\":\"http:\/\/77interactive.com\/?p=55#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"Home\",\"item\":\"http:\/\/77interactive.com\/\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"Python &#8211; Upload Sunday Files via S3\"}]},{\"@type\":\"WebSite\",\"@id\":\"http:\/\/77interactive.com\/#website\",\"url\":\"http:\/\/77interactive.com\/\",\"name\":\"77 Interactive\",\"description\":\"Rudy&#039;s Code snippets\",\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"http:\/\/77interactive.com\/?s={search_term_string}\"},\"query-input\":{\"@type\":\"PropertyValueSpecification\",\"valueRequired\":true,\"valueName\":\"search_term_string\"}}],\"inLanguage\":\"en-US\"},{\"@type\":\"Person\",\"@id\":\"http:\/\/77interactive.com\/#\/schema\/person\/0e61d2a984b8304618026b207e6121e9\",\"name\":\"Rudy\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"en-US\",\"@id\":\"http:\/\/77interactive.com\/#\/schema\/person\/image\/\",\"url\":\"https:\/\/secure.gravatar.com\/avatar\/e336b9aecd39b40691ff8ccfcd68506415072dbe8caffc0485b94a1bc22b774d?s=96&d=mm&r=g\",\"contentUrl\":\"https:\/\/secure.gravatar.com\/avatar\/e336b9aecd39b40691ff8ccfcd68506415072dbe8caffc0485b94a1bc22b774d?s=96&d=mm&r=g\",\"caption\":\"Rudy\"},\"url\":\"http:\/\/77interactive.com\/?author=1\"}]}<\/script>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"Python - Upload Sunday Files via S3 - 77 Interactive","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"http:\/\/77interactive.com\/?p=55","og_locale":"en_US","og_type":"article","og_title":"Python - Upload Sunday Files via S3 - 77 Interactive","og_description":"Upload Python Script The following code uploads files and directories via S3. This version gets files and directories created on a Sunday How to call from Command line We can then call the function from the command line like this: First of the month version The following version returns values for the first of the [&hellip;]","og_url":"http:\/\/77interactive.com\/?p=55","og_site_name":"77 Interactive","article_published_time":"2023-05-16T15:14:45+00:00","article_modified_time":"2023-10-06T18:08:53+00:00","author":"Rudy","twitter_card":"summary_large_image","twitter_misc":{"Written by":"Rudy","Est. reading time":"4 minutes"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"Article","@id":"http:\/\/77interactive.com\/?p=55#article","isPartOf":{"@id":"http:\/\/77interactive.com\/?p=55"},"author":{"name":"Rudy","@id":"http:\/\/77interactive.com\/#\/schema\/person\/0e61d2a984b8304618026b207e6121e9"},"headline":"Python &#8211; Upload Sunday Files via S3","datePublished":"2023-05-16T15:14:45+00:00","dateModified":"2023-10-06T18:08:53+00:00","mainEntityOfPage":{"@id":"http:\/\/77interactive.com\/?p=55"},"wordCount":63,"keywords":["backup","bash","python"],"articleSection":["Python"],"inLanguage":"en-US"},{"@type":"WebPage","@id":"http:\/\/77interactive.com\/?p=55","url":"http:\/\/77interactive.com\/?p=55","name":"Python - Upload Sunday Files via S3 - 77 Interactive","isPartOf":{"@id":"http:\/\/77interactive.com\/#website"},"datePublished":"2023-05-16T15:14:45+00:00","dateModified":"2023-10-06T18:08:53+00:00","author":{"@id":"http:\/\/77interactive.com\/#\/schema\/person\/0e61d2a984b8304618026b207e6121e9"},"breadcrumb":{"@id":"http:\/\/77interactive.com\/?p=55#breadcrumb"},"inLanguage":"en-US","potentialAction":[{"@type":"ReadAction","target":["http:\/\/77interactive.com\/?p=55"]}]},{"@type":"BreadcrumbList","@id":"http:\/\/77interactive.com\/?p=55#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"Home","item":"http:\/\/77interactive.com\/"},{"@type":"ListItem","position":2,"name":"Python &#8211; Upload Sunday Files via S3"}]},{"@type":"WebSite","@id":"http:\/\/77interactive.com\/#website","url":"http:\/\/77interactive.com\/","name":"77 Interactive","description":"Rudy&#039;s Code snippets","potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"http:\/\/77interactive.com\/?s={search_term_string}"},"query-input":{"@type":"PropertyValueSpecification","valueRequired":true,"valueName":"search_term_string"}}],"inLanguage":"en-US"},{"@type":"Person","@id":"http:\/\/77interactive.com\/#\/schema\/person\/0e61d2a984b8304618026b207e6121e9","name":"Rudy","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"http:\/\/77interactive.com\/#\/schema\/person\/image\/","url":"https:\/\/secure.gravatar.com\/avatar\/e336b9aecd39b40691ff8ccfcd68506415072dbe8caffc0485b94a1bc22b774d?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/e336b9aecd39b40691ff8ccfcd68506415072dbe8caffc0485b94a1bc22b774d?s=96&d=mm&r=g","caption":"Rudy"},"url":"http:\/\/77interactive.com\/?author=1"}]}},"_links":{"self":[{"href":"http:\/\/77interactive.com\/index.php?rest_route=\/wp\/v2\/posts\/55","targetHints":{"allow":["GET"]}}],"collection":[{"href":"http:\/\/77interactive.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/77interactive.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/77interactive.com\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/77interactive.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=55"}],"version-history":[{"count":0,"href":"http:\/\/77interactive.com\/index.php?rest_route=\/wp\/v2\/posts\/55\/revisions"}],"wp:attachment":[{"href":"http:\/\/77interactive.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=55"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/77interactive.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=55"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/77interactive.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=55"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}