Automated S3 Backup Tool
This guide demonstrates how to automate file backups to Amazon S3 using the boto3 library. It supports backing up individual files or recursively uploading entire directories while preserving the folder structure.
Modules Used:
boto3: The Amazon Web Services (AWS) SDK for Python.botocore: Used for handling AWS-specific exceptions.- argparse: To handle command-line arguments.
- pathlib: For robust file path handling.
Prerequisites
- AWS Account: You need an active AWS account.
- S3 Bucket: Create a bucket in S3 (e.g.,
my-backup-bucket). - IAM User: Create an IAM user with
AmazonS3FullAccess(or specific write permissions). - Configuration: Configure your credentials locally.
- Install AWS CLI:
pip install awscli - Run:
aws configure(Enter Key ID, Secret Key, Region).
- Install AWS CLI:
Installation
pip install boto3The Code
Save this as s3_backup.py.
import boto3
from botocore.exceptions import ClientError
import argparse
import os
from pathlib import Path
import sys
def upload_file(s3_client, file_name, bucket, object_name):
"""Uploads a single file to S3."""
try:
s3_client.upload_file(file_name, bucket, object_name)
print(f"Uploaded: {file_name} -> s3://{bucket}/{object_name}")
return True
except ClientError as e:
print(f"Error uploading {file_name}: {e}")
return False
def backup_to_s3(source, bucket, prefix=""):
source_path = Path(source).resolve()
if not source_path.exists():
print(f"Error: Source '{source}' not found.")
return
# Initialize S3 client once
s3 = boto3.client('s3')
files_to_upload = []
if source_path.is_file():
# Single file backup
# Use the filename as the object key, optionally prepended with a prefix
s3_key = os.path.join(prefix, source_path.name).replace("\\", "/")
files_to_upload.append((str(source_path), s3_key))
else:
# Directory backup (Recursive)
print(f"Scanning '{source}'...")
for file_path in source_path.rglob("*"):
if file_path.is_file():
# Calculate relative path to maintain folder structure in S3
# e.g., /home/user/data/img.png -> data/img.png
rel_path = file_path.relative_to(source_path.parent)
s3_key = os.path.join(prefix, rel_path).replace("\\", "/")
files_to_upload.append((str(file_path), s3_key))
print(f"Found {len(files_to_upload)} files to upload to bucket '{bucket}'.")
success_count = 0
for local_file, s3_key in files_to_upload:
if upload_file(s3, local_file, bucket, s3_key):
success_count += 1
print(f"\nBackup complete. Uploaded {success_count}/{len(files_to_upload)} files.")
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="S3 Backup Utility")
parser.add_argument("source", help="Local file or directory to backup")
parser.add_argument("bucket", help="Target S3 Bucket name")
parser.add_argument("--prefix", default="", help="S3 Folder prefix (optional)")
args = parser.parse_args()
backup_to_s3(args.source, args.bucket, args.prefix)Usage
# Backup a single file
python s3_backup.py important_doc.pdf my-backup-bucket
# Backup a directory to a specific folder in S3
python s3_backup.py ./projects my-backup-bucket --prefix backups/2023