diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..53cb80e --- /dev/null +++ b/.env.example @@ -0,0 +1,24 @@ +# Wasabi S3 endpoint (optional — defaults to s3..wasabisys.com) +WASABI_ENDPOINT=s3.us-east-2.wasabisys.com + +# Wasabi region, e.g. us-east-2 (used to derive endpoint when WASABI_ENDPOINT is not set) +WASABI_REGION= + +# Wasabi S3 bucket name (used when RESTIC_REPOSITORY is not set) +WASABI_BUCKET= + +# AWS-compatible credentials for Wasabi +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= + +# Restic repository encryption password +RESTIC_PASSWORD= + +# Full restic repository string (optional — overrides bucket/prefix/region) +RESTIC_REPOSITORY= + +# Prefix (folder) inside bucket (optional) +RESTIC_PREFIX= + +# Path to JSON config file containing {"paths": [...]} (optional, backup subcommand only) +FILE_PATH_CONFIG_PATH= diff --git a/.gitignore b/.gitignore index fbd0003..1591045 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -.env.* \ No newline at end of file +.env.local +.config/ \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..f2ea16f --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,86 @@ +# AGENTS.md + +## Repo Overview + +- Purpose: run `restic backup` against a Wasabi S3-compatible repository. +- Main entry point: `travel-backup-script.py`. +- Dependencies are intentionally minimal: + - Python standard library + - `python-dotenv` + - External binary: `restic` must already be installed and available on `PATH` +- There are currently no tests, package metadata files, or CI config in this repo. + +## File Map + +- `travel-backup-script.py`: single-script CLI implementation. +- `requirements.txt`: Python dependency list (`python-dotenv` only). +- `README.md`: user-facing setup and usage docs. +- `.env.example`: template for environment variables. +- `.gitignore`: ignores `.env.*` files (except `.env.example`). + +## Current Script Behavior + +The script uses two subcommands: `init` and `backup`. + +1. Loads `.env.local` automatically via `load_dotenv(".env.local")`. +2. Parses CLI args into subcommands with shared repo flags and backup-specific source flags. +3. Resolves the repository string: + - `--repository` / `RESTIC_REPOSITORY` takes priority (validated against `s3:s3..wasabisys.com/...` format). + - Otherwise built from `--bucket` / `WASABI_BUCKET`, `--region` / `WASABI_REGION`, `WASABI_ENDPOINT`, and `--prefix` / `RESTIC_PREFIX`. +4. Constructs a minimal subprocess environment for `restic`: + - `WASABI_ENDPOINT`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `RESTIC_PASSWORD` + - `PATH`, `HOME` if present + - `SYSTEMROOT`, `SystemRoot`, `COMSPEC` (Windows-specific, required for DNS resolution and shell access) +5. For `init`: runs `restic init` against the resolved repository. +6. For `backup`: resolves input from `--source`, `--file`, or `FILE_PATH_CONFIG_PATH` env var, then runs `restic backup`. +7. Prints the repo, command, and redacted env status before optionally executing `restic`. + +## Known Drift / Things To Watch + +- The JSON config file must contain a top-level `paths` key with a non-empty array of non-empty strings. +- `--source` and `--file` are mutually exclusive; one of them or `FILE_PATH_CONFIG_PATH` is required for backup. +- `--region` has no default — it must be provided via CLI or `WASABI_REGION` when not using `--repository`. + +## Local Development Workflow + +Use this order when changing the repo: + +1. Read `travel-backup-script.py` first. Nearly all behavior lives there. +2. Treat `README.md` as partially authoritative only after comparing it to the script. +3. If you change CLI flags, env loading, or repository construction, update code, README, and `.env.example` in the same change. +4. Preserve secret-handling behavior. Do not print raw credentials. + +## Suggested Verification + +Because the repo has no automated tests, prefer lightweight verification: + +- Static review of argument parsing and env handling. +- Dry-run init: + - `python travel-backup-script.py init --bucket --region --dry-run` +- Dry-run backup with a direct source: + - `python travel-backup-script.py backup --source --bucket --region --dry-run` +- Dry-run backup with a JSON config file: + - `python travel-backup-script.py backup --file --bucket --region --dry-run` + +Avoid real backup runs unless the user explicitly asks for them and valid credentials are present. + +## Editing Guidelines + +- Keep the implementation as a simple single-file CLI unless the user asks for a larger refactor. +- Favor backwards-compatible flag changes where practical. +- Be careful with subprocess environment changes: + - `PATH` must remain available so `restic` can be found. + - `HOME` is intentionally forwarded when present. + - On Windows, `SYSTEMROOT` and `COMSPEC` are required for DNS resolution and shell access. +- Do not commit real `.env.local` or other secret files. +- If adding tests later, prefer small CLI/unit tests around: + - `build_repo` + - `load_sources` + - `resolve_repo` (priority order, validation) + - missing `--bucket` handling + - env minimization/redaction behavior + +## Agent Notes + +- Start discovery from the script, not the README. +- Keep `README.md`, `.env.example`, and the script aligned when flags change. diff --git a/README.md b/README.md index 66cb487..a667a08 100644 --- a/README.md +++ b/README.md @@ -1,61 +1,127 @@ # Wasabi Restic Backup Script -This script automates backups to a [Wasabi](https://wasabi.com/) (S3-compatible) bucket using [restic](https://restic.net/). -It supports environment-based secrets (via `.env` + [python-dotenv](https://pypi.org/project/python-dotenv/)), CLI overrides, and dry-run/verbose modes. - ---- +This script automates backups to a [Wasabi](https://wasabi.com/) S3-compatible bucket using [restic](https://restic.net/). +It loads secrets from `.env.local`, builds a restic repository string, and runs `restic backup` with redacted environment logging. ## Features -- Backup any file or directory to Wasabi S3 storage with restic. -- Secrets loaded from a `.env` file (no need to type passwords on the CLI). -- CLI arguments override `.env` and system environment variables. -- Verbose and dry-run modes for debugging. -- Environment variable redaction in output (so logs won’t leak secrets). - ---- +- Two subcommands: `init` (initialize a new repository) and `backup` (run a backup). +- Backup a single source path with `-s/--source`. +- Backup multiple source paths from a JSON config file with `-f/--file`. +- Load secrets from `.env.local` via [python-dotenv](https://pypi.org/project/python-dotenv/). +- Support a prebuilt restic repository via `--repository` or `RESTIC_REPOSITORY`. +- Redact sensitive environment values in output. +- Support dry-run mode for command verification. ## Requirements - Python 3.8+ -- [restic](https://restic.net/) installed and in your `PATH` -- `python-dotenv` installed: - ```bash - pip install python-dotenv +- [restic](https://restic.net/) installed and available on `PATH` +- Install dependencies: +```bash +pip install -r requirements.txt +``` -## Initialize the Repository (first run only) +## Environment Variables -restic -r s3:s3.[REGION].wasabisys.com/[BUCKET_NAME]/[PREFIX] init +The script loads `.env.local` automatically and uses these variables: +| Variable | Required | Description | +|---|---|---| +| `AWS_ACCESS_KEY_ID` | Yes | Wasabi access key | +| `AWS_SECRET_ACCESS_KEY` | Yes | Wasabi secret key | +| `RESTIC_PASSWORD` | Yes | Restic repository encryption password | +| `WASABI_ENDPOINT` | No | S3 endpoint (defaults to `s3..wasabisys.com`) | +| `WASABI_REGION` | No | Wasabi region, e.g. `us-east-2` (used to derive endpoint) | +| `WASABI_BUCKET` | No | Bucket name (used when `RESTIC_REPOSITORY` is not set) | +| `RESTIC_REPOSITORY` | No | Full restic repository string (overrides bucket/prefix/region) | +| `RESTIC_PREFIX` | No | Prefix (folder) inside bucket | +| `FILE_PATH_CONFIG_PATH` | No | Path to JSON config file (backup subcommand only) | + +Only `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `RESTIC_PASSWORD`, and `WASABI_ENDPOINT` are passed to the restic subprocess. The script builds a minimal environment to avoid leaking unrelated variables. + +Create a `RESTIC_PASSWORD` via `openssl rand -base64 32` for best security. + +## Subcommands + +### `init` — Initialize a New Repository + +```bash +python travel-backup-script.py init --bucket my-bucket --region us-east-2 --dry-run +``` + +### `backup` — Run a Backup + +```bash +python travel-backup-script.py backup --source /etc --bucket my-bucket --dry-run +``` + +## Repository Resolution + +The repository string is resolved in this order: + +1. `--repository` CLI flag or `RESTIC_REPOSITORY` env var (must match `s3:s3..wasabisys.com/[/]`). +2. Otherwise, built from `--bucket` / `WASABI_BUCKET`, `--region` / `WASABI_REGION`, `WASABI_ENDPOINT`, and `--prefix` / `RESTIC_PREFIX`. ## CLI Options -Option Description ---source, -s Source path to back up (required) ---bucket, -b Wasabi bucket name (required unless --repository used) ---endpoint, -e S3 endpoint (default: s3.wasabisys.com or from env) ---prefix, -p Path inside bucket (default: travel-backup) ---access-key Wasabi access key (overrides env/.env) ---secret-key Wasabi secret key (overrides env/.env) ---password, -P Restic password (overrides env/.env) ---repository, -r Full restic repository string (overrides bucket/endpoint/prefix) ---env-file Path to .env file (default: .env) ---dry-run Show command and env but do not execute ---verbose Show extra debug info +Shared options (both `init` and `backup`): -## Example Dry Run -python travel-backup-backup.py --source /etc --bucket my-bucket --dry-run --verbose +- `--bucket`: Wasabi bucket name (or set `WASABI_BUCKET`) +- `--prefix`: Prefix inside the bucket (or set `RESTIC_PREFIX`) +- `--region`: Wasabi region, e.g. `us-east-2` (or set `WASABI_REGION`) +- `--repository`: Full restic repository string (or set `RESTIC_REPOSITORY`) +- `--dry-run`: Print the command and redacted environment without running `restic` +Backup-only options: -## Run the Backup +- `-s`, `--source`: Single file or directory to back up +- `-f`, `--file`: Path to a JSON file containing `{"paths": ["..."]}` -python backup.py --source /path/to/data --bucket my-bucket +`-s/--source` and `-f/--file` are mutually exclusive. One of them (or the `FILE_PATH_CONFIG_PATH` env var) is required for backup. -**Example with overrides:** -python backup.py \ - --source ~/Documents \ - --bucket my-bucket \ - --prefix laptop-backups \ - --verbose +## JSON File Format +Example config: + +```json +{ + "paths": [ + "/path/to/Documents", + "/path/to/Pictures" + ] +} +``` + +## Examples + +Initialize a repository (dry run): + +```bash +python travel-backup-script.py init --bucket my-bucket --region us-east-2 --dry-run +``` + +Backup a single source (dry run): + +```bash +python travel-backup-script.py backup --source /etc --bucket my-bucket --dry-run +``` + +Backup from a JSON file (dry run): + +```bash +python travel-backup-script.py backup --file backup-paths.json --bucket my-bucket --dry-run +``` + +Use a custom prefix: + +```bash +python travel-backup-script.py backup --source ~/Documents --bucket my-bucket --prefix laptop-backups +``` + +Use a full repository string: + +```bash +python travel-backup-script.py backup --source ~/Documents --repository s3:s3.us-east-2.wasabisys.com/my-bucket/laptop-backups +``` diff --git a/travel-backup-script.py b/travel-backup-script.py index 83bc07a..b401340 100755 --- a/travel-backup-script.py +++ b/travel-backup-script.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 import argparse +import json import os +import re import subprocess import sys from dotenv import load_dotenv @@ -17,69 +19,113 @@ def build_repo(endpoint: str, bucket: str, prefix: str = None) -> str: parts.append(prefix.strip("/")) return "/".join(parts) -def run_restic(repo: str, source: str, dry_run: bool = False, env: dict = None): - """Run restic backup command.""" - cmd = [ - "restic", - "-r", repo, - "backup", - source, - ] - - print("Repository:", repo) - print("Command:", " ".join(cmd)) - print("Environment (redacted):") - - # Use provided env or fall back to current process env - if env is None: - env = os.environ.copy() +def load_sources(source: str = None, source_file: str = None) -> list[str]: + """Resolve backup sources from --source or a JSON file.""" + if source is not None: + return [source] - # Check which env vars are available in the environment we're passing to restic + with open(source_file, "r", encoding="utf-8") as fh: + config = json.load(fh) + + paths = config.get("paths") + if not isinstance(paths, list) or not paths: + raise ValueError('config file must contain a non-empty "paths" array') + if not all(isinstance(path, str) and path.strip() for path in paths): + raise ValueError('each entry in "paths" must be a non-empty string') + + return paths + +def print_env_status(env: dict): + """Print redacted environment variable status.""" env_vars = ("WASABI_ENDPOINT", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "RESTIC_PASSWORD") for key in env_vars: if key in env and env.get(key) is not None: print(f" {key}=***REDACTED***") else: print(f" {key}=NOT SET") - + +def run_restic(repo: str, sources: list[str], dry_run: bool = False, env: dict = None, source_file: str = None): + """Run restic backup command.""" + cmd = [ + "restic", + "-r", repo, + "backup", + *sources, + ] + + print("Repository:", repo) + print("Command:", " ".join(cmd)) + print("Environment (redacted):") + + # Use provided env or fall back to current process env + if env is None: + env = os.environ.copy() + + print_env_status(env) + + if dry_run: + if source_file: + print("Config file:", source_file) + print("Dry-run mode: not running restic.") + return 0 + + # Pass the provided environment explicitly to the restic subprocess + subprocess.check_call(cmd, env=env) + return 0 + +def run_restic_init(repo: str, dry_run: bool = False, env: dict = None): + """Run restic init command to initialize a new repository.""" + cmd = [ + "restic", + "-r", repo, + "init", + ] + + print("Initializing repository:", repo) + print("Command:", " ".join(cmd)) + print("Environment (redacted):") + + if env is None: + env = os.environ.copy() + + print_env_status(env) + if dry_run: print("Dry-run mode: not running restic.") return 0 - - # Pass the provided environment explicitly to the restic subprocess - return subprocess.call(cmd, env=env) -def main(): - # Load environment variables from .env.local (if present) - load_dotenv(".env.local") - - parser = argparse.ArgumentParser(description="Backup files to Wasabi S3 with restic") - parser.add_argument("--source", required=True, help="Path to file or directory to back up") - parser.add_argument("--bucket", help="Wasabi S3 bucket name") - parser.add_argument("--prefix", help="Prefix (folder) inside bucket", default=None) - parser.add_argument("--region", default="us-west-1", help="Wasabi region, e.g. us-west-1") - parser.add_argument("--repository", help="Full restic repository string (overrides bucket/prefix)") - parser.add_argument("--dry-run", action="store_true", help="Print command instead of running it") - - args = parser.parse_args() - - # Endpoint based on region (unless already set in env) - endpoint = os.getenv("WASABI_ENDPOINT", f"s3.{args.region}.wasabisys.com") - - # Build repo string - if args.repository: - repo = args.repository - else: - if not args.bucket: - print("error: --bucket is required when --repository is not provided") - return 2 - repo = build_repo(endpoint, args.bucket, args.prefix) - - # Capture the current process environment (including values loaded by load_dotenv) + subprocess.check_call(cmd, env=env) + return 0 + +def resolve_repo(args) -> str: + """Resolve the restic repository string from args and environment.""" + # CLI --repository takes priority, then RESTIC_REPOSITORY env var + repository = getattr(args, "repository", None) or os.getenv("RESTIC_REPOSITORY") + if repository: + if not re.match(r"^s3:s3\.[a-z0-9-]+\.wasabisys\.com/.+", repository): + print(f"error: repository '{repository}' does not match expected format: s3:s3..wasabisys.com/[/]") + return None + return repository + + bucket = getattr(args, "bucket", None) or os.getenv("WASABI_BUCKET") + if not bucket: + print("error: --bucket or WASABI_BUCKET is required when --repository / RESTIC_REPOSITORY is not provided") + return None + + region = getattr(args, "region", None) or os.getenv("WASABI_REGION") + if not region: + print("error: --region or WASABI_REGION is required when --repository / RESTIC_REPOSITORY is not provided") + return None + + endpoint = os.getenv("WASABI_ENDPOINT", f"s3.{region}.wasabisys.com") + # CLI --prefix takes priority, then RESTIC_PREFIX env var + prefix = getattr(args, "prefix", None) or os.getenv("RESTIC_PREFIX") + return build_repo(endpoint, bucket, prefix) + +def build_minimal_env() -> dict: + """Build a minimal env dict to pass to restic subprocesses.""" proc_env = os.environ.copy() - # Build a minimal env dict to pass to restic. Include only the keys restic needs - # plus PATH and HOME so the restic binary can be resolved and user context is preserved. needed_keys = ("WASABI_ENDPOINT", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "RESTIC_PASSWORD") minimal_env = {} for k in needed_keys: @@ -87,16 +133,67 @@ def main(): if v is not None: minimal_env[k] = v - # Ensure WASABI_ENDPOINT is provided (use computed endpoint fallback) - if "WASABI_ENDPOINT" not in minimal_env or not minimal_env.get("WASABI_ENDPOINT"): - minimal_env["WASABI_ENDPOINT"] = endpoint + # Preserve system env vars needed by subprocesses + # PATH/HOME: find restic and resolve home directory + # SYSTEMROOT/COMSPEC: required on Windows for DNS resolution and shell access + system_keys = ("PATH", "HOME", "SYSTEMROOT", "SystemRoot", "COMSPEC") + for k in system_keys: + v = proc_env.get(k) + if v is not None: + minimal_env[k] = v - # Preserve PATH and HOME so subprocess can find restic and use user's home directory - minimal_env["PATH"] = proc_env.get("PATH", "") - if proc_env.get("HOME") is not None: - minimal_env["HOME"] = proc_env.get("HOME") + return minimal_env - return run_restic(repo, args.source, args.dry_run, env=minimal_env) +def main(): + # Load environment variables from .env.local (if present) + load_dotenv(".env.local") + + parser = argparse.ArgumentParser(description="Backup files to Wasabi S3 with restic") + subparsers = parser.add_subparsers(dest="command", required=True) + + # Shared repo arguments + repo_args = argparse.ArgumentParser(add_help=False) + repo_args.add_argument("--bucket", help="Wasabi S3 bucket name (or set WASABI_BUCKET)") + repo_args.add_argument("--prefix", help="Prefix (folder) inside bucket (or set RESTIC_PREFIX)", default=None) + repo_args.add_argument("--region", default=None, help="Wasabi region, e.g. us-west-1 (or set WASABI_REGION)") + repo_args.add_argument("--repository", help="Full restic repository string (or set RESTIC_REPOSITORY)") + repo_args.add_argument("--dry-run", action="store_true", help="Print command instead of running it") + + # init subcommand + subparsers.add_parser("init", parents=[repo_args], help="Initialize a new restic repository") + + # backup subcommand + backup_parser = subparsers.add_parser("backup", parents=[repo_args], help="Run a backup") + source_group = backup_parser.add_mutually_exclusive_group(required=False) + source_group.add_argument("--source", "-s", help="Path to file or directory to back up") + source_group.add_argument("--file", "-f", help='Path to a JSON config file containing {"paths": [...]} (or set FILE_PATH_CONFIG_PATH)') + + args = parser.parse_args() + + repo = resolve_repo(args) + if repo is None: + return 2 + + minimal_env = build_minimal_env() + + try: + if args.command == "init": + return run_restic_init(repo, args.dry_run, env=minimal_env) + + # backup command — resolve file path from CLI or env var + source_file = args.file or os.getenv("FILE_PATH_CONFIG_PATH") + if not args.source and not source_file: + print("error: --source, --file, or FILE_PATH_CONFIG_PATH is required for backup") + return 2 + + sources = load_sources(args.source, source_file) + return run_restic(repo, sources, args.dry_run, env=minimal_env, source_file=source_file) + except subprocess.CalledProcessError as exc: + print(f"error: restic exited with code {exc.returncode}: {exc}", file=sys.stderr) + return exc.returncode + except (OSError, json.JSONDecodeError, ValueError) as exc: + print(f"error: {exc}") + return 2 if __name__ == "__main__": - sys.exit(main()) \ No newline at end of file + sys.exit(main())