diff --git a/rwm.conf.example b/rwm.conf.example index a31b410acf939fa0c7112697c538b54dcbf56594..44ccb4a175de91133dd17d1ada5df291919ffc57 100644 --- a/rwm.conf.example +++ b/rwm.conf.example @@ -1,9 +1,16 @@ +# all commands RWM_S3_ENDPOINT_URL: "" RWM_S3_ACCESS_KEY: "" RWM_S3_SECRET_KEY: "" +# rclone_crypt RWM_RCLONE_CRYPT_BUCKET: "rwmcrypt" RWM_RCLONE_CRYPT_PASSWORD: "" +# restic, backup RWM_RESTIC_BUCKET: "rwmcrypt" -RWM_RESTIC_PASSWORD: "" \ No newline at end of file +RWM_RESTIC_PASSWORD: "" + +# backup +RWM_BACKUPS: [] +RWM_RETENTION: [] diff --git a/rwm.py b/rwm.py index 777de44249ffb9d32323337c55a7881953234bbf..911509a610555a69e150aed2037bd26ed5d52f92 100755 --- a/rwm.py +++ b/rwm.py @@ -4,6 +4,7 @@ import base64 import logging import os +import shlex import subprocess import sys from argparse import ArgumentParser @@ -16,7 +17,7 @@ from cryptography.hazmat.backends import default_backend __version__ = "0.2" logger = logging.getLogger("rwm") -logger.setLevel(logging.DEBUG) +logger.setLevel(logging.INFO) def is_sublist(needle, haystack): @@ -44,7 +45,7 @@ def run_command(*args, **kwargs): "text": True, "encoding": "utf-8", }) - logger.debug("run_command, %s", (args, kwargs)) + logger.debug("run_command: %s", shlex.join(args[0])) return subprocess.run(*args, **kwargs, check=False) @@ -147,14 +148,69 @@ class RWM: } return run_command(["restic"] + args, env=env) + def backup(self, name): + """do restic backup from config""" -def main(argv=None): - """main""" + if self.restic_cmd(["cat", "config"]).returncode != 0: + if (proc := self.restic_cmd(["init"])).returncode != 0: + logger.error("failed to autoinitialize restic repository") + return proc + + conf = self.config["RWM_BACKUPS"][name] + + # restic backup + excludes = conf.get("excludes", []) + excludes = [x for pair in zip(["--exclude"] * len(excludes), excludes) for x in pair] + extras = conf.get("extras", []) + cmd_args = ["backup"] + extras + excludes + conf["filesdirs"] + + logger.info("running backup") + backup_proc = self.restic_cmd(cmd_args) + wrap_output(backup_proc) + if backup_proc.returncode != 0: + logger.error("backup failed, %s", backup_proc) + return backup_proc + + # restic forget prune + keeps = [] + for key, val in self.config.get("RWM_RETENTION", {}).items(): + keeps += [f"--{key}", val] + if not keeps: + logger.error("no retention policy found") + cmd_args = ["forget", "--prune"] + keeps + + logger.info("running forget prune") + forget_proc = self.restic_cmd(cmd_args) + wrap_output(forget_proc) + if forget_proc.returncode != 0: + logger.error("forget prune failed, %s", forget_proc) + return forget_proc + + return backup_proc + + +def configure_logging(debug): + """configure logger""" + + log_handler = logging.StreamHandler(sys.stdout) + log_handler.setFormatter( + logging.Formatter( + fmt="%(asctime)s %(name)s[%(process)d]: %(levelname)s %(message)s" + ) + ) + logger.addHandler(log_handler) + if debug: # pragma: no cover ; would reconfigure pylint environment + logger.setLevel(logging.DEBUG) + + +def parse_arguments(argv): + """parse arguments""" parser = ArgumentParser(description="restics3 worm manager") + parser.add_argument("--debug", action="store_true") parser.add_argument("--config", default="rwm.conf") - subparsers = parser.add_subparsers(title="commands", dest="command", required=False) + subparsers = parser.add_subparsers(title="commands", dest="command", required=False) subparsers.add_parser("version", help="show version") aws_cmd_parser = subparsers.add_parser("aws", help="aws command") aws_cmd_parser.add_argument("cmd_args", nargs="*") @@ -164,27 +220,43 @@ def main(argv=None): rcc_cmd_parser.add_argument("cmd_args", nargs="*") res_cmd_parser = subparsers.add_parser("restic", help="restic command") res_cmd_parser.add_argument("cmd_args", nargs="*") + backup_cmd_parser = subparsers.add_parser("backup", help="backup command") + backup_cmd_parser.add_argument("name", help="backup config name") + + return parser.parse_args(argv) - args = parser.parse_args(argv) + +def main(argv=None): + """main""" + + args = parse_arguments(argv) + configure_logging(args.debug) config = {} if args.config: config.update(get_config(args.config)) + logger.debug("config, %s", config) # assert config ? rwmi = RWM(config) if args.command == "version": print(__version__) + return 0 + + ret = -1 if args.command == "aws": - return wrap_output(rwmi.aws_cmd(args.cmd_args)) + ret = wrap_output(rwmi.aws_cmd(args.cmd_args)) if args.command == "rclone": - return wrap_output(rwmi.rclone_cmd(args.cmd_args)) + ret = wrap_output(rwmi.rclone_cmd(args.cmd_args)) if args.command == "rclone_crypt": - return wrap_output(rwmi.rclone_crypt_cmd(args.cmd_args)) + ret = wrap_output(rwmi.rclone_crypt_cmd(args.cmd_args)) if args.command == "restic": - return wrap_output(rwmi.restic_cmd(args.cmd_args)) + ret = wrap_output(rwmi.restic_cmd(args.cmd_args)) + if args.command == "backup": + ret = rwmi.backup(args.name).returncode - return 0 + logger.info("rwm finished with %s (ret %d)", "success" if ret == 0 else "errors", ret) + return ret if __name__ == "__main__": # pragma: nocover diff --git a/tests/test_default.py b/tests/test_default.py index 2b6d7614f1420e749be4304764e02991cef5540b..da52e2b378669bd8b1693f39ed51a099f208872d 100644 --- a/tests/test_default.py +++ b/tests/test_default.py @@ -40,7 +40,7 @@ def test_main(tmpworkdir: str): # pylint: disable=unused-argument """test main""" # optional and default config hanling - assert rwm_main([]) == 0 + assert rwm_main(["version"]) == 0 Path("rwm.conf").touch() assert rwm_main(["version"]) == 0 @@ -48,7 +48,10 @@ def test_main(tmpworkdir: str): # pylint: disable=unused-argument mock = Mock(return_value=CompletedProcess(args='dummy', returncode=0)) for item in ["aws", "rclone", "rclone_crypt", "restic"]: with patch.object(rwm.RWM, f"{item}_cmd", mock): - assert rwm_main([item]) == 0 + assert rwm_main([item, "dummy"]) == 0 + + with patch.object(rwm.RWM, "backup", mock): + assert rwm_main(["backup", "dummy"]) == 0 def test_aws_cmd(tmpworkdir: str, motoserver: str): # pylint: disable=unused-argument @@ -137,3 +140,130 @@ def test_restic_cmd(tmpworkdir: str, motoserver: str): # pylint: disable=unused assert trwm.restic_cmd(["init"]).returncode == 0 proc = trwm.restic_cmd(["cat", "config"]) assert "id" in json.loads(proc.stdout) + + +def _list_snapshots(trwm): + """test helper""" + + return json.loads(trwm.restic_cmd(["snapshots", "--json"]).stdout) + + +def _list_files(trwm, snapshot_id): + """test helper""" + + snapshot_ls = [ + json.loads(x) + for x in + trwm.restic_cmd(["ls", snapshot_id, "--json"]).stdout.splitlines() + ] + return [ + x["path"] for x in snapshot_ls + if (x["struct_type"] == "node" and x["type"] == "file") + ] + + +def test_backup(tmpworkdir: str, motoserver: str): # pylint: disable=unused-argument + """test backup command""" + + trwm = RWM({ + "RWM_S3_ENDPOINT_URL": motoserver, + "RWM_S3_ACCESS_KEY": "dummy", + "RWM_S3_SECRET_KEY": "dummy", + "RWM_RESTIC_BUCKET": "restictest", + "RWM_RESTIC_PASSWORD": "dummydummydummydummydummydummydummydummy", + "RWM_BACKUPS": { + "testcfg": { + "filesdirs": ["testdatadir/"], + "excludes": ["testfile_to_be_ignored"], + "extras": ["--tag", "dummytag"], + } + }, + "RWM_RETENTION": { + "keep-daily": "1" + } + }) + + Path("testdatadir").mkdir() + Path("testdatadir/testdata1.txt").write_text("dummydata", encoding="utf-8") + Path("testdatadir/testfile_to_be_ignored").write_text("dummydata", encoding="utf-8") + + assert trwm.backup("testcfg").returncode == 0 + + snapshots = _list_snapshots(trwm) + assert len(snapshots) == 1 + snapshot_files = _list_files(trwm, snapshots[0]["id"]) + assert "/testdatadir/testdata1.txt" in snapshot_files + + +def test_backup_autoinit(): # pylint: disable=unused-argument + """mocked error handling test""" + + mock = Mock(return_value=CompletedProcess(args='dummy', returncode=1)) + with patch.object(rwm.RWM, "restic_cmd", mock): + RWM({}).backup("dummy") + + +def test_backup_excludes(tmpworkdir: str, motoserver: str): # pylint: disable=unused-argument + """test backup command""" + + trwm = RWM({ + "RWM_S3_ENDPOINT_URL": motoserver, + "RWM_S3_ACCESS_KEY": "dummy", + "RWM_S3_SECRET_KEY": "dummy", + "RWM_RESTIC_BUCKET": "restictest", + "RWM_RESTIC_PASSWORD": "dummydummydummydummydummydummydummydummy", + "RWM_BACKUPS": { + "testcfg": { + "filesdirs": ["testdatadir"], + "excludes": ["proc", "*.ignored"], + "extras": ["--tag", "dummytag"], + } + } + }) + + Path("testdatadir").mkdir() + Path("testdatadir/etc").mkdir() + Path("testdatadir/etc/config").write_text("dummydata", encoding="utf-8") + Path("testdatadir/etc/config2").write_text("dummydata", encoding="utf-8") + Path("testdatadir/etc/config3.ignored").write_text("dummydata", encoding="utf-8") + Path("testdatadir/proc").mkdir() + Path("testdatadir/proc/to_be_also_excluded").write_text("dummydata", encoding="utf-8") + + assert trwm.backup("testcfg").returncode == 0 + + snapshots = _list_snapshots(trwm) + assert len(snapshots) == 1 + snapshot_files = _list_files(trwm, snapshots[0]["id"]) + assert "/testdatadir/etc/config" in snapshot_files + assert "/testdatadir/etc/config2" in snapshot_files + + +def test_backup_error_handling(tmpworkdir: str, motoserver: str): # pylint: disable=unused-argument + """test backup command err cases""" + + mock = Mock(side_effect=[ + CompletedProcess(args='dummy', returncode=0), # autoinit + CompletedProcess(args='dummy', returncode=11) # backup + ]) + with patch.object(rwm.RWM, "restic_cmd", mock): + trwm = RWM({ + "RWM_BACKUPS": { + "dummycfg": {"filesdirs": ["dummydir"]} + } + }) + proc = trwm.backup("dummycfg") + assert proc.returncode == 11 + + mock = Mock(side_effect=[ + CompletedProcess(args='dummy', returncode=0), # autoinit + CompletedProcess(args='dummy', returncode=0), # backup + CompletedProcess(args='dummy', returncode=12) # forget + ]) + with patch.object(rwm.RWM, "restic_cmd", mock): + trwm = RWM({ + "RWM_BACKUPS": { + "dummycfg": {"filesdirs": ["dummydir"]} + } + }) + proc = trwm.backup("dummycfg") + assert proc.returncode == 12