diff --git a/.pylintrc b/.pylintrc
index cd01475054c5d0182557154387bc8134b3e07f31..d0683f0da1ba7e36ee9fe57a5427182ad400b2e2 100644
--- a/.pylintrc
+++ b/.pylintrc
@@ -1,9 +1,9 @@
+[MESSAGES]
+disable = logging-fstring-interpolation
+
[FORMAT]
max-line-length=150
-[TYPECHECK]
-ignored-classes=SQLAlchemy, sqlalchemy.orm.scoping.scoped_session
-
[SIMILARITIES]
min-similarity-lines=8
ignore-imports=yes
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 17aa5f2147fd8dc37a8a73d58547637175d374d7..cc7a31f5a8883e1fb310a06439bf17884d9e3912 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
all: lint
install:
- apt-get -y install awscli make python3-cryptography rclone restic yamllint
+ apt-get -y install awscli make python3-cryptography python3-tabulate rclone restic yamllint
venv:
apt-get -y install python3-venv
diff --git a/requirements.lock b/requirements.lock
index 54ee3a7939fd01ccb076f6f6337418899c407a4a..aa7dd3d80f9d1bc94462a13ca9d170ceb1599fe2 100644
--- a/requirements.lock
+++ b/requirements.lock
@@ -72,6 +72,7 @@ s3transfer==0.10.1
sarif-om==1.0.4
six==1.16.0
sympy==1.12
+tabulate==0.9.0
tomlkit==0.12.4
typing_extensions==4.10.0
urllib3==2.2.1
diff --git a/requirements.txt b/requirements.txt
index 98717c27767f70b11754c3cac6ee13643b7b884a..0f055546e5a8217e4b52cc6e657e8ef4712347b3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,6 @@
# runtime
cryptography
+tabulate
# dev
flake8
diff --git a/rwm.py b/rwm.py
index 911509a610555a69e150aed2037bd26ed5d52f92..a7cd88a26c41311dc02e9af1bdf7a31f22e2f699 100755
--- a/rwm.py
+++ b/rwm.py
@@ -2,17 +2,20 @@
"""rwm, restic/s3 worm manager"""
import base64
+import dataclasses
import logging
import os
import shlex
import subprocess
import sys
from argparse import ArgumentParser
+from datetime import datetime
from pathlib import Path
import yaml
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
from cryptography.hazmat.backends import default_backend
+from tabulate import tabulate
__version__ = "0.2"
@@ -74,6 +77,28 @@ def rclone_obscure_password(plaintext, iv=None):
return base64.urlsafe_b64encode(data).decode().rstrip("=")
+@dataclasses.dataclass
+class BackupResult:
+ """backup results data container"""
+
+ name: str
+ returncode: int
+ time_start: datetime
+ time_end: datetime
+
+ def to_dict(self):
+ """dict serializer"""
+
+ return {
+ "ident": "RESULT",
+ "name": self.name,
+ "status": "OK" if self.returncode == 0 else "ERROR",
+ "returncode": self.returncode,
+ "backup_start": self.time_start.isoformat(),
+ "backup_time": str(self.time_end-self.time_start),
+ }
+
+
class RWM:
"""rwm impl"""
@@ -148,46 +173,92 @@ class RWM:
}
return run_command(["restic"] + args, env=env)
- def backup(self, name):
- """do restic backup from config"""
+ def restic_autoinit(self):
+ """runs restic init"""
- if self.restic_cmd(["cat", "config"]).returncode != 0:
- if (proc := self.restic_cmd(["init"])).returncode != 0:
- logger.error("failed to autoinitialize restic repository")
- return proc
+ logger.info("run restic_autoinit")
+ if (proc := self.restic_cmd(["cat", "config"])).returncode != 0:
+ proc = self.restic_cmd(["init"])
+ return proc
+ def restic_backup(self, name):
+ """runs restic backup by name"""
+
+ logger.info(f"run restic_backup {name}")
conf = self.config["RWM_BACKUPS"][name]
+ excludes = []
+ for item in conf.get("excludes", []):
+ excludes += ["--exclude", item]
+ backup_extras = conf.get("backup_extras", [])
+ cmd_args = ["backup"] + backup_extras + excludes + conf["filesdirs"]
- # restic backup
- excludes = conf.get("excludes", [])
- excludes = [x for pair in zip(["--exclude"] * len(excludes), excludes) for x in pair]
- extras = conf.get("extras", [])
- cmd_args = ["backup"] + extras + excludes + conf["filesdirs"]
+ return self.restic_cmd(cmd_args)
- logger.info("running backup")
- backup_proc = self.restic_cmd(cmd_args)
- wrap_output(backup_proc)
- if backup_proc.returncode != 0:
- logger.error("backup failed, %s", backup_proc)
- return backup_proc
+ def restic_forget_prune(self):
+ """runs forget prune"""
- # restic forget prune
+ logger.info("run restic_forget_prune")
keeps = []
for key, val in self.config.get("RWM_RETENTION", {}).items():
keeps += [f"--{key}", val]
- if not keeps:
- logger.error("no retention policy found")
cmd_args = ["forget", "--prune"] + keeps
- logger.info("running forget prune")
- forget_proc = self.restic_cmd(cmd_args)
- wrap_output(forget_proc)
+ return self.restic_cmd(cmd_args)
+
+ def backup_cmd(self, name):
+ """backup command"""
+
+ autoinit_proc = self.restic_autoinit()
+ if autoinit_proc.returncode != 0:
+ logger.error("restic autoinit failed")
+ wrap_output(autoinit_proc)
+ return autoinit_proc
+
+ wrap_output(backup_proc := self.restic_backup(name))
+ if backup_proc.returncode != 0:
+ logger.error("restic_backup failed")
+ return backup_proc
+
+ wrap_output(forget_proc := self.restic_forget_prune())
if forget_proc.returncode != 0:
- logger.error("forget prune failed, %s", forget_proc)
+ logger.error("restic_forget_prune failed")
return forget_proc
return backup_proc
+ def backup_all_cmd(self):
+ """backup all command"""
+
+ stats = {}
+ ret = 0
+
+ time_start = datetime.now()
+ autoinit_proc = self.restic_autoinit()
+ time_end = datetime.now()
+ if autoinit_proc.returncode != 0:
+ logger.error("restic autoinit failed")
+ wrap_output(autoinit_proc)
+ return autoinit_proc.returncode
+ stats["_autoinit"] = BackupResult("_autoinit", autoinit_proc.returncode, time_start, time_end)
+
+ for name in self.config["RWM_BACKUPS"].keys():
+ time_start = datetime.now()
+ wrap_output(backup_proc := self.restic_backup(name))
+ time_end = datetime.now()
+ ret |= backup_proc.returncode
+ stats[name] = BackupResult(name, backup_proc.returncode, time_start, time_end)
+
+ if ret == 0:
+ time_start = datetime.now()
+ wrap_output(forget_proc := self.restic_forget_prune())
+ time_end = datetime.now()
+ ret |= forget_proc.returncode
+ stats["_forget_prune"] = BackupResult("_forget_prune", forget_proc.returncode, time_start, time_end)
+
+ logger.info("rwm backup_all results")
+ print(tabulate([item.to_dict() for item in stats.values()], headers="keys", numalign="left"))
+ return ret
+
def configure_logging(debug):
"""configure logger"""
@@ -212,6 +283,7 @@ def parse_arguments(argv):
subparsers = parser.add_subparsers(title="commands", dest="command", required=False)
subparsers.add_parser("version", help="show version")
+
aws_cmd_parser = subparsers.add_parser("aws", help="aws command")
aws_cmd_parser.add_argument("cmd_args", nargs="*")
rc_cmd_parser = subparsers.add_parser("rclone", help="rclone command")
@@ -220,8 +292,10 @@ def parse_arguments(argv):
rcc_cmd_parser.add_argument("cmd_args", nargs="*")
res_cmd_parser = subparsers.add_parser("restic", help="restic command")
res_cmd_parser.add_argument("cmd_args", nargs="*")
+
backup_cmd_parser = subparsers.add_parser("backup", help="backup command")
backup_cmd_parser.add_argument("name", help="backup config name")
+ subparsers.add_parser("backup_all", help="backup all command")
return parser.parse_args(argv)
@@ -252,8 +326,11 @@ def main(argv=None):
ret = wrap_output(rwmi.rclone_crypt_cmd(args.cmd_args))
if args.command == "restic":
ret = wrap_output(rwmi.restic_cmd(args.cmd_args))
+
if args.command == "backup":
- ret = rwmi.backup(args.name).returncode
+ ret = rwmi.backup_cmd(args.name).returncode
+ if args.command == "backup_all":
+ ret = rwmi.backup_all_cmd()
logger.info("rwm finished with %s (ret %d)", "success" if ret == 0 else "errors", ret)
return ret
diff --git a/tests/test_default.py b/tests/test_default.py
index da52e2b378669bd8b1693f39ed51a099f208872d..495b399c6b27981c9624b3d3d7b780257a6d33cc 100644
--- a/tests/test_default.py
+++ b/tests/test_default.py
@@ -46,12 +46,13 @@ def test_main(tmpworkdir: str): # pylint: disable=unused-argument
# command branches
mock = Mock(return_value=CompletedProcess(args='dummy', returncode=0))
- for item in ["aws", "rclone", "rclone_crypt", "restic"]:
+ for item in ["aws", "rclone", "rclone_crypt", "restic", "backup"]:
with patch.object(rwm.RWM, f"{item}_cmd", mock):
assert rwm_main([item, "dummy"]) == 0
- with patch.object(rwm.RWM, "backup", mock):
- assert rwm_main(["backup", "dummy"]) == 0
+ mock = Mock(return_value=0)
+ with patch.object(rwm.RWM, "backup_all_cmd", mock):
+ assert rwm_main(["backup_all"]) == 0
def test_aws_cmd(tmpworkdir: str, motoserver: str): # pylint: disable=unused-argument
@@ -162,8 +163,8 @@ def _list_files(trwm, snapshot_id):
]
-def test_backup(tmpworkdir: str, motoserver: str): # pylint: disable=unused-argument
- """test backup command"""
+def test_backup_cmd(tmpworkdir: str, motoserver: str): # pylint: disable=unused-argument
+ """test backup_cmd command"""
trwm = RWM({
"RWM_S3_ENDPOINT_URL": motoserver,
@@ -187,7 +188,7 @@ def test_backup(tmpworkdir: str, motoserver: str): # pylint: disable=unused-arg
Path("testdatadir/testdata1.txt").write_text("dummydata", encoding="utf-8")
Path("testdatadir/testfile_to_be_ignored").write_text("dummydata", encoding="utf-8")
- assert trwm.backup("testcfg").returncode == 0
+ assert trwm.backup_cmd("testcfg").returncode == 0
snapshots = _list_snapshots(trwm)
assert len(snapshots) == 1
@@ -195,15 +196,7 @@ def test_backup(tmpworkdir: str, motoserver: str): # pylint: disable=unused-arg
assert "/testdatadir/testdata1.txt" in snapshot_files
-def test_backup_autoinit(): # pylint: disable=unused-argument
- """mocked error handling test"""
-
- mock = Mock(return_value=CompletedProcess(args='dummy', returncode=1))
- with patch.object(rwm.RWM, "restic_cmd", mock):
- RWM({}).backup("dummy")
-
-
-def test_backup_excludes(tmpworkdir: str, motoserver: str): # pylint: disable=unused-argument
+def test_backup_cmd_excludes(tmpworkdir: str, motoserver: str): # pylint: disable=unused-argument
"""test backup command"""
trwm = RWM({
@@ -229,7 +222,7 @@ def test_backup_excludes(tmpworkdir: str, motoserver: str): # pylint: disable=u
Path("testdatadir/proc").mkdir()
Path("testdatadir/proc/to_be_also_excluded").write_text("dummydata", encoding="utf-8")
- assert trwm.backup("testcfg").returncode == 0
+ assert trwm.backup_cmd("testcfg").returncode == 0
snapshots = _list_snapshots(trwm)
assert len(snapshots) == 1
@@ -238,32 +231,61 @@ def test_backup_excludes(tmpworkdir: str, motoserver: str): # pylint: disable=u
assert "/testdatadir/etc/config2" in snapshot_files
-def test_backup_error_handling(tmpworkdir: str, motoserver: str): # pylint: disable=unused-argument
+def test_backup_cmd_error_handling(tmpworkdir: str, motoserver: str): # pylint: disable=unused-argument
"""test backup command err cases"""
- mock = Mock(side_effect=[
- CompletedProcess(args='dummy', returncode=0), # autoinit
- CompletedProcess(args='dummy', returncode=11) # backup
- ])
- with patch.object(rwm.RWM, "restic_cmd", mock):
- trwm = RWM({
- "RWM_BACKUPS": {
- "dummycfg": {"filesdirs": ["dummydir"]}
- }
- })
- proc = trwm.backup("dummycfg")
- assert proc.returncode == 11
-
- mock = Mock(side_effect=[
- CompletedProcess(args='dummy', returncode=0), # autoinit
- CompletedProcess(args='dummy', returncode=0), # backup
- CompletedProcess(args='dummy', returncode=12) # forget
- ])
- with patch.object(rwm.RWM, "restic_cmd", mock):
- trwm = RWM({
- "RWM_BACKUPS": {
- "dummycfg": {"filesdirs": ["dummydir"]}
- }
- })
- proc = trwm.backup("dummycfg")
- assert proc.returncode == 12
+ rwm_conf = {
+ "RWM_BACKUPS": {
+ "dummycfg": {"filesdirs": ["dummydir"]}
+ }
+ }
+ mock_ok = Mock(return_value=CompletedProcess(args='dummy', returncode=0))
+ mock_fail = Mock(return_value=CompletedProcess(args='dummy', returncode=11))
+
+ with patch.object(rwm.RWM, "restic_autoinit", mock_fail):
+ assert RWM(rwm_conf).backup_cmd("dummycfg").returncode == 11
+
+ with (
+ patch.object(rwm.RWM, "restic_autoinit", mock_ok),
+ patch.object(rwm.RWM, "restic_backup", mock_fail)
+ ):
+ assert RWM(rwm_conf).backup_cmd("dummycfg").returncode == 11
+
+ with (
+ patch.object(rwm.RWM, "restic_autoinit", mock_ok),
+ patch.object(rwm.RWM, "restic_backup", mock_ok),
+ patch.object(rwm.RWM, "restic_forget_prune", mock_fail)
+ ):
+ assert RWM(rwm_conf).backup_cmd("dummycfg").returncode == 11
+
+
+def test_backup_all_cmd(tmpworkdir: str): # pylint: disable=unused-argument
+ """test backup command err cases"""
+
+ rwm_conf = {
+ "RWM_BACKUPS": {
+ "dummycfg": {"filesdirs": ["dummydir"]}
+ }
+ }
+ mock = Mock(return_value=CompletedProcess(args='dummy', returncode=0))
+
+ with (
+ patch.object(rwm.RWM, "restic_autoinit", mock),
+ patch.object(rwm.RWM, "restic_backup", mock),
+ patch.object(rwm.RWM, "restic_forget_prune", mock)
+ ):
+ assert RWM(rwm_conf).backup_all_cmd() == 0
+
+
+def test_backup_all_cmd_error_handling(tmpworkdir: str): # pylint: disable=unused-argument
+ """test backup command err cases"""
+
+ rwm_conf = {
+ "RWM_BACKUPS": {
+ "dummycfg": {"filesdirs": ["dummydir"]}
+ }
+ }
+ mock_fail = Mock(return_value=CompletedProcess(args='dummy', returncode=11))
+
+ with patch.object(rwm.RWM, "restic_autoinit", mock_fail):
+ assert RWM(rwm_conf).backup_all_cmd() == 11