diff --git a/Makefile b/Makefile
index 092749561a1386272b842f11c59f5de044a0f0e0..c0bb4993489d8bdf18b7e69fe7d997454560bc3a 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
all: coverage lint
install:
- apt-get -y install awscli python3-boto3 python3-tabulate restic yamllint
+ apt-get -y install awscli python3-boto3 python3-pydantic python3-tabulate restic yamllint
install-dev:
apt-get -y install python3-venv snapd
diff --git a/examples/rwm-admin.conf b/examples/rwm-admin.conf
index aa438cf0653ad4286f6562a9287ef43871d65ec0..9d538726adbf57afc8f09a4e10597b76676a9ad0 100644
--- a/examples/rwm-admin.conf
+++ b/examples/rwm-admin.conf
@@ -1,5 +1,5 @@
# rwm aws, storage
-rwm_s3_endpoint_url: ""
-rwm_s3_access_key: ""
-rwm_s3_secret_key: ""
+s3_endpoint_url: ""
+s3_access_key: ""
+s3_secret_key: ""
diff --git a/examples/rwm-backups.conf b/examples/rwm-backups.conf
index b2477759e6cd0b9a20700d015a2f2954faf3e270..7356ae57cb1b9e1071b9a6b5a2ab04a17201d05a 100644
--- a/examples/rwm-backups.conf
+++ b/examples/rwm-backups.conf
@@ -1,13 +1,13 @@
# rwm aws, restic, backup, backup_all
-rwm_s3_endpoint_url: ""
-rwm_s3_access_key: ""
-rwm_s3_secret_key: ""
+s3_endpoint_url: ""
+s3_access_key: ""
+s3_secret_key: ""
-rwm_restic_bucket: "rwmbackups"
-rwm_restic_password: ""
+restic_bucket: "rwmbackups"
+restic_password: ""
-rwm_backups:
+backups:
simplefs:
filesdirs:
- "/"
@@ -38,7 +38,7 @@ rwm_backups:
postrun:
- unmount
-rwm_retention:
+retention:
keep-daily: "60"
keep-within: "60d"
keep-tag: "donotforget"
diff --git a/examples/rwm-linux.conf b/examples/rwm-linux.conf
index 89dec7c77bb9073761fcb1e341b92234be9be49f..35d4874dab77eccb4212b7bab69bac2bd6f7f0a5 100644
--- a/examples/rwm-linux.conf
+++ b/examples/rwm-linux.conf
@@ -1,13 +1,13 @@
# rwm aws, restic, backup, backup_all
-rwm_s3_endpoint_url: ""
-rwm_s3_access_key: ""
-rwm_s3_secret_key: ""
+s3_endpoint_url: ""
+s3_access_key: ""
+s3_secret_key: ""
-rwm_restic_bucket: "rwmbackups"
-rwm_restic_password: ""
+restic_bucket: "rwmbackups"
+restic_password: ""
-rwm_backups:
+backups:
linux1:
filesdirs:
- "/"
@@ -30,6 +30,6 @@ rwm_backups:
postrun:
- "mkdir -p /var/health && touch /var/health/backup"
-rwm_retention:
+retention:
keep-daily: "14"
keep-weekly: "20"
diff --git a/examples/rwm-restic.conf b/examples/rwm-restic.conf
index beeee3e905dcff057f36ee94930924f2a30317c3..fab285160046e109f1ced991ca9745499cbb8d71 100644
--- a/examples/rwm-restic.conf
+++ b/examples/rwm-restic.conf
@@ -1,8 +1,8 @@
# rwm aws, restic
-rwm_s3_endpoint_url: ""
-rwm_s3_access_key: ""
-rwm_s3_secret_key: ""
+s3_endpoint_url: ""
+s3_access_key: ""
+s3_secret_key: ""
-rwm_restic_bucket: "rwmrestic"
-rwm_restic_password: ""
+restic_bucket: "rwmrestic"
+restic_password: ""
diff --git a/requirements.txt b/requirements.txt
index eb406a46e589fc803692de4966e98497fe9d6b7f..c187ece0cc800d2bbcb1220b3fbf9293d7dfebfd 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,5 @@
# runtime
+pydantic
tabulate
# dev
diff --git a/rwm.py b/rwm.py
index 3bb14c418676987350dc7ce36cd4630e45b6d1cb..5cfc6cdfb556a88d28063562a653d4088a4c66fe 100755
--- a/rwm.py
+++ b/rwm.py
@@ -1,7 +1,6 @@
#!/usr/bin/env python3
"""rwm, restic/s3 worm manager"""
-import dataclasses
import gzip
import json
import logging
@@ -10,13 +9,16 @@ import shlex
import subprocess
import sys
from argparse import ArgumentParser
+from dataclasses import dataclass
from datetime import datetime
from io import BytesIO
from pathlib import Path
+from typing import List, Dict, Optional
import boto3
import yaml
from botocore.exceptions import BotoCoreError, ClientError
+from pydantic import BaseModel, ConfigDict
from tabulate import tabulate
@@ -34,14 +36,6 @@ def is_sublist(needle, haystack):
return any(haystack[i:i+len(needle)] == needle for i in range(len(haystack)))
-def get_config(path):
- """load config"""
-
- if Path(path).exists():
- return yaml.safe_load(Path(path).read_text(encoding='utf-8')) or {}
- return {}
-
-
def run_command(*args, **kwargs):
"""output capturing command executor"""
@@ -74,26 +68,30 @@ def size_fmt(num):
return f'{num:0.1f} YiB'
-@dataclasses.dataclass
-class BackupResult:
- """backup results data container"""
+class BackupConfig(BaseModel):
+ """backup config model"""
- name: str
- returncode: int
- time_start: datetime
- time_end: datetime
+ model_config = ConfigDict(extra='forbid')
- def to_dict(self):
- """dict serializer"""
+ filesdirs: List[str]
+ excludes: Optional[List[str]] = []
+ extras: Optional[List[str]] = []
+ prerun: Optional[List[str]] = []
+ postrun: Optional[List[str]] = []
- return {
- "ident": "RESULT",
- "name": self.name,
- "status": "OK" if self.returncode == 0 else "ERROR",
- "returncode": self.returncode,
- "backup_start": self.time_start.isoformat(),
- "backup_time": str(self.time_end-self.time_start),
- }
+
+class RWMConfig(BaseModel):
+ """main config model"""
+
+ model_config = ConfigDict(extra='forbid')
+
+ s3_endpoint_url: str
+ s3_access_key: str
+ s3_secret_key: str
+ restic_bucket: Optional[str] = None
+ restic_password: Optional[str] = None
+ backups: Dict[str, BackupConfig] = {}
+ retention: Dict[str, str] = {}
class RwmJSONEncoder(json.JSONEncoder):
@@ -373,15 +371,37 @@ class StorageManager:
return 0
+@dataclass
+class BackupResult:
+ """backup results data container"""
+
+ name: str
+ returncode: int
+ time_start: datetime
+ time_end: datetime
+
+ def to_dict(self):
+ """dict serializer"""
+
+ return {
+ "ident": "RESULT",
+ "name": self.name,
+ "status": "OK" if self.returncode == 0 else "ERROR",
+ "returncode": self.returncode,
+ "backup_start": self.time_start.isoformat(),
+ "backup_time": str(self.time_end-self.time_start),
+ }
+
+
class RWM:
"""rwm impl"""
- def __init__(self, config):
- self.config = config
+ def __init__(self, config_dict):
+ self.config = RWMConfig(**config_dict)
self.storage_manager = StorageManager(
- config.get("rwm_s3_endpoint_url"),
- config.get("rwm_s3_access_key"),
- config.get("rwm_s3_secret_key")
+ self.config.s3_endpoint_url,
+ self.config.s3_access_key,
+ self.config.s3_secret_key
)
def aws_cmd(self, args) -> subprocess.CompletedProcess:
@@ -390,15 +410,15 @@ class RWM:
env = {
"PATH": os.environ["PATH"],
"AWS_METADATA_SERVICE_NUM_ATTEMPTS": "0",
- "AWS_ACCESS_KEY_ID": self.config["rwm_s3_access_key"],
- "AWS_SECRET_ACCESS_KEY": self.config["rwm_s3_secret_key"]
+ "AWS_ACCESS_KEY_ID": self.config.s3_access_key,
+ "AWS_SECRET_ACCESS_KEY": self.config.s3_secret_key
}
if is_sublist(["s3", "mb"], args):
# region must be set and empty for awscil >=2.x and ?du? ceph s3
env.update({"AWS_DEFAULT_REGION": ""})
# aws cli does not have endpoint-url as env config option
- return run_command(["aws", "--endpoint-url", self.config["rwm_s3_endpoint_url"]] + args, env=env)
+ return run_command(["aws", "--endpoint-url", self.config.s3_endpoint_url] + args, env=env)
def restic_cmd(self, args) -> subprocess.CompletedProcess:
"""restic command wrapper"""
@@ -406,10 +426,10 @@ class RWM:
env = {
"HOME": os.environ["HOME"],
"PATH": os.environ["PATH"],
- "AWS_ACCESS_KEY_ID": self.config["rwm_s3_access_key"],
- "AWS_SECRET_ACCESS_KEY": self.config["rwm_s3_secret_key"],
- "RESTIC_PASSWORD": self.config["rwm_restic_password"],
- "RESTIC_REPOSITORY": f"s3:{self.config['rwm_s3_endpoint_url']}/{self.config['rwm_restic_bucket']}",
+ "AWS_ACCESS_KEY_ID": self.config.s3_access_key,
+ "AWS_SECRET_ACCESS_KEY": self.config.s3_secret_key,
+ "RESTIC_PASSWORD": self.config.restic_password,
+ "RESTIC_REPOSITORY": f"s3:{self.config.s3_endpoint_url}/{self.config.restic_bucket}",
}
return run_command(["restic"] + args, env=env)
@@ -417,12 +437,11 @@ class RWM:
"""runs restic backup by name"""
logger.info(f"run restic_backup {name}")
- conf = self.config["rwm_backups"][name]
+ conf = self.config.backups[name]
excludes = []
- for item in conf.get("excludes", []):
+ for item in conf.excludes:
excludes += ["--exclude", item]
- extras = conf.get("extras", [])
- cmd_args = ["backup"] + extras + excludes + conf["filesdirs"]
+ cmd_args = ["backup"] + conf.extras + excludes + conf.filesdirs
return self.restic_cmd(cmd_args)
@@ -431,7 +450,7 @@ class RWM:
logger.info("run restic_forget_prune")
keeps = []
- for key, val in self.config.get("rwm_retention", {}).items():
+ for key, val in self.config.retention.items():
keeps += [f"--{key}", val]
cmd_args = ["forget", "--prune"] + keeps
@@ -451,18 +470,16 @@ class RWM:
def _prerun(self, name) -> int:
"""prerun runparts stub"""
- return self._runparts("prerun", self.config["rwm_backups"][name].get("prerun", []))
+ return self._runparts("prerun", self.config.backups[name].prerun)
def _postrun(self, name) -> int:
"""postrun runparts stub"""
- return self._runparts("postrun", self.config["rwm_backups"][name].get("postrun", []))
+ return self._runparts("postrun", self.config.backups[name].prerun)
def backup(self, name) -> int:
"""backup command"""
- bucket_name = self.config["rwm_restic_bucket"]
-
- if not self.storage_manager.storage_check_policy(bucket_name):
+ if not self.storage_manager.storage_check_policy(self.config.restic_bucket):
logger.warning("used bucket does not have expected policy")
if self._prerun(name) != 0:
@@ -483,7 +500,7 @@ class RWM:
logger.error("rwm _restic_forget_prune failed")
return 1
- if self.storage_manager.storage_save_state(bucket_name) != 0:
+ if self.storage_manager.storage_save_state(self.config.restic_bucket) != 0:
logger.error("rwm storage_save_state failed")
return 1
@@ -495,7 +512,7 @@ class RWM:
stats = []
ret = 0
- for name in self.config["rwm_backups"].keys():
+ for name in self.config.backups:
time_start = datetime.now()
wrap_output(backup_proc := self._restic_backup(name))
time_end = datetime.now()
@@ -510,7 +527,7 @@ class RWM:
stats.append(BackupResult("_forget_prune", forget_proc.returncode, time_start, time_end))
time_start = datetime.now()
- save_state_ret = self.storage_manager.storage_save_state(self.config["rwm_restic_bucket"])
+ save_state_ret = self.storage_manager.storage_save_state(self.config.restic_bucket)
time_end = datetime.now()
ret |= save_state_ret
stats.append(BackupResult("_storage_save_state", save_state_ret, time_start, time_end))
@@ -636,18 +653,23 @@ def parse_arguments(argv):
return parser.parse_args(argv)
+def load_config(path):
+ """load config dict from file"""
+
+ config = {}
+ if path:
+ config = yaml.safe_load(Path(path).read_text(encoding='utf-8'))
+ logger.debug("config, %s", config)
+ return config
+
+
def main(argv=None): # pylint: disable=too-many-branches
"""main"""
args = parse_arguments(argv)
configure_logging(args.debug)
- config = {}
- if args.config:
- config.update(get_config(args.config))
- logger.debug("config, %s", config)
- # assert config ?
- rwmi = RWM(config)
+ rwmi = RWM(load_config(args.config))
ret = -1
if args.command == "version":
diff --git a/tests/rwmtest.conf b/tests/rwmtest.conf
new file mode 100644
index 0000000000000000000000000000000000000000..4e9e204ce469e4f5c442512dd3924bcd64691cbe
--- /dev/null
+++ b/tests/rwmtest.conf
@@ -0,0 +1,3 @@
+s3_endpoint_url: http://dummy
+s3_access_key: dummy
+s3_secret_key: dummy
\ No newline at end of file
diff --git a/tests/test_default.py b/tests/test_default.py
index b174ee8d8df119c944bfe8f45d281b3f40e2c204..ff9c793fe68907f461b6021b126aff47d217d144 100644
--- a/tests/test_default.py
+++ b/tests/test_default.py
@@ -1,6 +1,5 @@
"""default tests"""
-from pathlib import Path
from subprocess import CompletedProcess
from unittest.mock import Mock, patch
@@ -29,37 +28,37 @@ def test_size_fmt():
assert size_fmt(10**25) == "8.3 YiB"
-def test_main(tmpworkdir: str): # pylint: disable=unused-argument
+def test_main():
"""test main"""
- # optional and default config handling
- assert rwm_main(["version"]) == 0
- Path("rwm.conf").touch()
- assert rwm_main(["version"]) == 0
+ def rwm_main_minconfig(args):
+ return rwm_main(["--config", "tests/rwmtest.conf"] + args)
+
+ assert rwm_main_minconfig(["version"]) == 0
# command branches
mock_proc = Mock(return_value=CompletedProcess(args='dummy', returncode=0))
mock_ok = Mock(return_value=0)
with patch.object(rwm.RWM, "aws_cmd", mock_proc):
- assert rwm_main(["aws", "dummy"]) == 0
+ assert rwm_main_minconfig(["aws", "dummy"]) == 0
with patch.object(rwm.RWM, "restic_cmd", mock_proc):
- assert rwm_main(["restic", "dummy"]) == 0
+ assert rwm_main_minconfig(["restic", "dummy"]) == 0
with patch.object(rwm.RWM, "backup", mock_ok):
- assert rwm_main(["backup", "dummy"]) == 0
+ assert rwm_main_minconfig(["backup", "dummy"]) == 0
with patch.object(rwm.RWM, "backup_all", mock_ok):
- assert rwm_main(["backup_all"]) == 0
+ assert rwm_main_minconfig(["backup_all"]) == 0
with patch.object(rwm.RWM, "storage_create", mock_ok):
- assert rwm_main(["storage_create", "bucket", "user"]) == 0
+ assert rwm_main_minconfig(["storage_create", "bucket", "user"]) == 0
with patch.object(rwm.RWM, "storage_delete", mock_ok):
- assert rwm_main(["storage_delete", "bucket"]) == 0
+ assert rwm_main_minconfig(["storage_delete", "bucket"]) == 0
with patch.object(rwm.RWM, "storage_list", mock_ok):
- assert rwm_main(["storage_list"]) == 0
+ assert rwm_main_minconfig(["storage_list"]) == 0
with patch.object(rwm.RWM, "storage_info", mock_ok):
- assert rwm_main(["storage_info", "dummy"]) == 0
+ assert rwm_main_minconfig(["storage_info", "dummy"]) == 0
with patch.object(rwm.RWM, "storage_drop_versions", mock_ok):
- assert rwm_main(["storage_drop_versions", "bucket"]) == 0
+ assert rwm_main_minconfig(["storage_drop_versions", "bucket"]) == 0
with patch.object(rwm.RWM, "storage_restore_state", mock_ok):
- assert rwm_main(["storage_restore_state", "bucket", "bucket", "state"]) == 0
+ assert rwm_main_minconfig(["storage_restore_state", "bucket", "bucket", "state"]) == 0
diff --git a/tests/test_rwm.py b/tests/test_rwm.py
index 8687337804b9898864920c2aa53b17d3988b67dd..de89cbed4c405cbce744b6a02e00ce9b2da36425 100644
--- a/tests/test_rwm.py
+++ b/tests/test_rwm.py
@@ -14,9 +14,9 @@ def test_aws_cmd(tmpworkdir: str, motoserver: str): # pylint: disable=unused-ar
"""test aws command"""
trwm = rwm.RWM({
- "rwm_s3_endpoint_url": motoserver,
- "rwm_s3_access_key": "dummy",
- "rwm_s3_secret_key": "dummy",
+ "s3_endpoint_url": motoserver,
+ "s3_access_key": "dummy",
+ "s3_secret_key": "dummy",
})
test_bucket = "testbucket"
@@ -33,11 +33,11 @@ def test_restic_cmd(tmpworkdir: str, motoserver: str): # pylint: disable=unused
"""test restic command"""
trwm = rwm.RWM({
- "rwm_s3_endpoint_url": motoserver,
- "rwm_s3_access_key": "dummy",
- "rwm_s3_secret_key": "dummy",
- "rwm_restic_bucket": "restictest",
- "rwm_restic_password": "dummydummydummydummy",
+ "s3_endpoint_url": motoserver,
+ "s3_access_key": "dummy",
+ "s3_secret_key": "dummy",
+ "restic_bucket": "restictest",
+ "restic_password": "dummydummydummydummy",
})
assert trwm.restic_cmd(["init"]).returncode == 0
@@ -56,13 +56,13 @@ def _restic_list_snapshot_files(trwm, snapshot_id):
return [x["path"] for x in snapshot_ls if (x["struct_type"] == "node") and (x["type"] == "file")]
-def test_runparts(tmpworkdir: str, motoserver: str): # pylint: disable=unused-argument
+def test_runparts(tmpworkdir: str): # pylint: disable=unused-argument
"""test runparts"""
trwm = rwm.RWM({
- "rwm_s3_endpoint_url": motoserver,
- "rwm_s3_access_key": "dummy",
- "rwm_s3_secret_key": "dummy",
+ "s3_endpoint_url": "http://dummy",
+ "s3_access_key": "dummy",
+ "s3_secret_key": "dummy",
})
assert trwm._runparts("tests", ["false"]) == 1 # pylint: disable=protected-access
@@ -72,19 +72,19 @@ def test_backup(tmpworkdir: str, motoserver: str): # pylint: disable=unused-arg
"""test backup"""
trwm = rwm.RWM({
- "rwm_s3_endpoint_url": motoserver,
- "rwm_s3_access_key": "dummy",
- "rwm_s3_secret_key": "dummy",
- "rwm_restic_bucket": "restictest",
- "rwm_restic_password": "dummydummydummydummy",
- "rwm_backups": {
+ "s3_endpoint_url": motoserver,
+ "s3_access_key": "dummy",
+ "s3_secret_key": "dummy",
+ "restic_bucket": "restictest",
+ "restic_password": "dummydummydummydummy",
+ "backups": {
"testcfg": {
"filesdirs": ["testdatadir/"],
"excludes": ["testfile_to_be_ignored"],
"extras": ["--tag", "dummytag"],
}
},
- "rwm_retention": {
+ "retention": {
"keep-daily": "1"
}
})
@@ -106,12 +106,12 @@ def test_backup_excludes(tmpworkdir: str, motoserver: str): # pylint: disable=u
"""test backu"""
trwm = rwm.RWM({
- "rwm_s3_endpoint_url": motoserver,
- "rwm_s3_access_key": "dummy",
- "rwm_s3_secret_key": "dummy",
- "rwm_restic_bucket": "restictest",
- "rwm_restic_password": "dummydummydummydummy",
- "rwm_backups": {
+ "s3_endpoint_url": motoserver,
+ "s3_access_key": "dummy",
+ "s3_secret_key": "dummy",
+ "restic_bucket": "restictest",
+ "restic_password": "dummydummydummydummy",
+ "backups": {
"testcfg": {
"filesdirs": ["testdatadir/"],
"excludes": ["testdatadir/proc/*", "*.ignored"],
@@ -145,16 +145,16 @@ def test_backup_excludes(tmpworkdir: str, motoserver: str): # pylint: disable=u
assert "/testdatadir/var/proc/data" in snapshot_files
-def test_backup_runparts(tmpworkdir: str, motoserver: str): # pylint: disable=unused-argument
+def test_backup_runparts(tmpworkdir: str): # pylint: disable=unused-argument
"""test backup"""
trwm = rwm.RWM({
- "rwm_s3_endpoint_url": motoserver,
- "rwm_s3_access_key": "dummy",
- "rwm_s3_secret_key": "dummy",
- "rwm_restic_bucket": "restictest",
- "rwm_restic_password": "dummydummydummydummy",
- "rwm_backups": {
+ "s3_endpoint_url": "http://dummy",
+ "s3_access_key": "dummy",
+ "s3_secret_key": "dummy",
+ "restic_bucket": "restictest",
+ "restic_password": "dummydummydummydummy",
+ "backups": {
"testcfg": {
"filesdirs": ["testdatadir/"],
"prerun": ["false || true"],
@@ -175,12 +175,15 @@ def test_backup_runparts(tmpworkdir: str, motoserver: str): # pylint: disable=u
assert trwm.backup("testcfg") == 0
-def test_backup_error_handling(tmpworkdir: str, motoserver: str): # pylint: disable=unused-argument
+def test_backup_error_handling(tmpworkdir: str): # pylint: disable=unused-argument
"""test backup command err cases"""
rwm_conf = {
- "rwm_restic_bucket": "restictest",
- "rwm_backups": {
+ "s3_endpoint_url": "http://dummy",
+ "s3_access_key": "dummy",
+ "s3_secret_key": "dummy",
+ "restic_bucket": "restictest",
+ "backups": {
"dummycfg": {"filesdirs": ["dummydir"]}
}
}
@@ -229,8 +232,11 @@ def test_backup_all(tmpworkdir: str): # pylint: disable=unused-argument
"""test backup_all"""
rwm_conf = {
- "rwm_restic_bucket": "restictest",
- "rwm_backups": {
+ "s3_endpoint_url": "http://dummy",
+ "s3_access_key": "dummy",
+ "s3_secret_key": "dummy",
+ "restic_bucket": "restictest",
+ "backups": {
"dummycfg": {"filesdirs": ["dummydir"]}
}
}
@@ -249,9 +255,9 @@ def test_storage_create(tmpworkdir: str, microceph: str, radosuser_admin: rwm.St
"""test_storage_create"""
trwm = rwm.RWM({
- "rwm_s3_endpoint_url": radosuser_admin.url,
- "rwm_s3_access_key": radosuser_admin.access_key,
- "rwm_s3_secret_key": radosuser_admin.secret_key,
+ "s3_endpoint_url": radosuser_admin.url,
+ "s3_access_key": radosuser_admin.access_key,
+ "s3_secret_key": radosuser_admin.secret_key,
})
bucket_name = "testbuck"
@@ -264,18 +270,17 @@ def test_storage_delete(tmpworkdir: str, microceph: str, radosuser_admin: rwm.St
"""test_storage_delete"""
trwm = rwm.RWM({
- "rwm_s3_endpoint_url": radosuser_admin.url,
- "rwm_s3_access_key": radosuser_admin.access_key,
- "rwm_s3_secret_key": radosuser_admin.secret_key,
-
- "rwm_restic_bucket": "testbuck",
- "rwm_restic_password": "dummydummydummydummy",
- "rwm_backups": {
+ "s3_endpoint_url": radosuser_admin.url,
+ "s3_access_key": radosuser_admin.access_key,
+ "s3_secret_key": radosuser_admin.secret_key,
+ "restic_bucket": "testbuck",
+ "restic_password": "dummydummydummydummy",
+ "backups": {
"testcfg": {"filesdirs": ["testdatadir/"]}
}
})
- bucket_name = trwm.config["rwm_restic_bucket"]
+ bucket_name = trwm.config.restic_bucket
Path("testdatadir").mkdir()
Path("testdatadir/testdata1.txt").write_text("dummydata", encoding="utf-8")
@@ -296,7 +301,11 @@ def test_storage_delete(tmpworkdir: str, microceph: str, radosuser_admin: rwm.St
def test_storage_list(tmpworkdir: str): # pylint: disable=unused-argument
"""test storage_list"""
- trwm = rwm.RWM({})
+ trwm = rwm.RWM({
+ "s3_endpoint_url": "http://dummy",
+ "s3_access_key": "dummy",
+ "s3_secret_key": "dummy"
+ })
mock = Mock(return_value=[])
with patch.object(rwm.StorageManager, "storage_list", mock):
@@ -307,9 +316,9 @@ def test_storage_info(tmpworkdir: str, microceph: str, radosuser_admin: rwm.Stor
"""test storage_list"""
trwm = rwm.RWM({
- "rwm_s3_endpoint_url": radosuser_admin.url,
- "rwm_s3_access_key": radosuser_admin.access_key,
- "rwm_s3_secret_key": radosuser_admin.secret_key,
+ "s3_endpoint_url": radosuser_admin.url,
+ "s3_access_key": radosuser_admin.access_key,
+ "s3_secret_key": radosuser_admin.secret_key,
})
trwm.storage_create("dummy", "dummy")
@@ -319,7 +328,11 @@ def test_storage_info(tmpworkdir: str, microceph: str, radosuser_admin: rwm.Stor
def test_storage_drop_versions(tmpworkdir: str): # pylint: disable=unused-argument
"""test storage drop versions"""
- trwm = rwm.RWM({})
+ trwm = rwm.RWM({
+ "s3_endpoint_url": "http://dummy",
+ "s3_access_key": "dummy",
+ "s3_secret_key": "dummy"
+ })
mock = Mock(return_value=0)
with patch.object(rwm.StorageManager, "storage_drop_versions", mock):
@@ -330,12 +343,12 @@ def test_storage_restore_state_restic(tmpworkdir: str, radosuser_admin: rwm.Stor
"""test restore bucket from previous saved state"""
trwm = rwm.RWM({
- "rwm_s3_endpoint_url": radosuser_admin.url,
- "rwm_s3_access_key": radosuser_admin.access_key,
- "rwm_s3_secret_key": radosuser_admin.secret_key,
- "rwm_restic_bucket": "restictest",
- "rwm_restic_password": "dummydummydummydummy",
- "rwm_backups": {
+ "s3_endpoint_url": radosuser_admin.url,
+ "s3_access_key": radosuser_admin.access_key,
+ "s3_secret_key": radosuser_admin.secret_key,
+ "restic_bucket": "restictest",
+ "restic_password": "dummydummydummydummy",
+ "backups": {
"testcfg": {
"filesdirs": ["testdatadir/"],
}
@@ -343,7 +356,7 @@ def test_storage_restore_state_restic(tmpworkdir: str, radosuser_admin: rwm.Stor
})
# create and initialize storage
- assert trwm.storage_create(trwm.config["rwm_restic_bucket"], "dummy") == 0
+ assert trwm.storage_create(trwm.config.restic_bucket, "dummy") == 0
assert trwm.restic_cmd(["init"]).returncode == 0
# do backups
@@ -360,17 +373,17 @@ def test_storage_restore_state_restic(tmpworkdir: str, radosuser_admin: rwm.Stor
assert len(snapshots) == 2
assert len(snapshot_files) == 1
assert "/testdatadir/testdata2.txt" == snapshot_files[0]
- states = sorted([x.key for x in trwm.storage_manager.s3.Bucket(trwm.config["rwm_restic_bucket"]).object_versions.filter(Prefix="rwm")])
+ states = sorted([x.key for x in trwm.storage_manager.s3.Bucket(trwm.config.restic_bucket).object_versions.filter(Prefix="rwm")])
assert len(states) == 2
# create restore bucket
- restore_bucket_name = f'{trwm.config["rwm_restic_bucket"]}-restore'
- trwm.storage_restore_state(trwm.config["rwm_restic_bucket"], restore_bucket_name, states[0])
+ restore_bucket_name = f"{trwm.config.restic_bucket}-restore"
+ trwm.storage_restore_state(trwm.config.restic_bucket, restore_bucket_name, states[0])
# check restore bucket contents
trwm_restore = rwm.RWM({
- **trwm.config,
- "rwm_restic_bucket": restore_bucket_name
+ **dict(trwm.config),
+ "restic_bucket": restore_bucket_name
})
snapshots = _restic_list_snapshots(trwm_restore)
snapshot_files = _restic_list_snapshot_files(trwm_restore, snapshots[0]["id"])
diff --git a/tests/test_storage.py b/tests/test_storage.py
index 12bbd19435f42ab488fb9cb2715e7e8ea81d28d6..30f0213393a10345581b98dad1ef3b930b3d0057 100644
--- a/tests/test_storage.py
+++ b/tests/test_storage.py
@@ -129,12 +129,12 @@ def test_storage_backup_usage(
Path("testdir/testdata").write_text('dummy', encoding="utf-8")
trwm = rwm.RWM({
- "rwm_s3_endpoint_url": radosuser_test1.url,
- "rwm_s3_access_key": radosuser_test1.access_key,
- "rwm_s3_secret_key": radosuser_test1.secret_key,
- "rwm_restic_bucket": bucket_name,
- "rwm_restic_password": "dummydummydummydummy",
- "rwm_backups": {
+ "s3_endpoint_url": radosuser_test1.url,
+ "s3_access_key": radosuser_test1.access_key,
+ "s3_secret_key": radosuser_test1.secret_key,
+ "restic_bucket": bucket_name,
+ "restic_password": "dummydummydummydummy",
+ "backups": {
"dummy": {"filesdirs": ["testdir"]}
}
})