Skip to content
Snippets Groups Projects
Commit 2644e4d8 authored by Radoslav Bodó's avatar Radoslav Bodó
Browse files

general: config keys lowercasing

parent 444ebde2
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
rwm.conf.example
+ 9
− 9
View file @ 2644e4d8
# all commands # all commands
RWM_S3_ENDPOINT_URL: "" rwm_s3_endpoint_url: ""
RWM_S3_ACCESS_KEY: "" rwm_s3_access_key: ""
RWM_S3_SECRET_KEY: "" rwm_s3_secret_key: ""
# rclone_crypt # rclone_crypt
RWM_RCLONE_CRYPT_BUCKET: "rwmcrypt" rwm_rclone_crypt_bucket: "rwmcrypt"
RWM_RCLONE_CRYPT_PASSWORD: "" rwm_rclone_crypt_password: ""
# restic, backup # restic, backup
RWM_RESTIC_BUCKET: "rwmcrypt" rwm_restic_bucket: "rwmcrypt"
RWM_RESTIC_PASSWORD: "" rwm_restic_password: ""
# backup # backup
RWM_BACKUPS: [] rwm_backups: []
RWM_RETENTION: [] rwm_retention: []
rwm.py
+ 19
− 19
View file @ 2644e4d8
...@@ -111,15 +111,15 @@ class RWM: ...@@ -111,15 +111,15 @@ class RWM:
env = { env = {
"PATH": os.environ["PATH"], "PATH": os.environ["PATH"],
"AWS_METADATA_SERVICE_NUM_ATTEMPTS": "0", "AWS_METADATA_SERVICE_NUM_ATTEMPTS": "0",
"AWS_ACCESS_KEY_ID": self.config["RWM_S3_ACCESS_KEY"], "AWS_ACCESS_KEY_ID": self.config["rwm_s3_access_key"],
"AWS_SECRET_ACCESS_KEY": self.config["RWM_S3_SECRET_KEY"] "AWS_SECRET_ACCESS_KEY": self.config["rwm_s3_secret_key"]
} }
if is_sublist(["s3", "mb"], args): if is_sublist(["s3", "mb"], args):
# region must be set and empty for awscil >=2.x and ?du? ceph s3 # region must be set and empty for awscil >=2.x and ?du? ceph s3
env.update({"AWS_DEFAULT_REGION": ""}) env.update({"AWS_DEFAULT_REGION": ""})
# aws cli does not have endpoint-url as env config option # aws cli does not have endpoint-url as env config option
return run_command(["aws", "--endpoint-url", self.config["RWM_S3_ENDPOINT_URL"]] + args, env=env) return run_command(["aws", "--endpoint-url", self.config["rwm_s3_endpoint_url"]] + args, env=env)
def rclone_cmd(self, args): def rclone_cmd(self, args):
"""rclone wrapper""" """rclone wrapper"""
...@@ -127,9 +127,9 @@ class RWM: ...@@ -127,9 +127,9 @@ class RWM:
env = { env = {
"RCLONE_CONFIG": "", "RCLONE_CONFIG": "",
"RCLONE_CONFIG_RWMBE_TYPE": "s3", "RCLONE_CONFIG_RWMBE_TYPE": "s3",
"RCLONE_CONFIG_RWMBE_ENDPOINT": self.config["RWM_S3_ENDPOINT_URL"], "RCLONE_CONFIG_RWMBE_ENDPOINT": self.config["rwm_s3_endpoint_url"],
"RCLONE_CONFIG_RWMBE_ACCESS_KEY_ID": self.config["RWM_S3_ACCESS_KEY"], "RCLONE_CONFIG_RWMBE_ACCESS_KEY_ID": self.config["rwm_s3_access_key"],
"RCLONE_CONFIG_RWMBE_SECRET_ACCESS_KEY": self.config["RWM_S3_SECRET_KEY"], "RCLONE_CONFIG_RWMBE_SECRET_ACCESS_KEY": self.config["rwm_s3_secret_key"],
"RCLONE_CONFIG_RWMBE_PROVIDER": "Ceph", "RCLONE_CONFIG_RWMBE_PROVIDER": "Ceph",
"RCLONE_CONFIG_RWMBE_ENV_AUTH": "false", "RCLONE_CONFIG_RWMBE_ENV_AUTH": "false",
"RCLONE_CONFIG_RWMBE_REGION": "", "RCLONE_CONFIG_RWMBE_REGION": "",
...@@ -146,14 +146,14 @@ class RWM: ...@@ -146,14 +146,14 @@ class RWM:
env = { env = {
"RCLONE_CONFIG": "", "RCLONE_CONFIG": "",
"RCLONE_CONFIG_RWMBE_TYPE": "crypt", "RCLONE_CONFIG_RWMBE_TYPE": "crypt",
"RCLONE_CONFIG_RWMBE_REMOTE": f"rwmbes3:/{self.config['RWM_RCLONE_CRYPT_BUCKET']}", "RCLONE_CONFIG_RWMBE_REMOTE": f"rwmbes3:/{self.config['rwm_rclone_crypt_bucket']}",
"RCLONE_CONFIG_RWMBE_PASSWORD": rclone_obscure_password(self.config["RWM_RCLONE_CRYPT_PASSWORD"]), "RCLONE_CONFIG_RWMBE_PASSWORD": rclone_obscure_password(self.config["rwm_rclone_crypt_password"]),
"RCLONE_CONFIG_RWMBE_PASSWORD2": rclone_obscure_password(self.config["RWM_RCLONE_CRYPT_PASSWORD"]), "RCLONE_CONFIG_RWMBE_PASSWORD2": rclone_obscure_password(self.config["rwm_rclone_crypt_password"]),
"RCLONE_CONFIG_RWMBES3_TYPE": "s3", "RCLONE_CONFIG_RWMBES3_TYPE": "s3",
"RCLONE_CONFIG_RWMBES3_ENDPOINT": self.config["RWM_S3_ENDPOINT_URL"], "RCLONE_CONFIG_RWMBES3_ENDPOINT": self.config["rwm_s3_endpoint_url"],
"RCLONE_CONFIG_RWMBES3_ACCESS_KEY_ID": self.config["RWM_S3_ACCESS_KEY"], "RCLONE_CONFIG_RWMBES3_ACCESS_KEY_ID": self.config["rwm_s3_access_key"],
"RCLONE_CONFIG_RWMBES3_SECRET_ACCESS_KEY": self.config["RWM_S3_SECRET_KEY"], "RCLONE_CONFIG_RWMBES3_SECRET_ACCESS_KEY": self.config["rwm_s3_secret_key"],
"RCLONE_CONFIG_RWMBES3_PROVIDER": "Ceph", "RCLONE_CONFIG_RWMBES3_PROVIDER": "Ceph",
"RCLONE_CONFIG_RWMBES3_ENV_AUTH": "false", "RCLONE_CONFIG_RWMBES3_ENV_AUTH": "false",
"RCLONE_CONFIG_RWMBES3_REGION": "", "RCLONE_CONFIG_RWMBES3_REGION": "",
...@@ -166,10 +166,10 @@ class RWM: ...@@ -166,10 +166,10 @@ class RWM:
env = { env = {
"HOME": os.environ["HOME"], "HOME": os.environ["HOME"],
"PATH": os.environ["PATH"], "PATH": os.environ["PATH"],
"AWS_ACCESS_KEY_ID": self.config["RWM_S3_ACCESS_KEY"], "AWS_ACCESS_KEY_ID": self.config["rwm_s3_access_key"],
"AWS_SECRET_ACCESS_KEY": self.config["RWM_S3_SECRET_KEY"], "AWS_SECRET_ACCESS_KEY": self.config["rwm_s3_secret_key"],
"RESTIC_PASSWORD": self.config["RWM_RESTIC_PASSWORD"], "RESTIC_PASSWORD": self.config["rwm_restic_password"],
"RESTIC_REPOSITORY": f"s3:{self.config['RWM_S3_ENDPOINT_URL']}/{self.config['RWM_RESTIC_BUCKET']}", "RESTIC_REPOSITORY": f"s3:{self.config['rwm_s3_endpoint_url']}/{self.config['rwm_restic_bucket']}",
} }
return run_command(["restic"] + args, env=env) return run_command(["restic"] + args, env=env)
...@@ -185,7 +185,7 @@ class RWM: ...@@ -185,7 +185,7 @@ class RWM:
"""runs restic backup by name""" """runs restic backup by name"""
logger.info(f"run restic_backup {name}") logger.info(f"run restic_backup {name}")
conf = self.config["RWM_BACKUPS"][name] conf = self.config["rwm_backups"][name]
excludes = [] excludes = []
for item in conf.get("excludes", []): for item in conf.get("excludes", []):
excludes += ["--exclude", item] excludes += ["--exclude", item]
...@@ -199,7 +199,7 @@ class RWM: ...@@ -199,7 +199,7 @@ class RWM:
logger.info("run restic_forget_prune") logger.info("run restic_forget_prune")
keeps = [] keeps = []
for key, val in self.config.get("RWM_RETENTION", {}).items(): for key, val in self.config.get("rwm_retention", {}).items():
keeps += [f"--{key}", val] keeps += [f"--{key}", val]
cmd_args = ["forget", "--prune"] + keeps cmd_args = ["forget", "--prune"] + keeps
...@@ -241,7 +241,7 @@ class RWM: ...@@ -241,7 +241,7 @@ class RWM:
return autoinit_proc.returncode return autoinit_proc.returncode
stats["_autoinit"] = BackupResult("_autoinit", autoinit_proc.returncode, time_start, time_end) stats["_autoinit"] = BackupResult("_autoinit", autoinit_proc.returncode, time_start, time_end)
for name in self.config["RWM_BACKUPS"].keys(): for name in self.config["rwm_backups"].keys():
time_start = datetime.now() time_start = datetime.now()
wrap_output(backup_proc := self.restic_backup(name)) wrap_output(backup_proc := self.restic_backup(name))
time_end = datetime.now() time_end = datetime.now()
......
tests/conftest.py
+ 1
− 1
View file @ 2644e4d8
...@@ -37,7 +37,7 @@ def tmpworkdir(): ...@@ -37,7 +37,7 @@ def tmpworkdir():
""" """
cwd = os.getcwd() cwd = os.getcwd()
tmpdir = mkdtemp(prefix='rwm_test-') tmpdir = mkdtemp(prefix='rwmtest_')
os.chdir(tmpdir) os.chdir(tmpdir)
yield tmpdir yield tmpdir
os.chdir(cwd) os.chdir(cwd)
......
tests/test_default.py
+ 36
− 36
View file @ 2644e4d8
...@@ -59,9 +59,9 @@ def test_aws_cmd(tmpworkdir: str, motoserver: str): # pylint: disable=unused-ar ...@@ -59,9 +59,9 @@ def test_aws_cmd(tmpworkdir: str, motoserver: str): # pylint: disable=unused-ar
"""test aws command""" """test aws command"""
trwm = RWM({ trwm = RWM({
"RWM_S3_ENDPOINT_URL": motoserver, "rwm_s3_endpoint_url": motoserver,
"RWM_S3_ACCESS_KEY": "dummy", "rwm_s3_access_key": "dummy",
"RWM_S3_SECRET_KEY": "dummy", "rwm_s3_secret_key": "dummy",
}) })
s3 = boto3.client('s3', endpoint_url=motoserver, aws_access_key_id="dummy", aws_secret_access_key="dummy") s3 = boto3.client('s3', endpoint_url=motoserver, aws_access_key_id="dummy", aws_secret_access_key="dummy")
test_bucket = "testbucket" test_bucket = "testbucket"
...@@ -79,9 +79,9 @@ def test_rclone_cmd(tmpworkdir: str, motoserver: str): # pylint: disable=unused ...@@ -79,9 +79,9 @@ def test_rclone_cmd(tmpworkdir: str, motoserver: str): # pylint: disable=unused
"""test rclone command""" """test rclone command"""
trwm = RWM({ trwm = RWM({
"RWM_S3_ENDPOINT_URL": motoserver, "rwm_s3_endpoint_url": motoserver,
"RWM_S3_ACCESS_KEY": "dummy", "rwm_s3_access_key": "dummy",
"RWM_S3_SECRET_KEY": "dummy", "rwm_s3_secret_key": "dummy",
}) })
s3 = boto3.client('s3', endpoint_url=motoserver, aws_access_key_id="dummy", aws_secret_access_key="dummy") s3 = boto3.client('s3', endpoint_url=motoserver, aws_access_key_id="dummy", aws_secret_access_key="dummy")
...@@ -99,11 +99,11 @@ def test_rclone_crypt_cmd(tmpworkdir: str, motoserver: str): # pylint: disable= ...@@ -99,11 +99,11 @@ def test_rclone_crypt_cmd(tmpworkdir: str, motoserver: str): # pylint: disable=
"""test rclone with crypt overlay""" """test rclone with crypt overlay"""
trwm = RWM({ trwm = RWM({
"RWM_S3_ENDPOINT_URL": motoserver, "rwm_s3_endpoint_url": motoserver,
"RWM_S3_ACCESS_KEY": "dummy", "rwm_s3_access_key": "dummy",
"RWM_S3_SECRET_KEY": "dummy", "rwm_s3_secret_key": "dummy",
"RWM_RCLONE_CRYPT_BUCKET": "cryptdata_test", "rwm_rclone_crypt_bucket": "cryptdata_test",
"RWM_RCLONE_CRYPT_PASSWORD": rclone_obscure_password("dummydummydummydummydummydummydummydummy"), "rwm_rclone_crypt_password": rclone_obscure_password("dummydummydummydummy"),
}) })
s3 = boto3.client('s3', endpoint_url=motoserver, aws_access_key_id="dummy", aws_secret_access_key="dummy") s3 = boto3.client('s3', endpoint_url=motoserver, aws_access_key_id="dummy", aws_secret_access_key="dummy")
...@@ -112,30 +112,30 @@ def test_rclone_crypt_cmd(tmpworkdir: str, motoserver: str): # pylint: disable= ...@@ -112,30 +112,30 @@ def test_rclone_crypt_cmd(tmpworkdir: str, motoserver: str): # pylint: disable=
Path(test_file).write_text('1234', encoding='utf-8') Path(test_file).write_text('1234', encoding='utf-8')
trwm.rclone_crypt_cmd(["copy", test_file, f"rwmbe:/{test_bucket}/"]) trwm.rclone_crypt_cmd(["copy", test_file, f"rwmbe:/{test_bucket}/"])
assert len(objects_plain_list(s3.list_objects_v2(Bucket=trwm.config["RWM_RCLONE_CRYPT_BUCKET"]))) == 1 assert len(objects_plain_list(s3.list_objects_v2(Bucket=trwm.config["rwm_rclone_crypt_bucket"]))) == 1
trwm.rclone_crypt_cmd(["delete", f"rwmbe:/{test_bucket}/{test_file}"]) trwm.rclone_crypt_cmd(["delete", f"rwmbe:/{test_bucket}/{test_file}"])
assert s3.list_objects_v2(Bucket=trwm.config["RWM_RCLONE_CRYPT_BUCKET"])["KeyCount"] == 0 assert s3.list_objects_v2(Bucket=trwm.config["rwm_rclone_crypt_bucket"])["KeyCount"] == 0
test_file1 = "testfile1.txt" test_file1 = "testfile1.txt"
Path(test_file1).write_text('4321', encoding='utf-8') Path(test_file1).write_text('4321', encoding='utf-8')
trwm.rclone_crypt_cmd(["sync", ".", f"rwmbe:/{test_bucket}/"]) trwm.rclone_crypt_cmd(["sync", ".", f"rwmbe:/{test_bucket}/"])
assert s3.list_objects_v2(Bucket=trwm.config["RWM_RCLONE_CRYPT_BUCKET"])["KeyCount"] == 2 assert s3.list_objects_v2(Bucket=trwm.config["rwm_rclone_crypt_bucket"])["KeyCount"] == 2
Path(test_file1).unlink() Path(test_file1).unlink()
trwm.rclone_crypt_cmd(["sync", ".", f"rwmbe:/{test_bucket}/"]) trwm.rclone_crypt_cmd(["sync", ".", f"rwmbe:/{test_bucket}/"])
assert s3.list_objects_v2(Bucket=trwm.config["RWM_RCLONE_CRYPT_BUCKET"])["KeyCount"] == 1 assert s3.list_objects_v2(Bucket=trwm.config["rwm_rclone_crypt_bucket"])["KeyCount"] == 1
def test_restic_cmd(tmpworkdir: str, motoserver: str): # pylint: disable=unused-argument def test_restic_cmd(tmpworkdir: str, motoserver: str): # pylint: disable=unused-argument
"""test restic command""" """test restic command"""
trwm = RWM({ trwm = RWM({
"RWM_S3_ENDPOINT_URL": motoserver, "rwm_s3_endpoint_url": motoserver,
"RWM_S3_ACCESS_KEY": "dummy", "rwm_s3_access_key": "dummy",
"RWM_S3_SECRET_KEY": "dummy", "rwm_s3_secret_key": "dummy",
"RWM_RESTIC_BUCKET": "restictest", "rwm_restic_bucket": "restictest",
"RWM_RESTIC_PASSWORD": "dummydummydummydummydummydummydummydummy", "rwm_restic_password": "dummydummydummydummy",
}) })
assert trwm.restic_cmd(["init"]).returncode == 0 assert trwm.restic_cmd(["init"]).returncode == 0
...@@ -167,19 +167,19 @@ def test_backup_cmd(tmpworkdir: str, motoserver: str): # pylint: disable=unused ...@@ -167,19 +167,19 @@ def test_backup_cmd(tmpworkdir: str, motoserver: str): # pylint: disable=unused
"""test backup_cmd command""" """test backup_cmd command"""
trwm = RWM({ trwm = RWM({
"RWM_S3_ENDPOINT_URL": motoserver, "rwm_s3_endpoint_url": motoserver,
"RWM_S3_ACCESS_KEY": "dummy", "rwm_s3_access_key": "dummy",
"RWM_S3_SECRET_KEY": "dummy", "rwm_s3_secret_key": "dummy",
"RWM_RESTIC_BUCKET": "restictest", "rwm_restic_bucket": "restictest",
"RWM_RESTIC_PASSWORD": "dummydummydummydummydummydummydummydummy", "rwm_restic_password": "dummydummydummydummy",
"RWM_BACKUPS": { "rwm_backups": {
"testcfg": { "testcfg": {
"filesdirs": ["testdatadir/"], "filesdirs": ["testdatadir/"],
"excludes": ["testfile_to_be_ignored"], "excludes": ["testfile_to_be_ignored"],
"extras": ["--tag", "dummytag"], "extras": ["--tag", "dummytag"],
} }
}, },
"RWM_RETENTION": { "rwm_retention": {
"keep-daily": "1" "keep-daily": "1"
} }
}) })
...@@ -200,12 +200,12 @@ def test_backup_cmd_excludes(tmpworkdir: str, motoserver: str): # pylint: disab ...@@ -200,12 +200,12 @@ def test_backup_cmd_excludes(tmpworkdir: str, motoserver: str): # pylint: disab
"""test backup command""" """test backup command"""
trwm = RWM({ trwm = RWM({
"RWM_S3_ENDPOINT_URL": motoserver, "rwm_s3_endpoint_url": motoserver,
"RWM_S3_ACCESS_KEY": "dummy", "rwm_s3_access_key": "dummy",
"RWM_S3_SECRET_KEY": "dummy", "rwm_s3_secret_key": "dummy",
"RWM_RESTIC_BUCKET": "restictest", "rwm_restic_bucket": "restictest",
"RWM_RESTIC_PASSWORD": "dummydummydummydummydummydummydummydummy", "rwm_restic_password": "dummydummydummydummy",
"RWM_BACKUPS": { "rwm_backups": {
"testcfg": { "testcfg": {
"filesdirs": ["testdatadir"], "filesdirs": ["testdatadir"],
"excludes": ["proc", "*.ignored"], "excludes": ["proc", "*.ignored"],
...@@ -235,7 +235,7 @@ def test_backup_cmd_error_handling(tmpworkdir: str, motoserver: str): # pylint: ...@@ -235,7 +235,7 @@ def test_backup_cmd_error_handling(tmpworkdir: str, motoserver: str): # pylint:
"""test backup command err cases""" """test backup command err cases"""
rwm_conf = { rwm_conf = {
"RWM_BACKUPS": { "rwm_backups": {
"dummycfg": {"filesdirs": ["dummydir"]} "dummycfg": {"filesdirs": ["dummydir"]}
} }
} }
...@@ -263,7 +263,7 @@ def test_backup_all_cmd(tmpworkdir: str): # pylint: disable=unused-argument ...@@ -263,7 +263,7 @@ def test_backup_all_cmd(tmpworkdir: str): # pylint: disable=unused-argument
"""test backup command err cases""" """test backup command err cases"""
rwm_conf = { rwm_conf = {
"RWM_BACKUPS": { "rwm_backups": {
"dummycfg": {"filesdirs": ["dummydir"]} "dummycfg": {"filesdirs": ["dummydir"]}
} }
} }
...@@ -281,7 +281,7 @@ def test_backup_all_cmd_error_handling(tmpworkdir: str): # pylint: disable=unus ...@@ -281,7 +281,7 @@ def test_backup_all_cmd_error_handling(tmpworkdir: str): # pylint: disable=unus
"""test backup command err cases""" """test backup command err cases"""
rwm_conf = { rwm_conf = {
"RWM_BACKUPS": { "rwm_backups": {
"dummycfg": {"filesdirs": ["dummydir"]} "dummycfg": {"filesdirs": ["dummydir"]}
} }
} }
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment