Skip to content
Snippets Groups Projects
Commit 2644e4d8 authored by Radoslav Bodó's avatar Radoslav Bodó
Browse files

general: config keys lowercasing

parent 444ebde2
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
rwm.conf.example
+ 9
− 9
View file @ 2644e4d8
# all commands
RWM_S3_ENDPOINT_URL: ""
RWM_S3_ACCESS_KEY: ""
RWM_S3_SECRET_KEY: ""
rwm_s3_endpoint_url: ""
rwm_s3_access_key: ""
rwm_s3_secret_key: ""
# rclone_crypt
RWM_RCLONE_CRYPT_BUCKET: "rwmcrypt"
RWM_RCLONE_CRYPT_PASSWORD: ""
rwm_rclone_crypt_bucket: "rwmcrypt"
rwm_rclone_crypt_password: ""
# restic, backup
RWM_RESTIC_BUCKET: "rwmcrypt"
RWM_RESTIC_PASSWORD: ""
rwm_restic_bucket: "rwmcrypt"
rwm_restic_password: ""
# backup
RWM_BACKUPS: []
RWM_RETENTION: []
rwm_backups: []
rwm_retention: []
rwm.py
+ 19
− 19
View file @ 2644e4d8
......@@ -111,15 +111,15 @@ class RWM:
env = {
"PATH": os.environ["PATH"],
"AWS_METADATA_SERVICE_NUM_ATTEMPTS": "0",
"AWS_ACCESS_KEY_ID": self.config["RWM_S3_ACCESS_KEY"],
"AWS_SECRET_ACCESS_KEY": self.config["RWM_S3_SECRET_KEY"]
"AWS_ACCESS_KEY_ID": self.config["rwm_s3_access_key"],
"AWS_SECRET_ACCESS_KEY": self.config["rwm_s3_secret_key"]
}
if is_sublist(["s3", "mb"], args):
# region must be set and empty for awscil >=2.x and ?du? ceph s3
env.update({"AWS_DEFAULT_REGION": ""})
# aws cli does not have endpoint-url as env config option
return run_command(["aws", "--endpoint-url", self.config["RWM_S3_ENDPOINT_URL"]] + args, env=env)
return run_command(["aws", "--endpoint-url", self.config["rwm_s3_endpoint_url"]] + args, env=env)
def rclone_cmd(self, args):
"""rclone wrapper"""
......@@ -127,9 +127,9 @@ class RWM:
env = {
"RCLONE_CONFIG": "",
"RCLONE_CONFIG_RWMBE_TYPE": "s3",
"RCLONE_CONFIG_RWMBE_ENDPOINT": self.config["RWM_S3_ENDPOINT_URL"],
"RCLONE_CONFIG_RWMBE_ACCESS_KEY_ID": self.config["RWM_S3_ACCESS_KEY"],
"RCLONE_CONFIG_RWMBE_SECRET_ACCESS_KEY": self.config["RWM_S3_SECRET_KEY"],
"RCLONE_CONFIG_RWMBE_ENDPOINT": self.config["rwm_s3_endpoint_url"],
"RCLONE_CONFIG_RWMBE_ACCESS_KEY_ID": self.config["rwm_s3_access_key"],
"RCLONE_CONFIG_RWMBE_SECRET_ACCESS_KEY": self.config["rwm_s3_secret_key"],
"RCLONE_CONFIG_RWMBE_PROVIDER": "Ceph",
"RCLONE_CONFIG_RWMBE_ENV_AUTH": "false",
"RCLONE_CONFIG_RWMBE_REGION": "",
......@@ -146,14 +146,14 @@ class RWM:
env = {
"RCLONE_CONFIG": "",
"RCLONE_CONFIG_RWMBE_TYPE": "crypt",
"RCLONE_CONFIG_RWMBE_REMOTE": f"rwmbes3:/{self.config['RWM_RCLONE_CRYPT_BUCKET']}",
"RCLONE_CONFIG_RWMBE_PASSWORD": rclone_obscure_password(self.config["RWM_RCLONE_CRYPT_PASSWORD"]),
"RCLONE_CONFIG_RWMBE_PASSWORD2": rclone_obscure_password(self.config["RWM_RCLONE_CRYPT_PASSWORD"]),
"RCLONE_CONFIG_RWMBE_REMOTE": f"rwmbes3:/{self.config['rwm_rclone_crypt_bucket']}",
"RCLONE_CONFIG_RWMBE_PASSWORD": rclone_obscure_password(self.config["rwm_rclone_crypt_password"]),
"RCLONE_CONFIG_RWMBE_PASSWORD2": rclone_obscure_password(self.config["rwm_rclone_crypt_password"]),
"RCLONE_CONFIG_RWMBES3_TYPE": "s3",
"RCLONE_CONFIG_RWMBES3_ENDPOINT": self.config["RWM_S3_ENDPOINT_URL"],
"RCLONE_CONFIG_RWMBES3_ACCESS_KEY_ID": self.config["RWM_S3_ACCESS_KEY"],
"RCLONE_CONFIG_RWMBES3_SECRET_ACCESS_KEY": self.config["RWM_S3_SECRET_KEY"],
"RCLONE_CONFIG_RWMBES3_ENDPOINT": self.config["rwm_s3_endpoint_url"],
"RCLONE_CONFIG_RWMBES3_ACCESS_KEY_ID": self.config["rwm_s3_access_key"],
"RCLONE_CONFIG_RWMBES3_SECRET_ACCESS_KEY": self.config["rwm_s3_secret_key"],
"RCLONE_CONFIG_RWMBES3_PROVIDER": "Ceph",
"RCLONE_CONFIG_RWMBES3_ENV_AUTH": "false",
"RCLONE_CONFIG_RWMBES3_REGION": "",
......@@ -166,10 +166,10 @@ class RWM:
env = {
"HOME": os.environ["HOME"],
"PATH": os.environ["PATH"],
"AWS_ACCESS_KEY_ID": self.config["RWM_S3_ACCESS_KEY"],
"AWS_SECRET_ACCESS_KEY": self.config["RWM_S3_SECRET_KEY"],
"RESTIC_PASSWORD": self.config["RWM_RESTIC_PASSWORD"],
"RESTIC_REPOSITORY": f"s3:{self.config['RWM_S3_ENDPOINT_URL']}/{self.config['RWM_RESTIC_BUCKET']}",
"AWS_ACCESS_KEY_ID": self.config["rwm_s3_access_key"],
"AWS_SECRET_ACCESS_KEY": self.config["rwm_s3_secret_key"],
"RESTIC_PASSWORD": self.config["rwm_restic_password"],
"RESTIC_REPOSITORY": f"s3:{self.config['rwm_s3_endpoint_url']}/{self.config['rwm_restic_bucket']}",
}
return run_command(["restic"] + args, env=env)
......@@ -185,7 +185,7 @@ class RWM:
"""runs restic backup by name"""
logger.info(f"run restic_backup {name}")
conf = self.config["RWM_BACKUPS"][name]
conf = self.config["rwm_backups"][name]
excludes = []
for item in conf.get("excludes", []):
excludes += ["--exclude", item]
......@@ -199,7 +199,7 @@ class RWM:
logger.info("run restic_forget_prune")
keeps = []
for key, val in self.config.get("RWM_RETENTION", {}).items():
for key, val in self.config.get("rwm_retention", {}).items():
keeps += [f"--{key}", val]
cmd_args = ["forget", "--prune"] + keeps
......@@ -241,7 +241,7 @@ class RWM:
return autoinit_proc.returncode
stats["_autoinit"] = BackupResult("_autoinit", autoinit_proc.returncode, time_start, time_end)
for name in self.config["RWM_BACKUPS"].keys():
for name in self.config["rwm_backups"].keys():
time_start = datetime.now()
wrap_output(backup_proc := self.restic_backup(name))
time_end = datetime.now()
......
tests/conftest.py
+ 1
− 1
View file @ 2644e4d8
......@@ -37,7 +37,7 @@ def tmpworkdir():
"""
cwd = os.getcwd()
tmpdir = mkdtemp(prefix='rwm_test-')
tmpdir = mkdtemp(prefix='rwmtest_')
os.chdir(tmpdir)
yield tmpdir
os.chdir(cwd)
......
tests/test_default.py
+ 36
− 36
View file @ 2644e4d8
......@@ -59,9 +59,9 @@ def test_aws_cmd(tmpworkdir: str, motoserver: str): # pylint: disable=unused-ar
"""test aws command"""
trwm = RWM({
"RWM_S3_ENDPOINT_URL": motoserver,
"RWM_S3_ACCESS_KEY": "dummy",
"RWM_S3_SECRET_KEY": "dummy",
"rwm_s3_endpoint_url": motoserver,
"rwm_s3_access_key": "dummy",
"rwm_s3_secret_key": "dummy",
})
s3 = boto3.client('s3', endpoint_url=motoserver, aws_access_key_id="dummy", aws_secret_access_key="dummy")
test_bucket = "testbucket"
......@@ -79,9 +79,9 @@ def test_rclone_cmd(tmpworkdir: str, motoserver: str): # pylint: disable=unused
"""test rclone command"""
trwm = RWM({
"RWM_S3_ENDPOINT_URL": motoserver,
"RWM_S3_ACCESS_KEY": "dummy",
"RWM_S3_SECRET_KEY": "dummy",
"rwm_s3_endpoint_url": motoserver,
"rwm_s3_access_key": "dummy",
"rwm_s3_secret_key": "dummy",
})
s3 = boto3.client('s3', endpoint_url=motoserver, aws_access_key_id="dummy", aws_secret_access_key="dummy")
......@@ -99,11 +99,11 @@ def test_rclone_crypt_cmd(tmpworkdir: str, motoserver: str): # pylint: disable=
"""test rclone with crypt overlay"""
trwm = RWM({
"RWM_S3_ENDPOINT_URL": motoserver,
"RWM_S3_ACCESS_KEY": "dummy",
"RWM_S3_SECRET_KEY": "dummy",
"RWM_RCLONE_CRYPT_BUCKET": "cryptdata_test",
"RWM_RCLONE_CRYPT_PASSWORD": rclone_obscure_password("dummydummydummydummydummydummydummydummy"),
"rwm_s3_endpoint_url": motoserver,
"rwm_s3_access_key": "dummy",
"rwm_s3_secret_key": "dummy",
"rwm_rclone_crypt_bucket": "cryptdata_test",
"rwm_rclone_crypt_password": rclone_obscure_password("dummydummydummydummy"),
})
s3 = boto3.client('s3', endpoint_url=motoserver, aws_access_key_id="dummy", aws_secret_access_key="dummy")
......@@ -112,30 +112,30 @@ def test_rclone_crypt_cmd(tmpworkdir: str, motoserver: str): # pylint: disable=
Path(test_file).write_text('1234', encoding='utf-8')
trwm.rclone_crypt_cmd(["copy", test_file, f"rwmbe:/{test_bucket}/"])
assert len(objects_plain_list(s3.list_objects_v2(Bucket=trwm.config["RWM_RCLONE_CRYPT_BUCKET"]))) == 1
assert len(objects_plain_list(s3.list_objects_v2(Bucket=trwm.config["rwm_rclone_crypt_bucket"]))) == 1
trwm.rclone_crypt_cmd(["delete", f"rwmbe:/{test_bucket}/{test_file}"])
assert s3.list_objects_v2(Bucket=trwm.config["RWM_RCLONE_CRYPT_BUCKET"])["KeyCount"] == 0
assert s3.list_objects_v2(Bucket=trwm.config["rwm_rclone_crypt_bucket"])["KeyCount"] == 0
test_file1 = "testfile1.txt"
Path(test_file1).write_text('4321', encoding='utf-8')
trwm.rclone_crypt_cmd(["sync", ".", f"rwmbe:/{test_bucket}/"])
assert s3.list_objects_v2(Bucket=trwm.config["RWM_RCLONE_CRYPT_BUCKET"])["KeyCount"] == 2
assert s3.list_objects_v2(Bucket=trwm.config["rwm_rclone_crypt_bucket"])["KeyCount"] == 2
Path(test_file1).unlink()
trwm.rclone_crypt_cmd(["sync", ".", f"rwmbe:/{test_bucket}/"])
assert s3.list_objects_v2(Bucket=trwm.config["RWM_RCLONE_CRYPT_BUCKET"])["KeyCount"] == 1
assert s3.list_objects_v2(Bucket=trwm.config["rwm_rclone_crypt_bucket"])["KeyCount"] == 1
def test_restic_cmd(tmpworkdir: str, motoserver: str): # pylint: disable=unused-argument
"""test restic command"""
trwm = RWM({
"RWM_S3_ENDPOINT_URL": motoserver,
"RWM_S3_ACCESS_KEY": "dummy",
"RWM_S3_SECRET_KEY": "dummy",
"RWM_RESTIC_BUCKET": "restictest",
"RWM_RESTIC_PASSWORD": "dummydummydummydummydummydummydummydummy",
"rwm_s3_endpoint_url": motoserver,
"rwm_s3_access_key": "dummy",
"rwm_s3_secret_key": "dummy",
"rwm_restic_bucket": "restictest",
"rwm_restic_password": "dummydummydummydummy",
})
assert trwm.restic_cmd(["init"]).returncode == 0
......@@ -167,19 +167,19 @@ def test_backup_cmd(tmpworkdir: str, motoserver: str): # pylint: disable=unused
"""test backup_cmd command"""
trwm = RWM({
"RWM_S3_ENDPOINT_URL": motoserver,
"RWM_S3_ACCESS_KEY": "dummy",
"RWM_S3_SECRET_KEY": "dummy",
"RWM_RESTIC_BUCKET": "restictest",
"RWM_RESTIC_PASSWORD": "dummydummydummydummydummydummydummydummy",
"RWM_BACKUPS": {
"rwm_s3_endpoint_url": motoserver,
"rwm_s3_access_key": "dummy",
"rwm_s3_secret_key": "dummy",
"rwm_restic_bucket": "restictest",
"rwm_restic_password": "dummydummydummydummy",
"rwm_backups": {
"testcfg": {
"filesdirs": ["testdatadir/"],
"excludes": ["testfile_to_be_ignored"],
"extras": ["--tag", "dummytag"],
}
},
"RWM_RETENTION": {
"rwm_retention": {
"keep-daily": "1"
}
})
......@@ -200,12 +200,12 @@ def test_backup_cmd_excludes(tmpworkdir: str, motoserver: str): # pylint: disab
"""test backup command"""
trwm = RWM({
"RWM_S3_ENDPOINT_URL": motoserver,
"RWM_S3_ACCESS_KEY": "dummy",
"RWM_S3_SECRET_KEY": "dummy",
"RWM_RESTIC_BUCKET": "restictest",
"RWM_RESTIC_PASSWORD": "dummydummydummydummydummydummydummydummy",
"RWM_BACKUPS": {
"rwm_s3_endpoint_url": motoserver,
"rwm_s3_access_key": "dummy",
"rwm_s3_secret_key": "dummy",
"rwm_restic_bucket": "restictest",
"rwm_restic_password": "dummydummydummydummy",
"rwm_backups": {
"testcfg": {
"filesdirs": ["testdatadir"],
"excludes": ["proc", "*.ignored"],
......@@ -235,7 +235,7 @@ def test_backup_cmd_error_handling(tmpworkdir: str, motoserver: str): # pylint:
"""test backup command err cases"""
rwm_conf = {
"RWM_BACKUPS": {
"rwm_backups": {
"dummycfg": {"filesdirs": ["dummydir"]}
}
}
......@@ -263,7 +263,7 @@ def test_backup_all_cmd(tmpworkdir: str): # pylint: disable=unused-argument
"""test backup command err cases"""
rwm_conf = {
"RWM_BACKUPS": {
"rwm_backups": {
"dummycfg": {"filesdirs": ["dummydir"]}
}
}
......@@ -281,7 +281,7 @@ def test_backup_all_cmd_error_handling(tmpworkdir: str): # pylint: disable=unus
"""test backup command err cases"""
rwm_conf = {
"RWM_BACKUPS": {
"rwm_backups": {
"dummycfg": {"filesdirs": ["dummydir"]}
}
}
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment