diff --git a/Makefile b/Makefile index e6ab5102c2bd8522ddf1a811f8b4d8e62fe1df3b..092749561a1386272b842f11c59f5de044a0f0e0 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ all: coverage lint install: - apt-get -y install awscli python3-boto3 python3-cryptography python3-tabulate rclone restic yamllint + apt-get -y install awscli python3-boto3 python3-tabulate restic yamllint install-dev: apt-get -y install python3-venv snapd diff --git a/README.md b/README.md index 5cd5138650aedfd0241fd41b27909df1c15d84f0..3d93d8144514800dbbe3e2113448794ede7a7793 100644 --- a/README.md +++ b/README.md @@ -29,8 +29,7 @@ credentials for the managed bucket. ## Features -* low-level S3 access for aws cli and rclone -* rclone "crypt over S3" backend +* low-level S3 access for aws cli * restic with S3 repository * simple backup manager/executor * storage manager @@ -43,11 +42,14 @@ TODO: * recreate bucket contents on local filesystem (or remote bucket) acording to specified state data * ??? check completeness of the current state of the bucket -* unlike in other backup solutions, attacker with credentials can restore - old data from the repository/bucket, this should be discussed (howto threat modeling ?) -* rgw leaks objects on tests -* drop rclone use-cases + +## Known issues + +* During tests RGW is suspected to leak RADOS objects + +* Unlike in other backup solutions, attacker with credentials can restore + old data from the repository/bucket, this should be discussed (howto threat modeling ?) ## Usage @@ -103,21 +105,11 @@ rwm --confg admin.conf storage_drop_versions bucket1 #### AWS cli ``` -cp examples/rwm-rclone.conf rwm.conf +cp examples/rwm-restic.conf rwm.conf rwm aws s3 ls s3:// rwm aws s3api list-buckets -rwm rclone lsd rwmbe:/ ``` -#### rclone with crypt overlay - -rclone_crypt defines single default remote named "rwmbe:/" pointed to `rwm_rclone_crypt_bucket` path. - -``` -cp examples/rwm-rclone.conf rwm.conf -rwm rclone_crypt sync /data rwmbe:/ -rwm rclone_crypt lsl rwmbe:/ -``` #### Restic: manual restic backup @@ -133,7 +125,7 @@ rwm restic mount /mnt/restore ## Notes * executed tools stdout is buffered, eg. `restic mount` does not print immediate output as normal -* passthrough full arguments to underlying tool with "--" (eg. `rwm rclone -- ls --help`). +* passthrough full arguments to underlying tool with "--" (eg. `rwm aws -- s3api --help`). * runner microceph breaks on reboot because of symlink at /etc/ceph diff --git a/examples/rwm-backups.conf b/examples/rwm-backups.conf index 93899e43fb6fd0ddadf9aa6203553ff548234f72..b2477759e6cd0b9a20700d015a2f2954faf3e270 100644 --- a/examples/rwm-backups.conf +++ b/examples/rwm-backups.conf @@ -1,4 +1,4 @@ -# rwm aws, rclone, restic, backup, backup_all +# rwm aws, restic, backup, backup_all rwm_s3_endpoint_url: "" rwm_s3_access_key: "" diff --git a/examples/rwm-linux.conf b/examples/rwm-linux.conf index 58c6c1b64f2b0d59b40c171080d8b32b23ec8dac..139d090fc54d48e33564e3411aea34b628657840 100644 --- a/examples/rwm-linux.conf +++ b/examples/rwm-linux.conf @@ -1,4 +1,4 @@ -# rwm aws, rclone, restic, backup, backup_all +# rwm aws, restic, backup, backup_all rwm_s3_endpoint_url: "" rwm_s3_access_key: "" diff --git a/examples/rwm-rclone.conf b/examples/rwm-rclone.conf deleted file mode 100644 index f287c8bba400c7d84330dbbd97158712c7835232..0000000000000000000000000000000000000000 --- a/examples/rwm-rclone.conf +++ /dev/null @@ -1,8 +0,0 @@ -# rwm aws, rclone, rclone_crypt - -rwm_s3_endpoint_url: "" -rwm_s3_access_key: "" -rwm_s3_secret_key: "" - -rwm_rclone_crypt_bucket: "rwmcrypt" -rwm_rclone_crypt_password: "" diff --git a/examples/rwm-restic.conf b/examples/rwm-restic.conf index 70a4d5a98a8830b5b36f51e29d7318661a201a35..beeee3e905dcff057f36ee94930924f2a30317c3 100644 --- a/examples/rwm-restic.conf +++ b/examples/rwm-restic.conf @@ -1,4 +1,4 @@ -# rwm aws, rclone, restic +# rwm aws, restic rwm_s3_endpoint_url: "" rwm_s3_access_key: "" diff --git a/requirements.lock b/requirements.lock index aa7dd3d80f9d1bc94462a13ca9d170ceb1599fe2..606c2ee5d5518f9956a9249bb58c1767e140a98b 100644 --- a/requirements.lock +++ b/requirements.lock @@ -1,14 +1,15 @@ annotated-types==0.6.0 +antlr4-python3-runtime==4.13.1 astroid==3.1.0 attrs==23.2.0 -aws-sam-translator==1.86.0 +aws-sam-translator==1.87.0 aws-xray-sdk==2.13.0 blinker==1.7.0 -boto3==1.34.69 -botocore==1.34.69 +boto3==1.34.81 +botocore==1.34.81 certifi==2024.2.2 cffi==1.16.0 -cfn-lint==0.86.1 +cfn-lint==0.86.2 charset-normalizer==3.3.2 click==8.1.7 coverage==7.4.4 @@ -16,7 +17,7 @@ cryptography==42.0.5 dill==0.3.8 docker==7.0.0 flake8==7.0.0 -Flask==3.0.2 +Flask==3.0.3 Flask-Cors==4.0.0 graphql-core==3.2.3 idna==3.6 @@ -29,6 +30,7 @@ joserfc==0.9.0 jschema-to-python==1.2.3 jsondiff==2.0.0 jsonpatch==1.33 +jsonpath-ng==1.6.1 jsonpickle==3.0.3 jsonpointer==2.4 jsonschema==4.21.1 @@ -38,9 +40,9 @@ junit-xml==1.9 lazy-object-proxy==1.10.0 MarkupSafe==2.1.5 mccabe==0.7.0 -moto==5.0.3 +moto==5.0.5 mpmath==1.3.0 -networkx==3.2.1 +networkx==3.3 openapi-schema-validator==0.6.2 openapi-spec-validator==0.7.1 packaging==24.0 @@ -48,18 +50,18 @@ pathable==0.4.3 pbr==6.0.0 platformdirs==4.2.0 pluggy==1.4.0 +ply==3.11 psutil==5.9.8 -py-partiql-parser==0.5.1 +py-partiql-parser==0.5.4 pycodestyle==2.11.1 -pycparser==2.21 -pycryptodome==3.20.0 +pycparser==2.22 pydantic==2.6.4 pydantic_core==2.16.3 pyflakes==3.2.0 pylint==3.1.0 pyparsing==3.1.2 pytest==8.1.1 -pytest-xprocess==1.0.0 +pytest-xprocess==1.0.1 python-dateutil==2.9.0.post0 PyYAML==6.0.1 referencing==0.31.1 @@ -74,8 +76,8 @@ six==1.16.0 sympy==1.12 tabulate==0.9.0 tomlkit==0.12.4 -typing_extensions==4.10.0 +typing_extensions==4.11.0 urllib3==2.2.1 -Werkzeug==3.0.1 +Werkzeug==3.0.2 wrapt==1.16.0 xmltodict==0.13.0 diff --git a/requirements.txt b/requirements.txt index 0f055546e5a8217e4b52cc6e657e8ef4712347b3..eb406a46e589fc803692de4966e98497fe9d6b7f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ # runtime -cryptography tabulate # dev diff --git a/rwm.py b/rwm.py index a70765a4cb7f0950227b214309672a744a8813bc..c09d42dedbfcf68baa64aeb0b5841c75db11e672 100755 --- a/rwm.py +++ b/rwm.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 """rwm, restic/s3 worm manager""" -import base64 import dataclasses import json import logging @@ -17,8 +16,6 @@ from pathlib import Path import boto3 import botocore import yaml -from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes -from cryptography.hazmat.backends import default_backend from tabulate import tabulate @@ -66,21 +63,6 @@ def wrap_output(process): return process.returncode -def rclone_obscure_password(plaintext, iv=None): - """rclone obscure password algorithm""" - - # https://github.com/rclone/rclone/blob/master/fs/config/obscure/obscure.go - # https://github.com/maaaaz/rclonedeobscure - # GTP translate to python cryptography - - secret_key = b"\x9c\x93\x5b\x48\x73\x0a\x55\x4d\x6b\xfd\x7c\x63\xc8\x86\xa9\x2b\xd3\x90\x19\x8e\xb8\x12\x8a\xfb\xf4\xde\x16\x2b\x8b\x95\xf6\x38" - if not iv: - iv = os.urandom(16) - encryptor = Cipher(algorithms.AES(secret_key), modes.CTR(iv), backend=default_backend()).encryptor() - data = iv + encryptor.update(plaintext.encode()) + encryptor.finalize() - return base64.urlsafe_b64encode(data).decode().rstrip("=") - - @dataclasses.dataclass class BackupResult: """backup results data container""" @@ -323,45 +305,6 @@ class RWM: # aws cli does not have endpoint-url as env config option return run_command(["aws", "--endpoint-url", self.config["rwm_s3_endpoint_url"]] + args, env=env) - def rclone_cmd(self, args) -> subprocess.CompletedProcess: - """rclone wrapper""" - - env = { - "RCLONE_CONFIG": "", - "RCLONE_CONFIG_RWMBE_TYPE": "s3", - "RCLONE_CONFIG_RWMBE_ENDPOINT": self.config["rwm_s3_endpoint_url"], - "RCLONE_CONFIG_RWMBE_ACCESS_KEY_ID": self.config["rwm_s3_access_key"], - "RCLONE_CONFIG_RWMBE_SECRET_ACCESS_KEY": self.config["rwm_s3_secret_key"], - "RCLONE_CONFIG_RWMBE_PROVIDER": "Ceph", - "RCLONE_CONFIG_RWMBE_ENV_AUTH": "false", - "RCLONE_CONFIG_RWMBE_REGION": "", - } - return run_command(["rclone"] + args, env=env) - - def rclone_crypt_cmd(self, args) -> subprocess.CompletedProcess: - """ - rclone crypt wrapper - * https://rclone.org/docs/#config-file - * https://rclone.org/crypt/ - """ - - env = { - "RCLONE_CONFIG": "", - "RCLONE_CONFIG_RWMBE_TYPE": "crypt", - "RCLONE_CONFIG_RWMBE_REMOTE": f"rwmbes3:/{self.config['rwm_rclone_crypt_bucket']}", - "RCLONE_CONFIG_RWMBE_PASSWORD": rclone_obscure_password(self.config["rwm_rclone_crypt_password"]), - "RCLONE_CONFIG_RWMBE_PASSWORD2": rclone_obscure_password(self.config["rwm_rclone_crypt_password"]), - - "RCLONE_CONFIG_RWMBES3_TYPE": "s3", - "RCLONE_CONFIG_RWMBES3_ENDPOINT": self.config["rwm_s3_endpoint_url"], - "RCLONE_CONFIG_RWMBES3_ACCESS_KEY_ID": self.config["rwm_s3_access_key"], - "RCLONE_CONFIG_RWMBES3_SECRET_ACCESS_KEY": self.config["rwm_s3_secret_key"], - "RCLONE_CONFIG_RWMBES3_PROVIDER": "Ceph", - "RCLONE_CONFIG_RWMBES3_ENV_AUTH": "false", - "RCLONE_CONFIG_RWMBES3_REGION": "", - } - return run_command(["rclone"] + args, env=env) - def restic_cmd(self, args) -> subprocess.CompletedProcess: """restic command wrapper""" @@ -516,10 +459,6 @@ def parse_arguments(argv): aws_cmd_parser = subparsers.add_parser("aws", help="run aws cli") aws_cmd_parser.add_argument("cmd_args", nargs="*") - rclone_cmd_parser = subparsers.add_parser("rclone", help="run rclone") - rclone_cmd_parser.add_argument("cmd_args", nargs="*") - rclone_crypt_cmd_parser = subparsers.add_parser("rclone_crypt", help="run rclone with crypt overlay") - rclone_crypt_cmd_parser.add_argument("cmd_args", nargs="*") restic_cmd_parser = subparsers.add_parser("restic", help="run restic") restic_cmd_parser.add_argument("cmd_args", nargs="*") @@ -571,10 +510,6 @@ def main(argv=None): # pylint: disable=too-many-branches if args.command == "aws": ret = wrap_output(rwmi.aws_cmd(args.cmd_args)) - if args.command == "rclone": - ret = wrap_output(rwmi.rclone_cmd(args.cmd_args)) - if args.command == "rclone_crypt": - ret = wrap_output(rwmi.rclone_crypt_cmd(args.cmd_args)) if args.command == "restic": ret = wrap_output(rwmi.restic_cmd(args.cmd_args)) diff --git a/tests/test_default.py b/tests/test_default.py index 59547ff1f14e8cb1178df47f649428a56e031926..3917374bec28eef9b465727d57d9d9977ce2be36 100644 --- a/tests/test_default.py +++ b/tests/test_default.py @@ -36,10 +36,6 @@ def test_main(tmpworkdir: str): # pylint: disable=unused-argument with patch.object(rwm.RWM, "aws_cmd", mock_proc): assert rwm_main(["aws", "dummy"]) == 0 - with patch.object(rwm.RWM, "rclone_cmd", mock_proc): - assert rwm_main(["rclone", "dummy"]) == 0 - with patch.object(rwm.RWM, "rclone_crypt_cmd", mock_proc): - assert rwm_main(["rclone_crypt", "dummy"]) == 0 with patch.object(rwm.RWM, "restic_cmd", mock_proc): assert rwm_main(["restic", "dummy"]) == 0 diff --git a/tests/test_rwm.py b/tests/test_rwm.py index e1773cab1c4a615bf6c73d0ab66c8bdcdfce1211..7af5d0804d37d2a3e03c29ae03043e62121bd7bf 100644 --- a/tests/test_rwm.py +++ b/tests/test_rwm.py @@ -27,57 +27,6 @@ def test_aws_cmd(tmpworkdir: str, motoserver: str): # pylint: disable=unused-ar assert not trwm.storage_manager.bucket_exist(test_bucket) -def test_rclone_cmd(tmpworkdir: str, motoserver: str): # pylint: disable=unused-argument - """test rclone command""" - - trwm = rwm.RWM({ - "rwm_s3_endpoint_url": motoserver, - "rwm_s3_access_key": "dummy", - "rwm_s3_secret_key": "dummy", - }) - - test_bucket = "testbucket" - test_file = "testfile.txt" - Path(test_file).write_text('1234', encoding='utf-8') - - trwm.rclone_cmd(["mkdir", f"rwmbe:/{test_bucket}/"]) - assert trwm.storage_manager.bucket_exist(test_bucket) - - trwm.rclone_cmd(["copy", test_file, f"rwmbe:/{test_bucket}/"]) - assert len(trwm.storage_manager.list_objects(test_bucket)) == 1 - - -def test_rclone_crypt_cmd(tmpworkdir: str, motoserver: str): # pylint: disable=unused-argument - """test rclone with crypt overlay""" - - trwm = rwm.RWM({ - "rwm_s3_endpoint_url": motoserver, - "rwm_s3_access_key": "dummy", - "rwm_s3_secret_key": "dummy", - "rwm_rclone_crypt_bucket": "cryptdata_test", - "rwm_rclone_crypt_password": rwm.rclone_obscure_password("dummydummydummydummy"), - }) - - test_bucket = "testbucket" - test_file = "testfile.txt" - Path(test_file).write_text('1234', encoding='utf-8') - - trwm.rclone_crypt_cmd(["copy", test_file, f"rwmbe:/{test_bucket}/"]) - assert len(trwm.storage_manager.list_objects(trwm.config["rwm_rclone_crypt_bucket"])) == 1 - - trwm.rclone_crypt_cmd(["delete", f"rwmbe:/{test_bucket}/{test_file}"]) - assert len(trwm.storage_manager.list_objects(trwm.config["rwm_rclone_crypt_bucket"])) == 0 - - test_file1 = "testfile1.txt" - Path(test_file1).write_text('4321', encoding='utf-8') - trwm.rclone_crypt_cmd(["sync", ".", f"rwmbe:/{test_bucket}/"]) - assert len(trwm.storage_manager.list_objects(trwm.config["rwm_rclone_crypt_bucket"])) == 2 - - Path(test_file1).unlink() - trwm.rclone_crypt_cmd(["sync", ".", f"rwmbe:/{test_bucket}/"]) - assert len(trwm.storage_manager.list_objects(trwm.config["rwm_rclone_crypt_bucket"])) == 1 - - def test_restic_cmd(tmpworkdir: str, motoserver: str): # pylint: disable=unused-argument """test restic command"""