rwm: add storage_drop_versions

Makefile

 all: coverage lint
 
 install:
-	apt-get -y install awscli python3-cryptography python3-tabulate rclone restic yamllint
+	apt-get -y install awscli python3-boto3 python3-cryptography python3-tabulate rclone restic yamllint
 
 install-dev:
 	apt-get -y install python3-venv snapd

README.md

@@ -26,23 +26,25 @@ versions of the bucket objects reclaiming free space on the underlying storage.
 must be delegated to secure element residing outside of attacker's reach and would use privileged
 credentials for the managed bucket.
 
-RWM can:
+
+## Features
 
 * low-level S3 access for aws cli and rclone
 * rclone "crypt over S3" backend
 * restic with S3 repository
-* configurable backup manager/executor
-
-* create, delete and list policed storage buckets
-* check if used bucket is configured with expected policies
+* simple backup manager/executor
+* storage manager
+  * create, delete and list policed storage buckets
+  * check if used bucket is configured with expected policies
+  * drop all versions to reclaim storage space
 
 TODO:
 * generate and store current bucket state state-data
 * recreate bucket contents on local filesystem (or remote bucket) acording to specified
   state data
 * ??? check completeness of the current state of the bucket
-* prune all non-recent object versions to reclaim storage space
-* unlike in other backup solutions, attacker with credentials can restore any old data from the repository/bucket
+* unlike in other backup solutions, attacker with credentials can restore
+  old data from the repository/bucket, this should be discussed (howto threat modeling ?)
 
 
 ## Usage
@@ -87,6 +89,9 @@ rwm storage_check_policy bucket1
 rwm backup_all
 rwm restic snapshots
 rwm restic mount /mnt/restore
+
+# if current storage state is consistent, one can drop old object versions from time to time to reclaim storage space
+rwm --confg admin.conf storage_drop_versions bucket1
 ```
 
 

rwm.py

@@ -198,6 +198,7 @@ class StorageManager:
         bucket.objects.all().delete()
         bucket.object_versions.all().delete()
         bucket.delete()
+        return 0
 
     @staticmethod
     def _policy_statements_admin(policy):
@@ -255,6 +256,24 @@ class StorageManager:
 
         return output
 
+    def storage_drop_versions(self, bucket_name):
+        """deletes all old versions and delete markers from storage to reclaim space"""
+
+        # ? lock repo
+
+        # drop all active object versions
+        object_versions = self.s3.meta.client.list_object_versions(Bucket=bucket_name)
+        for item in object_versions["Versions"]:
+            if not item["IsLatest"]:
+                self.s3.ObjectVersion(bucket_name, item["Key"], item["VersionId"]).delete()
+
+        # drop all delete markers
+        object_versions = self.s3.meta.client.list_object_versions(Bucket=bucket_name)
+        for item in object_versions["DeleteMarkers"]:
+            self.s3.ObjectVersion(bucket_name, item["Key"], item["VersionId"]).delete()
+
+        return 0
+
 
 class RWM:
     """rwm impl"""
@@ -443,6 +462,11 @@ class RWM:
         ))
         return 0
 
+    def storage_drop_versions_cmd(self, bucket_name):
+        """storage_drop_versions command"""
+
+        return self.storage_manager.storage_drop_versions(bucket_name)
+
 
 def configure_logging(debug):
     """configure logger"""
@@ -466,34 +490,45 @@ def parse_arguments(argv):
     parser.add_argument("--config", default="rwm.conf")
 
     subparsers = parser.add_subparsers(title="commands", dest="command", required=False)
+
     subparsers.add_parser("version", help="show version")
 
-    aws_cmd_parser = subparsers.add_parser("aws", help="aws command")
+    aws_cmd_parser = subparsers.add_parser("aws", help="run aws cli")
     aws_cmd_parser.add_argument("cmd_args", nargs="*")
-    rc_cmd_parser = subparsers.add_parser("rclone", help="rclone command")
-    rc_cmd_parser.add_argument("cmd_args", nargs="*")
-    rcc_cmd_parser = subparsers.add_parser("rclone_crypt", help="rclone command with crypt overlay")
-    rcc_cmd_parser.add_argument("cmd_args", nargs="*")
-    res_cmd_parser = subparsers.add_parser("restic", help="restic command")
-    res_cmd_parser.add_argument("cmd_args", nargs="*")
-
-    backup_cmd_parser = subparsers.add_parser("backup", help="backup command")
-    backup_cmd_parser.add_argument("name", help="backup config name")
-    _ = subparsers.add_parser("backup_all", help="backup all command")
-
-    storage_create_cmd_parser = subparsers.add_parser("storage_create", help="storage_create command")
+    rclone_cmd_parser = subparsers.add_parser("rclone", help="run rclone")
+    rclone_cmd_parser.add_argument("cmd_args", nargs="*")
+    rclone_crypt_cmd_parser = subparsers.add_parser("rclone_crypt", help="run rclone with crypt overlay")
+    rclone_crypt_cmd_parser.add_argument("cmd_args", nargs="*")
+    restic_cmd_parser = subparsers.add_parser("restic", help="run restic")
+    restic_cmd_parser.add_argument("cmd_args", nargs="*")
+
+    backup_cmd_parser = subparsers.add_parser("backup", help="perform backup")
+    backup_cmd_parser.add_argument("name", help="backup name")
+
+    _ = subparsers.add_parser("backup_all", help="run all backups in config")
+
+    storage_create_cmd_parser = subparsers.add_parser("storage_create", help="create policed storage bucked")
     storage_create_cmd_parser.add_argument("bucket_name", help="bucket name")
-    storage_create_cmd_parser.add_argument("target_username", help="actual bucket user with limited RW access")
-    storage_delete_cmd_parser = subparsers.add_parser("storage_delete", help="storage_delete command")
+    storage_create_cmd_parser.add_argument("target_username", help="user to be granted limited RW access")
+
+    storage_delete_cmd_parser = subparsers.add_parser("storage_delete", help="delete storage")
     storage_delete_cmd_parser.add_argument("bucket_name", help="bucket name")
-    storage_check_policy_cmd_parser = subparsers.add_parser("storage_check_policy", help="storage_check_policy command; use --debug to show policy")
+
+    storage_check_policy_cmd_parser = subparsers.add_parser("storage_check_policy", help="check bucket policies; use --debug to show policy")
     storage_check_policy_cmd_parser.add_argument("bucket_name", help="bucket name")
-    _ = subparsers.add_parser("storage_list", help="storage_list command")
+
+    _ = subparsers.add_parser("storage_list", help="list storages")
+
+    storage_drop_versions_cmd_parser = subparsers.add_parser(
+        "storage_drop_versions",
+        help="reclaim storage space; drops any old object versions from bucket"
+    )
+    storage_drop_versions_cmd_parser.add_argument("bucket_name", help="bucket name")
 
     return parser.parse_args(argv)
 
 
-def main(argv=None):
+def main(argv=None):  # pylint: disable=too-many-branches
     """main"""
 
     args = parse_arguments(argv)
@@ -505,12 +540,12 @@ def main(argv=None):
     logger.debug("config, %s", config)
     # assert config ?
     rwmi = RWM(config)
+    ret = -1
 
     if args.command == "version":
         print(__version__)
-        return 0
+        ret = 0
 
-    ret = -1
     if args.command == "aws":
         ret = wrap_output(rwmi.aws_cmd(args.cmd_args))
     if args.command == "rclone":
@@ -535,6 +570,8 @@ def main(argv=None):
         ret = rwmi.storage_check_policy_cmd(args.bucket_name)
     if args.command == "storage_list":
         ret = rwmi.storage_list_cmd()
+    if args.command == "storage_drop_versions":
+        ret = rwmi.storage_drop_versions_cmd(args.bucket_name)
 
     logger.debug("rwm finished with %s (ret %d)", "success" if ret == 0 else "errors", ret)
     return ret

tests/test_default.py

@@ -56,3 +56,5 @@ def test_main(tmpworkdir: str):  # pylint: disable=unused-argument
         assert rwm_main(["storage_check_policy", "bucket"]) == 0
     with patch.object(rwm.RWM, "storage_list_cmd", mock_ok):
         assert rwm_main(["storage_list"]) == 0
+    with patch.object(rwm.RWM, "storage_drop_versions_cmd", mock_ok):
+        assert rwm_main(["storage_drop_versions", "bucket"]) == 0

tests/test_rwm.py

@@ -273,29 +273,31 @@ def test_storage_delete_cmd(tmpworkdir: str, microceph: str, radosuser_admin: rw
     assert trwm.storage_delete_cmd(bucket_name) == 1
 
 
-def test_storage_check_policy_cmd(tmpworkdir: str, microceph: str, radosuser_admin: rwm.StorageManager):  # pylint: disable=unused-argument
+def test_storage_check_policy_cmd(tmpworkdir: str):  # pylint: disable=unused-argument
     """test storage check policy command"""
 
-    trwm = rwm.RWM({
-        "rwm_s3_endpoint_url": radosuser_admin.url,
-        "rwm_s3_access_key": radosuser_admin.access_key,
-        "rwm_s3_secret_key": radosuser_admin.secret_key,
-    })
+    trwm = rwm.RWM({})
 
     mock = Mock(return_value=False)
     with patch.object(rwm.StorageManager, "storage_check_policy", mock):
         assert trwm.storage_check_policy_cmd("dummy") == 1
 
 
-def test_storage_list_cmd(tmpworkdir: str, microceph: str, radosuser_admin: rwm.StorageManager):  # pylint: disable=unused-argument
+def test_storage_list_cmd(tmpworkdir: str):  # pylint: disable=unused-argument
     """test storage check policy command"""
 
-    trwm = rwm.RWM({
-        "rwm_s3_endpoint_url": radosuser_admin.url,
-        "rwm_s3_access_key": radosuser_admin.access_key,
-        "rwm_s3_secret_key": radosuser_admin.secret_key,
-    })
+    trwm = rwm.RWM({})
 
     mock = Mock(return_value=[])
     with patch.object(rwm.StorageManager, "storage_list", mock):
         assert trwm.storage_list_cmd() == 0
+
+
+def test_storage_drop_versions_cmd(tmpworkdir: str):  # pylint: disable=unused-argument
+    """test storage drop versions command"""
+
+    trwm = rwm.RWM({})
+
+    mock = Mock(return_value=0)
+    with patch.object(rwm.StorageManager, "storage_drop_versions", mock):
+        assert trwm.storage_drop_versions_cmd("dummy") == 0

tests/test_storage.py

@@ -159,3 +159,29 @@ def test_storage_list(
     radosuser_admin.bucket_create("no-acl-dummy")
     radosuser_admin.storage_create(bucket_name, target_username)
     assert radosuser_admin.storage_list()
+
+
+def test_storage_drop_versions(tmpworkdir: str, microceph: str, radosuser_admin: rwm.StorageManager):  # pylint: disable=unused-argument
+    """test manager storage_drop_versions"""
+
+    bucket_name = "testbuckx"
+    target_username = "test1"
+    bucket = radosuser_admin.storage_create(bucket_name, target_username)
+
+    bucket.upload_fileobj(BytesIO(b"dummydata1"), "dummykey")
+    bucket.upload_fileobj(BytesIO(b"dummydata2"), "dummykey")
+    bucket.Object("dummykey").delete()
+    bucket.upload_fileobj(BytesIO(b"dummydata3"), "dummykey")
+
+    # boto3 resource api
+    object_versions = list(bucket.object_versions.all())
+    assert len(object_versions) == 4
+    # boto3 client api
+    object_versions = radosuser_admin.s3.meta.client.list_object_versions(Bucket=bucket.name)
+    assert len(object_versions["Versions"]) == 3
+    assert len(object_versions["DeleteMarkers"]) == 1
+
+    assert radosuser_admin.storage_drop_versions(bucket.name) == 0
+
+    object_versions = list(bucket.object_versions.all())
+    assert len(object_versions) == 1