rwm: storage_drop_versions use pagination and tests

README.md

@@ -45,6 +45,11 @@ TODO:
 * ??? check completeness of the current state of the bucket
 * unlike in other backup solutions, attacker with credentials can restore
   old data from the repository/bucket, this should be discussed (howto threat modeling ?)
+* rgw leaks objects on tests
+
+* rwm drop _cmd from methods which are not other commands wrapper
+* fix microceph start on node reboot
+* drop rclone use-cases
 
 
 ## Usage

rwm.py

@@ -260,17 +260,24 @@ class StorageManager:
         """deletes all old versions and delete markers from storage to reclaim space"""
 
         # ? lock repo
+        paginator = self.s3.meta.client.get_paginator('list_object_versions')
 
         # drop all active object versions
-        object_versions = self.s3.meta.client.list_object_versions(Bucket=bucket_name)
-        for item in object_versions["Versions"]:
-            if not item["IsLatest"]:
-                self.s3.ObjectVersion(bucket_name, item["Key"], item["VersionId"]).delete()
+        objects = []
+        for page in paginator.paginate(Bucket=bucket_name):
+            for item in page.get("Versions", []):
+                if not item["IsLatest"]:
+                    objects.append([bucket_name, item["Key"], item["VersionId"]])
+        for item in objects:
+            self.s3.ObjectVersion(*item).delete()
 
         # drop all delete markers
-        object_versions = self.s3.meta.client.list_object_versions(Bucket=bucket_name)
-        for item in object_versions["DeleteMarkers"]:
-            self.s3.ObjectVersion(bucket_name, item["Key"], item["VersionId"]).delete()
+        objects = []
+        for page in paginator.paginate(Bucket=bucket_name):
+            for item in page.get("DeleteMarkers", []):
+                objects.append([bucket_name, item["Key"], item["VersionId"]])
+        for item in objects:
+            self.s3.ObjectVersion(*item).delete()
 
         return 0
 

tests/conftest.py

@@ -59,7 +59,7 @@ def radosuser(microceph_url, username, tenant="tenant1"):
     """rgwuser fixture"""
 
     subprocess.run(
-        ["/snap/bin/radosgw-admin", "user", "rm", f"--uid={tenant}${username}", "--purge-data"],
+        ["/snap/bin/radosgw-admin", "user", "rm", f"--uid={tenant}${username}", "--purge-data", "--purge-keys"],
         stdout=subprocess.DEVNULL,
         stderr=subprocess.DEVNULL,
         check=False
@@ -74,7 +74,7 @@ def radosuser(microceph_url, username, tenant="tenant1"):
     user = json.loads(proc.stdout)
     yield StorageManager(microceph_url, user["keys"][0]["access_key"], user["keys"][0]["secret_key"])
 
-    subprocess.run(["/snap/bin/radosgw-admin", "user", "rm", f"--uid={tenant}${username}", "--purge-data"], check=True)
+    subprocess.run(["/snap/bin/radosgw-admin", "user", "rm", f"--uid={tenant}${username}", "--purge-data", "--purge-keys"], check=True)
 
 
 @pytest.fixture

tests/test_storage.py

@@ -185,3 +185,21 @@ def test_storage_drop_versions(tmpworkdir: str, microceph: str, radosuser_admin:
 
     object_versions = list(bucket.object_versions.all())
     assert len(object_versions) == 1
+
+
+def test_storage_drop_versions_many(tmpworkdir: str, microceph: str, radosuser_admin: rwm.StorageManager):  # pylint: disable=unused-argument
+    """test manager storage_drop_versions"""
+
+    bucket_name = "testbuckx"
+    target_username = "test1"
+    bucket = radosuser_admin.storage_create(bucket_name, target_username)
+
+    bucket.upload_fileobj(BytesIO(b"dummydata0"), "dummykey")
+    for idx in range(801):
+        bucket.Object("dummykey").delete()
+        bucket.upload_fileobj(BytesIO(f"dummydata{idx}".encode()), "dummykey")
+
+    assert radosuser_admin.storage_drop_versions(bucket.name) == 0
+
+    object_versions = list(bucket.object_versions.all())
+    assert len(object_versions) == 1