rwm: storage_drop_versions also prune rwm saved states

rwm.py

@@ -376,15 +376,29 @@ class StorageManager:
                     result["old_size"] += obj["Size"]
             result["delete_markers"] += len(page.get("DeleteMarkers", []))
 
-        for page in paginator.paginate(Bucket=bucket_name, Prefix="rwm"):
-            result["saved_states"] += [x["Key"] for x in page.get("Versions", [])]
+        paginator = self.s3.meta.client.get_paginator('list_objects')
+        for page in paginator.paginate(Bucket=bucket_name, Prefix="rwm/"):
+            result["saved_states"] += [x["Key"] for x in page.get("Contents", [])]
 
         return result
 
     def storage_drop_versions(self, bucket_name):
-        """deletes all old versions and delete markers from storage to reclaim space"""
+        """
+        Delete all old versions and delete markers from storage to reclaim space.
+        Also delete all rwm saved states and generate one state after pruning.
+        """
 
         # ? lock repo
+
+        # drop all saved rwm states
+        paginator = self.s3.meta.client.get_paginator('list_objects')
+        objects = []
+        for page in paginator.paginate(Bucket=bucket_name, Prefix="rwm/"):
+            for item in page.get("Contents", []):
+                objects.append((bucket_name, item["Key"]))
+        for item in objects:
+            self.s3.Object(*item).delete()
+
         paginator = self.s3.meta.client.get_paginator('list_object_versions')
 
         # drop all active object versions
@@ -392,7 +406,7 @@ class StorageManager:
         for page in paginator.paginate(Bucket=bucket_name):
             for item in page.get("Versions", []):
                 if not item["IsLatest"]:
-                    objects.append([bucket_name, item["Key"], item["VersionId"]])
+                    objects.append((bucket_name, item["Key"], item["VersionId"]))
         for item in objects:
             self.s3.ObjectVersion(*item).delete()
 
@@ -400,11 +414,14 @@ class StorageManager:
         objects = []
         for page in paginator.paginate(Bucket=bucket_name):
             for item in page.get("DeleteMarkers", []):
-                objects.append([bucket_name, item["Key"], item["VersionId"]])
+                objects.append((bucket_name, item["Key"], item["VersionId"]))
         for item in objects:
             self.s3.ObjectVersion(*item).delete()
 
-        return 0
+        # save current state
+        ret = self.storage_save_state(bucket_name)
+
+        return ret
 
     def _bucket_state(self, bucket_name):
         """dumps current bucket state into dict"""

tests/test_rwm.py

@@ -376,7 +376,7 @@ def test_storage_restore_state_restic(tmpworkdir: str, radosuser_admin: rwm.Stor
     assert len(snapshots) == 2
     assert len(snapshot_files) == 1
     assert "/testdatadir/testdata2.txt" == snapshot_files[0]
-    states = sorted([x.key for x in trwm.storage_manager.s3.Bucket(trwm.config.restic_bucket).object_versions.filter(Prefix="rwm")])
+    states = sorted([x.key for x in trwm.storage_manager.s3.Bucket(trwm.config.restic_bucket).object_versions.filter(Prefix="rwm/")])
     assert len(states) == 2
 
     # create restore bucket

tests/test_storage.py

@@ -189,19 +189,21 @@ def test_storage_drop_versions(tmpworkdir: str, radosuser_admin: rwm.StorageMana
     bucket.upload_fileobj(BytesIO(b"dummydata2"), "dummykey")
     bucket.Object("dummykey").delete()
     bucket.upload_fileobj(BytesIO(b"dummydata3"), "dummykey")
+    radosuser_admin.storage_save_state(bucket.name)
 
     # boto3 resource api
     object_versions = list(bucket.object_versions.all())
-    assert len(object_versions) == 4
+    assert len(object_versions) == 5
     # boto3 client api
     object_versions = radosuser_admin.s3.meta.client.list_object_versions(Bucket=bucket.name)
-    assert len(object_versions["Versions"]) == 3
+    assert len(object_versions["Versions"]) == 4
     assert len(object_versions["DeleteMarkers"]) == 1
 
     assert radosuser_admin.storage_drop_versions(bucket.name) == 0
 
     object_versions = list(bucket.object_versions.all())
-    assert len(object_versions) == 1
+    # should be one object and one saved state
+    assert len(object_versions) == 2
 
 
 @pytest.mark.skipif('PYTEST_SLOW' not in os.environ, reason='slow on devnode, runs in CI')
@@ -220,7 +222,8 @@ def test_storage_drop_versions_many(tmpworkdir: str, radosuser_admin: rwm.Storag
     assert radosuser_admin.storage_drop_versions(bucket.name) == 0
 
     object_versions = list(bucket.object_versions.all())
-    assert len(object_versions) == 1
+    # should be one object and one saved state
+    assert len(object_versions) == 2
 
 
 def test_storage_save_state_error_handling(tmpworkdir: str):  # pylint: disable=unused-argument