diff --git a/rwm.py b/rwm.py index b759b5b0fda9c5c3cba704477c4a9b3ec98224eb..94f302de56d244a8fc368006a1aedb8a699941b7 100755 --- a/rwm.py +++ b/rwm.py @@ -376,15 +376,29 @@ class StorageManager: result["old_size"] += obj["Size"] result["delete_markers"] += len(page.get("DeleteMarkers", [])) - for page in paginator.paginate(Bucket=bucket_name, Prefix="rwm"): - result["saved_states"] += [x["Key"] for x in page.get("Versions", [])] + paginator = self.s3.meta.client.get_paginator('list_objects') + for page in paginator.paginate(Bucket=bucket_name, Prefix="rwm/"): + result["saved_states"] += [x["Key"] for x in page.get("Contents", [])] return result def storage_drop_versions(self, bucket_name): - """deletes all old versions and delete markers from storage to reclaim space""" + """ + Delete all old versions and delete markers from storage to reclaim space. + Also delete all rwm saved states and generate one state after pruning. + """ # ? lock repo + + # drop all saved rwm states + paginator = self.s3.meta.client.get_paginator('list_objects') + objects = [] + for page in paginator.paginate(Bucket=bucket_name, Prefix="rwm/"): + for item in page.get("Contents", []): + objects.append((bucket_name, item["Key"])) + for item in objects: + self.s3.Object(*item).delete() + paginator = self.s3.meta.client.get_paginator('list_object_versions') # drop all active object versions @@ -392,7 +406,7 @@ class StorageManager: for page in paginator.paginate(Bucket=bucket_name): for item in page.get("Versions", []): if not item["IsLatest"]: - objects.append([bucket_name, item["Key"], item["VersionId"]]) + objects.append((bucket_name, item["Key"], item["VersionId"])) for item in objects: self.s3.ObjectVersion(*item).delete() @@ -400,11 +414,14 @@ class StorageManager: objects = [] for page in paginator.paginate(Bucket=bucket_name): for item in page.get("DeleteMarkers", []): - objects.append([bucket_name, item["Key"], item["VersionId"]]) + objects.append((bucket_name, item["Key"], item["VersionId"])) for item in objects: self.s3.ObjectVersion(*item).delete() - return 0 + # save current state + ret = self.storage_save_state(bucket_name) + + return ret def _bucket_state(self, bucket_name): """dumps current bucket state into dict""" diff --git a/tests/test_rwm.py b/tests/test_rwm.py index a351cced706e932f62c5b309daeac6ef1c97f333..e383ee87fe47d3813bf895f4a16deaa83b0b48f8 100644 --- a/tests/test_rwm.py +++ b/tests/test_rwm.py @@ -376,7 +376,7 @@ def test_storage_restore_state_restic(tmpworkdir: str, radosuser_admin: rwm.Stor assert len(snapshots) == 2 assert len(snapshot_files) == 1 assert "/testdatadir/testdata2.txt" == snapshot_files[0] - states = sorted([x.key for x in trwm.storage_manager.s3.Bucket(trwm.config.restic_bucket).object_versions.filter(Prefix="rwm")]) + states = sorted([x.key for x in trwm.storage_manager.s3.Bucket(trwm.config.restic_bucket).object_versions.filter(Prefix="rwm/")]) assert len(states) == 2 # create restore bucket diff --git a/tests/test_storage.py b/tests/test_storage.py index 90185b81b7494484175c4efaaf5c2a9e9e5e24ff..85394dbea8b008d0121f4d81efa6fd0c21f63448 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -189,19 +189,21 @@ def test_storage_drop_versions(tmpworkdir: str, radosuser_admin: rwm.StorageMana bucket.upload_fileobj(BytesIO(b"dummydata2"), "dummykey") bucket.Object("dummykey").delete() bucket.upload_fileobj(BytesIO(b"dummydata3"), "dummykey") + radosuser_admin.storage_save_state(bucket.name) # boto3 resource api object_versions = list(bucket.object_versions.all()) - assert len(object_versions) == 4 + assert len(object_versions) == 5 # boto3 client api object_versions = radosuser_admin.s3.meta.client.list_object_versions(Bucket=bucket.name) - assert len(object_versions["Versions"]) == 3 + assert len(object_versions["Versions"]) == 4 assert len(object_versions["DeleteMarkers"]) == 1 assert radosuser_admin.storage_drop_versions(bucket.name) == 0 object_versions = list(bucket.object_versions.all()) - assert len(object_versions) == 1 + # should be one object and one saved state + assert len(object_versions) == 2 @pytest.mark.skipif('PYTEST_SLOW' not in os.environ, reason='slow on devnode, runs in CI') @@ -220,7 +222,8 @@ def test_storage_drop_versions_many(tmpworkdir: str, radosuser_admin: rwm.Storag assert radosuser_admin.storage_drop_versions(bucket.name) == 0 object_versions = list(bucket.object_versions.all()) - assert len(object_versions) == 1 + # should be one object and one saved state + assert len(object_versions) == 2 def test_storage_save_state_error_handling(tmpworkdir: str): # pylint: disable=unused-argument