diff --git a/README.md b/README.md
index 3d93d8144514800dbbe3e2113448794ede7a7793..875740a2a52a723efceb970eba4efac72d9d6ef4 100644
--- a/README.md
+++ b/README.md
@@ -83,9 +83,9 @@ Two distinct S3 accounts required (*admin*, *user1*)
```
cp examples/rwm-admin.conf admin.conf
-rwm --confg admin.conf create_storage bucket1 user1
-rwm --confg admin.conf storage_check_policy bucket1
rwm --confg admin.conf storage_list
+rwm --confg admin.conf storage_create bucket1 user1
+rwm --confg admin.conf storage_info bucket1
cp examples/rwm-backups.conf rwm.conf
rwm restic init
diff --git a/rwm.py b/rwm.py
index 1a8692ee5ab438c280399c10b7c7c5cf00aa4b42..9c455a4fd0cb63c958e8e8d00d530990ec300574 100755
--- a/rwm.py
+++ b/rwm.py
@@ -6,7 +6,6 @@ import gzip
import json
import logging
import os
-import re
import shlex
import subprocess
import sys
@@ -152,6 +151,11 @@ class StorageManager:
logger.error("rwm bucket_policy error, %s", (exc))
return None
+ def bucket_acl(self, name):
+ """api stub"""
+ acl = self.s3.Bucket(name).Acl()
+ return {"owner": acl.owner, "grants": acl.grants}
+
def list_buckets(self):
"""aws s3 resource api stub"""
return list(self.s3.buckets.all())
@@ -240,19 +244,15 @@ class StorageManager:
return False
- def storage_list(self, show_full=False, name_filter=""):
+ def storage_list(self):
"""storage list"""
- pattern = re.compile(name_filter)
- buckets = [bucket for bucket in self.list_buckets() if pattern.search(bucket.name)]
- paginator = self.s3.meta.client.get_paginator('list_object_versions')
output = []
-
- for bucket in buckets:
+ for bucket in self.list_buckets():
result = {}
result["name"] = bucket.name
result["policy"] = "OK" if self.storage_check_policy(bucket.name) else "FAILED"
- result["owner"] = self.bucket_owner(bucket.name).split("$")[-1]
+ result["short_owner"] = self.bucket_owner(bucket.name).split("$")[-1]
if result["policy"] == "OK":
user_statement = self._policy_statements_user(self.bucket_policy(bucket.name))[0]
@@ -260,29 +260,42 @@ class StorageManager:
else:
result["target_user"] = None
- if show_full:
- result["objects"] = 0
- result["delete_markers"] = 0
- result["old_versions"] = 0
- result["size"] = 0
- result["old_size"] = 0
-
- for page in paginator.paginate(Bucket=bucket.name):
- for obj in page.get("Versions", []):
- if obj["IsLatest"]:
- result["objects"] += 1
- result["size"] += obj["Size"]
- else:
- result["old_versions"] += 1
- result["old_size"] += obj["Size"]
- result["delete_markers"] += len(page.get("DeleteMarkers", []))
- result["size"] = size_fmt(result["size"])
- result["old_size"] = size_fmt(result["old_size"])
-
output.append(result)
return output
+ def storage_info(self, bucket_name):
+ """grabs storage bucket detailed info"""
+
+ result = {}
+ result["name"] = bucket_name
+ result["check_policy"] = "OK" if self.storage_check_policy(bucket_name) else "FAILED"
+ result["owner"] = self.bucket_owner(bucket_name)
+ result["acl"] = self.bucket_acl(bucket_name)
+ result["policy"] = self.bucket_policy(bucket_name)
+ result["objects"] = 0
+ result["delete_markers"] = 0
+ result["old_versions"] = 0
+ result["size"] = 0
+ result["old_size"] = 0
+ result["saved_states"] = []
+
+ paginator = self.s3.meta.client.get_paginator('list_object_versions')
+ for page in paginator.paginate(Bucket=bucket_name):
+ for obj in page.get("Versions", []):
+ if obj["IsLatest"]:
+ result["objects"] += 1
+ result["size"] += obj["Size"]
+ else:
+ result["old_versions"] += 1
+ result["old_size"] += obj["Size"]
+ result["delete_markers"] += len(page.get("DeleteMarkers", []))
+
+ for page in paginator.paginate(Bucket=bucket_name, Prefix="rwm"):
+ result["saved_states"] += [x["Key"] for x in page.get("Versions", [])]
+
+ return result
+
def storage_drop_versions(self, bucket_name):
"""deletes all old versions and delete markers from storage to reclaim space"""
@@ -311,10 +324,9 @@ class StorageManager:
def _bucket_state(self, bucket_name):
"""dumps current bucket state into dict"""
- acl = self.s3.Bucket(bucket_name).Acl()
state = {
"bucket_name": bucket_name,
- "bucket_acl": {"owner": acl.owner, "grants": acl.grants},
+ "bucket_acl": self.bucket_acl(bucket_name),
"bucket_policy": self.bucket_policy(bucket_name),
"time_start": datetime.now(),
"time_end": None,
@@ -490,22 +502,46 @@ class RWM:
return self.storage_manager.storage_delete(bucket_name)
- def storage_list(self, show_full=False, name_filter="") -> int:
+ def storage_list(self) -> int:
"""storage_list command"""
- print(tabulate(
- self.storage_manager.storage_list(show_full, name_filter),
- headers="keys",
- numalign="left"
- ))
+ print(tabulate(self.storage_manager.storage_list(), headers="keys", numalign="left"))
return 0
- def storage_drop_versions(self, bucket_name):
+ def storage_info(self, bucket_name) -> int:
+ """storage_list command"""
+
+ sinfo = self.storage_manager.storage_info(bucket_name)
+ total_size = sinfo["size"] + sinfo["old_size"]
+
+ print(f'Storage bucket: {sinfo["name"]}')
+ print("----------------------------------------")
+ print(f'RWM policy check: {sinfo["check_policy"]}')
+ print(f'Owner: {sinfo["owner"]}')
+ print(f'Objects: {sinfo["objects"]}')
+ print(f'Delete markers: {sinfo["delete_markers"]}')
+ print(f'Old versions: {sinfo["old_versions"]}')
+ print(f'Size: {size_fmt(sinfo["size"])}')
+ print(f'Old size: {size_fmt(sinfo["old_size"])}')
+ print(f'Total size: {size_fmt(total_size)}')
+ print("----------------------------------------")
+ print("Bucket ACL:")
+ print(json.dumps(sinfo["acl"], indent=2))
+ print("----------------------------------------")
+ print("Bucket policy:")
+ print(json.dumps(sinfo["policy"], indent=2))
+ print("----------------------------------------")
+ print("RWM saved states:")
+ print("\n".join(sorted(sinfo["saved_states"])))
+
+ return 0
+
+ def storage_drop_versions(self, bucket_name) -> int:
"""storage_drop_versions command"""
return self.storage_manager.storage_drop_versions(bucket_name)
- def storage_restore_state(self, source_bucket, target_bucket, state_object_key):
+ def storage_restore_state(self, source_bucket, target_bucket, state_object_key) -> int:
"""storage restore state"""
return self.storage_manager.storage_restore_state(source_bucket, target_bucket, state_object_key)
@@ -553,9 +589,10 @@ def parse_arguments(argv):
storage_delete_cmd_parser = subparsers.add_parser("storage_delete", help="delete storage")
storage_delete_cmd_parser.add_argument("bucket_name", help="bucket name")
- storage_list_cmd_parser = subparsers.add_parser("storage_list", help="list storages")
- storage_list_cmd_parser.add_argument("--full", action="store_true", help="show object counts")
- storage_list_cmd_parser.add_argument("--filter", default="", help="name filter regex")
+ _ = subparsers.add_parser("storage_list", help="list storages")
+
+ storage_info_cmd_parser = subparsers.add_parser("storage_info", help="show detailed storage info")
+ storage_info_cmd_parser.add_argument("bucket_name", help="bucket name")
storage_drop_versions_cmd_parser = subparsers.add_parser(
"storage_drop_versions",
@@ -606,7 +643,9 @@ def main(argv=None): # pylint: disable=too-many-branches
if args.command == "storage_delete":
ret = rwmi.storage_delete(args.bucket_name)
if args.command == "storage_list":
- ret = rwmi.storage_list(args.full, args.filter)
+ ret = rwmi.storage_list()
+ if args.command == "storage_info":
+ ret = rwmi.storage_info(args.bucket_name)
if args.command == "storage_drop_versions":
ret = rwmi.storage_drop_versions(args.bucket_name)
if args.command == "storage_restore_state":
diff --git a/tests/test_default.py b/tests/test_default.py
index 549de110f691a65dc32e5d2999d50fea5f63bcb3..b174ee8d8df119c944bfe8f45d281b3f40e2c204 100644
--- a/tests/test_default.py
+++ b/tests/test_default.py
@@ -57,6 +57,8 @@ def test_main(tmpworkdir: str): # pylint: disable=unused-argument
assert rwm_main(["storage_delete", "bucket"]) == 0
with patch.object(rwm.RWM, "storage_list", mock_ok):
assert rwm_main(["storage_list"]) == 0
+ with patch.object(rwm.RWM, "storage_info", mock_ok):
+ assert rwm_main(["storage_info", "dummy"]) == 0
with patch.object(rwm.RWM, "storage_drop_versions", mock_ok):
assert rwm_main(["storage_drop_versions", "bucket"]) == 0
with patch.object(rwm.RWM, "storage_restore_state", mock_ok):
diff --git a/tests/test_rwm.py b/tests/test_rwm.py
index 1f5628cbb47d21c1dbbbc568b56b6ce3b6475034..515ed9bb2a4d84a503bea20980037b26829526b3 100644
--- a/tests/test_rwm.py
+++ b/tests/test_rwm.py
@@ -243,6 +243,19 @@ def test_storage_list(tmpworkdir: str): # pylint: disable=unused-argument
assert trwm.storage_list() == 0
+def test_storage_info(tmpworkdir: str, microceph: str, radosuser_admin: rwm.StorageManager): # pylint: disable=unused-argument
+ """test storage_list"""
+
+ trwm = rwm.RWM({
+ "rwm_s3_endpoint_url": radosuser_admin.url,
+ "rwm_s3_access_key": radosuser_admin.access_key,
+ "rwm_s3_secret_key": radosuser_admin.secret_key,
+ })
+
+ trwm.storage_create("dummy", "dummy")
+ assert trwm.storage_info("dummy") == 0
+
+
def test_storage_drop_versions(tmpworkdir: str): # pylint: disable=unused-argument
"""test storage drop versions"""
@@ -287,7 +300,7 @@ def test_storage_restore_state_restic(tmpworkdir: str, radosuser_admin: rwm.Stor
assert len(snapshots) == 2
assert len(snapshot_files) == 1
assert "/testdatadir/testdata2.txt" == snapshot_files[0]
- states = sorted([x.key for x in trwm.storage_manager.s3.Bucket(trwm.config["rwm_restic_bucket"]).objects.filter(Prefix="rwm")])
+ states = sorted([x.key for x in trwm.storage_manager.s3.Bucket(trwm.config["rwm_restic_bucket"]).object_versions.filter(Prefix="rwm")])
assert len(states) == 2
# create restore bucket
diff --git a/tests/test_storage.py b/tests/test_storage.py
index 99d18b94da41bee04a5e5fd782bbcdd088ab348b..12bbd19435f42ab488fb9cb2715e7e8ea81d28d6 100644
--- a/tests/test_storage.py
+++ b/tests/test_storage.py
@@ -159,11 +159,26 @@ def test_storage_list(
target_username = "test1"
radosuser_admin.bucket_create("no-acl-dummy")
+ radosuser_admin.storage_create(bucket_name, target_username)
+ assert radosuser_admin.storage_list()
+
+
+def test_storage_info(
+ tmpworkdir: str,
+ microceph: str,
+ radosuser_admin: rwm.StorageManager,
+): # pylint: disable=unused-argument
+ """test managet list storage"""
+
+ bucket_name = "rwmbackup-test1"
+ target_username = "test1"
+
bucket = radosuser_admin.storage_create(bucket_name, target_username)
+ assert radosuser_admin.storage_list()
bucket.upload_fileobj(BytesIO(b"dummydata1"), "dummykey")
bucket.upload_fileobj(BytesIO(b"dummydata1"), "dummykey1")
bucket.Object("dummykey1").delete()
- assert len(radosuser_admin.storage_list(show_full=True, name_filter="a")) == 2
+ assert radosuser_admin.storage_info(bucket_name)["delete_markers"] == 1
def test_storage_drop_versions(tmpworkdir: str, microceph: str, radosuser_admin: rwm.StorageManager): # pylint: disable=unused-argument