Skip to content
Snippets Groups Projects
Commit b0c64ddb authored by Radoslav Bodó's avatar Radoslav Bodó
Browse files

rwm: storage_list add full object statistics and list filter

parent 8406a491
No related branches found
No related tags found
No related merge requests found
Pipeline #7561 passed
...@@ -6,6 +6,7 @@ import dataclasses ...@@ -6,6 +6,7 @@ import dataclasses
import json import json
import logging import logging
import os import os
import re
import shlex import shlex
import subprocess import subprocess
import sys import sys
...@@ -235,23 +236,36 @@ class StorageManager: ...@@ -235,23 +236,36 @@ class StorageManager:
return False return False
def storage_list(self): def storage_list(self, show_full=False, name_filter=""):
"""storage list""" """storage list"""
pattern = re.compile(name_filter)
buckets = [bucket for bucket in self.list_buckets() if pattern.search(bucket.name)]
paginator = self.s3.meta.client.get_paginator('list_object_versions')
output = [] output = []
for item in self.list_buckets():
for bucket in buckets:
result = { result = {
"name": item.name, "name": bucket.name,
"policy": "OK" if self.storage_check_policy(item.name) else "FAILED", "policy": "OK" if self.storage_check_policy(bucket.name) else "FAILED",
"owner": self.bucket_owner(item.name).split("$")[-1] "owner": self.bucket_owner(bucket.name).split("$")[-1]
} }
if result["policy"] == "OK": if result["policy"] == "OK":
user_statement = self._policy_statements_user(self.bucket_policy(item.name))[0] user_statement = self._policy_statements_user(self.bucket_policy(bucket.name))[0]
result["target_user"] = user_statement["Principal"]["AWS"][0].split("/")[-1] result["target_user"] = user_statement["Principal"]["AWS"][0].split("/")[-1]
else: else:
result["target_user"] = None result["target_user"] = None
if show_full:
result["objects"] = 0
result["old_versions"] = 0
result["delete_markers"] = 0
for page in paginator.paginate(Bucket=bucket.name):
for obj in page.get("Versions", []):
result["objects" if obj["IsLatest"] else "old_versions"] += 1
result["delete_markers"] += len(page.get("DeleteMarkers", []))
output.append(result) output.append(result)
return output return output
...@@ -459,11 +473,11 @@ class RWM: ...@@ -459,11 +473,11 @@ class RWM:
print(msg) print(msg)
return ret return ret
def storage_list(self) -> int: def storage_list(self, show_full=False, name_filter="") -> int:
"""storage_list command""" """storage_list command"""
print(tabulate( print(tabulate(
self.storage_manager.storage_list(), self.storage_manager.storage_list(show_full, name_filter),
headers="keys", headers="keys",
numalign="left" numalign="left"
)) ))
...@@ -524,7 +538,9 @@ def parse_arguments(argv): ...@@ -524,7 +538,9 @@ def parse_arguments(argv):
storage_check_policy_cmd_parser = subparsers.add_parser("storage_check_policy", help="check bucket policies; use --debug to show policy") storage_check_policy_cmd_parser = subparsers.add_parser("storage_check_policy", help="check bucket policies; use --debug to show policy")
storage_check_policy_cmd_parser.add_argument("bucket_name", help="bucket name") storage_check_policy_cmd_parser.add_argument("bucket_name", help="bucket name")
_ = subparsers.add_parser("storage_list", help="list storages") storage_list_cmd_parser = subparsers.add_parser("storage_list", help="list storages")
storage_list_cmd_parser.add_argument("--full", action="store_true", help="show object counts")
storage_list_cmd_parser.add_argument("--filter", default="", help="name filter regex")
storage_drop_versions_cmd_parser = subparsers.add_parser( storage_drop_versions_cmd_parser = subparsers.add_parser(
"storage_drop_versions", "storage_drop_versions",
...@@ -576,7 +592,7 @@ def main(argv=None): # pylint: disable=too-many-branches ...@@ -576,7 +592,7 @@ def main(argv=None): # pylint: disable=too-many-branches
if args.command == "storage_check_policy": if args.command == "storage_check_policy":
ret = rwmi.storage_check_policy(args.bucket_name) ret = rwmi.storage_check_policy(args.bucket_name)
if args.command == "storage_list": if args.command == "storage_list":
ret = rwmi.storage_list() ret = rwmi.storage_list(args.full, args.filter)
if args.command == "storage_drop_versions": if args.command == "storage_drop_versions":
ret = rwmi.storage_drop_versions(args.bucket_name) ret = rwmi.storage_drop_versions(args.bucket_name)
......
...@@ -158,8 +158,9 @@ def test_storage_list( ...@@ -158,8 +158,9 @@ def test_storage_list(
target_username = "test1" target_username = "test1"
radosuser_admin.bucket_create("no-acl-dummy") radosuser_admin.bucket_create("no-acl-dummy")
radosuser_admin.storage_create(bucket_name, target_username) bucket = radosuser_admin.storage_create(bucket_name, target_username)
assert radosuser_admin.storage_list() bucket.upload_fileobj(BytesIO(b"dummydata1"), "dummykey")
assert len(radosuser_admin.storage_list(show_full=True, name_filter="a")) == 2
def test_storage_drop_versions(tmpworkdir: str, microceph: str, radosuser_admin: rwm.StorageManager): # pylint: disable=unused-argument def test_storage_drop_versions(tmpworkdir: str, microceph: str, radosuser_admin: rwm.StorageManager): # pylint: disable=unused-argument
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment