rwm: add storage restore state

rwm.py

@@ -346,6 +346,19 @@ class StorageManager:
 
         return 0
 
+    def storage_restore_state(self, source_bucket_name, target_bucket_name, state_object_key):
+        """create new bucket, copy data by selected state_file"""
+
+        target_bucket = self.storage_create(target_bucket_name, "dummy")
+        resp = self.s3.Bucket(source_bucket_name).Object(state_object_key).get()
+        state = json.loads(gzip.decompress(resp['Body'].read()))
+
+        for obj in state["versions"]:
+            if obj["IsLatest"]:
+                target_bucket.Object(obj["Key"]).copy({"Bucket": source_bucket_name, "Key": obj["Key"], "VersionId": obj["VersionId"]})
+
+        return 0
+
 
 class RWM:
     """rwm impl"""
@@ -508,6 +521,11 @@ class RWM:
 
         return self.storage_manager.storage_drop_versions(bucket_name)
 
+    def storage_restore_state(self, source_bucket, target_bucket, state_object_key):
+        """storage restore state"""
+
+        return self.storage_manager.storage_restore_state(source_bucket, target_bucket, state_object_key)
+
 
 def configure_logging(debug):
     """configure logger"""
@@ -564,6 +582,11 @@ def parse_arguments(argv):
     )
     storage_drop_versions_cmd_parser.add_argument("bucket_name", help="bucket name")
 
+    storage_restore_state_cmd_parser = subparsers.add_parser("storage_restore_state", help="restore bucketX stateX1 to bucketY")
+    storage_restore_state_cmd_parser.add_argument("source_bucket", help="source_bucket")
+    storage_restore_state_cmd_parser.add_argument("target_bucket", help="target_bucket; should not exist")
+    storage_restore_state_cmd_parser.add_argument("state", help="state object key in source bucket")
+
     return parser.parse_args(argv)
 
 
@@ -607,6 +630,8 @@ def main(argv=None):  # pylint: disable=too-many-branches
         ret = rwmi.storage_list(args.full, args.filter)
     if args.command == "storage_drop_versions":
         ret = rwmi.storage_drop_versions(args.bucket_name)
+    if args.command == "storage_restore_state":
+        ret = rwmi.storage_restore_state(args.source_bucket, args.target_bucket, args.state)
 
     logger.debug("rwm finished with %s (ret %d)", "success" if ret == 0 else "errors", ret)
     return ret

tests/test_default.py

@@ -61,3 +61,5 @@ def test_main(tmpworkdir: str):  # pylint: disable=unused-argument
         assert rwm_main(["storage_list"]) == 0
     with patch.object(rwm.RWM, "storage_drop_versions", mock_ok):
         assert rwm_main(["storage_drop_versions", "bucket"]) == 0
+    with patch.object(rwm.RWM, "storage_restore_state", mock_ok):
+        assert rwm_main(["storage_restore_state", "bucket", "bucket", "state"]) == 0

tests/test_rwm.py

@@ -261,3 +261,57 @@ def test_storage_drop_versions(tmpworkdir: str):  # pylint: disable=unused-argum
     mock = Mock(return_value=0)
     with patch.object(rwm.StorageManager, "storage_drop_versions", mock):
         assert trwm.storage_drop_versions("dummy") == 0
+
+
+def test_storage_restore_state_restic(tmpworkdir: str, radosuser_admin: rwm.StorageManager):  # pylint: disable=unused-argument
+    """test restore bucket from previous saved state"""
+
+    trwm = rwm.RWM({
+        "rwm_s3_endpoint_url": radosuser_admin.url,
+        "rwm_s3_access_key": radosuser_admin.access_key,
+        "rwm_s3_secret_key": radosuser_admin.secret_key,
+        "rwm_restic_bucket": "restictest",
+        "rwm_restic_password": "dummydummydummydummy",
+        "rwm_backups": {
+            "testcfg": {
+                "filesdirs": ["testdatadir/"],
+            }
+        }
+    })
+
+    # create and initialize storage
+    assert trwm.storage_create(trwm.config["rwm_restic_bucket"], "dummy") == 0
+    assert trwm.restic_cmd(["init"]).returncode == 0
+
+    # do backups
+    Path("testdatadir").mkdir()
+    Path("testdatadir/testdata1.txt").write_text("dummydata1", encoding="utf-8")
+    assert trwm.backup("testcfg") == 0
+    Path("testdatadir/testdata1.txt").unlink()
+    Path("testdatadir/testdata2.txt").write_text("dummydata2", encoding="utf-8")
+    assert trwm.backup("testcfg") == 0
+
+    # check two snapshots exists with expected content
+    snapshots = _restic_list_snapshots(trwm)
+    snapshot_files = _restic_list_snapshot_files(trwm, snapshots[1]["id"])
+    assert len(snapshots) == 2
+    assert len(snapshot_files) == 1
+    assert "/testdatadir/testdata2.txt" == snapshot_files[0]
+    states = sorted([x.key for x in trwm.storage_manager.s3.Bucket(trwm.config["rwm_restic_bucket"]).objects.filter(Prefix="rwm")])
+    assert len(states) == 2
+
+    # create restore bucket
+    restore_bucket_name = f'{trwm.config["rwm_restic_bucket"]}-restore'
+    trwm.storage_restore_state(trwm.config["rwm_restic_bucket"], restore_bucket_name, states[0])
+
+    # check restore bucket contents
+    trwm_restore = rwm.RWM({
+        **trwm.config,
+        "rwm_restic_bucket": restore_bucket_name
+    })
+    snapshots = _restic_list_snapshots(trwm_restore)
+    snapshot_files = _restic_list_snapshot_files(trwm_restore, snapshots[0]["id"])
+    assert len(snapshots) == 1
+    assert len(snapshot_files) == 1
+    assert "/testdatadir/testdata1.txt" == snapshot_files[0]
+    assert trwm_restore.restic_cmd(["check"]).returncode == 0