Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
R
rwm
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package Registry
Container Registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Terms and privacy
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Radoslav Bodó
rwm
Commits
79773811
Commit
79773811
authored
10 months ago
by
Radoslav Bodó
Browse files
Options
Downloads
Patches
Plain Diff
rwm: require to select state version for storage-restore-state
parent
57eaa6eb
No related branches found
No related tags found
No related merge requests found
Pipeline
#7920
passed
10 months ago
Stage: code_quality
Changes
4
Pipelines
2
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
README.md
+2
-2
2 additions, 2 deletions
README.md
rwm.py
+8
-8
8 additions, 8 deletions
rwm.py
tests/test_default.py
+1
-1
1 addition, 1 deletion
tests/test_default.py
tests/test_rwm.py
+5
-2
5 additions, 2 deletions
tests/test_rwm.py
with
16 additions
and
13 deletions
README.md
+
2
−
2
View file @
79773811
...
...
@@ -114,9 +114,9 @@ rwm restic mount /mnt/restore
rwm --confg admin.conf storage-drop-versions bucket1
# if storage gets corrupted, state can be restored to other bucket
## select existing state
file
from storage-info
## select existing state
and version
from storage-info
rwm --confg admin.conf storage-info bucket1
rwm --confg admin.conf storage-restore-state bucket1 bucket1-restore rwm/state_[timestamp].json.gz
rwm --confg admin.conf storage-restore-state bucket1 bucket1-restore rwm/state_[timestamp].json.gz
versionid
```
...
...
This diff is collapsed.
Click to expand it.
rwm.py
+
8
−
8
View file @
79773811
...
...
@@ -427,9 +427,8 @@ class StorageManager:
result
[
"
old_size
"
]
+=
obj
[
"
Size
"
]
result
[
"
delete_markers
"
]
+=
len
(
page
.
get
(
"
DeleteMarkers
"
,
[]))
paginator
=
self
.
s3
.
meta
.
client
.
get_paginator
(
'
list_objects
'
)
for
page
in
paginator
.
paginate
(
Bucket
=
bucket_name
,
Prefix
=
"
rwm/
"
):
result
[
"
saved_states
"
]
+=
[
x
[
"
Key
"
]
for
x
in
page
.
get
(
"
Content
s
"
,
[])]
result
[
"
saved_states
"
]
+=
[
(
x
[
"
Key
"
]
,
x
[
"
VersionId
"
])
for
x
in
page
.
get
(
"
Version
s
"
,
[])]
return
result
...
...
@@ -512,11 +511,11 @@ class StorageManager:
return
0
def
storage_restore_state
(
self
,
source_bucket_name
,
target_bucket_name
,
state_object_key
):
def
storage_restore_state
(
self
,
source_bucket_name
,
target_bucket_name
,
state_object_key
,
state_version
):
"""
create new bucket, copy data by selected state_file
"""
target_bucket
=
self
.
storage_create
(
target_bucket_name
,
"
dummy
"
)
resp
=
self
.
s3
.
Bucket
(
source_bucket_name
).
Object
(
state_object_key
).
get
()
resp
=
self
.
s3
.
Bucket
(
source_bucket_name
).
Object
(
state_object_key
).
get
(
VersionId
=
state_version
)
state
=
json
.
loads
(
gzip
.
decompress
(
resp
[
'
Body
'
].
read
()))
for
obj
in
state
[
"
versions
"
]:
...
...
@@ -764,7 +763,7 @@ class RWM:
print
(
json
.
dumps
(
sinfo
[
"
policy
"
],
indent
=
2
))
print
(
"
----------------------------------------
"
)
print
(
"
RWM saved states:
"
)
print
(
"
\n
"
.
join
(
sorted
(
sinfo
[
"
saved_states
"
])))
print
(
"
\n
"
.
join
(
[
f
"
{
key
}
{
ver
}
"
for
key
,
ver
in
sorted
(
sinfo
[
"
saved_states
"
])
]
))
return
0
...
...
@@ -783,10 +782,10 @@ class RWM:
return
self
.
storage_manager
.
storage_drop_versions
(
bucket_name
)
def
storage_restore_state
(
self
,
source_bucket
,
target_bucket
,
state_object_key
)
->
int
:
def
storage_restore_state
(
self
,
source_bucket
,
target_bucket
,
state_object_key
,
state_version
)
->
int
:
"""
storage restore state
"""
return
self
.
storage_manager
.
storage_restore_state
(
source_bucket
,
target_bucket
,
state_object_key
)
return
self
.
storage_manager
.
storage_restore_state
(
source_bucket
,
target_bucket
,
state_object_key
,
state_version
)
def
configure_logging
(
debug
):
...
...
@@ -849,6 +848,7 @@ def parse_arguments(argv):
storage_restore_state_cmd_parser
.
add_argument
(
"
source_bucket
"
,
help
=
"
source_bucket
"
)
storage_restore_state_cmd_parser
.
add_argument
(
"
target_bucket
"
,
help
=
"
target_bucket; should not exist
"
)
storage_restore_state_cmd_parser
.
add_argument
(
"
state
"
,
help
=
"
state object key in source bucket
"
)
storage_restore_state_cmd_parser
.
add_argument
(
"
version
"
,
help
=
"
state object version in source bucket
"
)
return
parser
.
parse_args
(
argv
)
...
...
@@ -916,7 +916,7 @@ def main(argv=None): # pylint: disable=too-many-branches
ret
=
rwmi
.
storage_drop_versions
(
args
.
bucket_name
)
if
args
.
command
==
"
storage-restore-state
"
:
ret
=
rwmi
.
storage_restore_state
(
args
.
source_bucket
,
args
.
target_bucket
,
args
.
state
)
ret
=
rwmi
.
storage_restore_state
(
args
.
source_bucket
,
args
.
target_bucket
,
args
.
state
,
args
.
version
)
logger
.
debug
(
"
finished with %s (ret %d)
"
,
"
success
"
if
ret
==
0
else
"
errors
"
,
ret
)
return
ret
...
...
This diff is collapsed.
Click to expand it.
tests/test_default.py
+
1
−
1
View file @
79773811
...
...
@@ -72,7 +72,7 @@ def test_main():
assert
_rwm_minconfig
([
"
storage-drop-versions
"
,
"
bucket
"
])
==
0
with
patch
.
object
(
rwm
.
RWM
,
"
storage_restore_state
"
,
mock_ok
):
assert
_rwm_minconfig
([
"
storage-restore-state
"
,
"
bucket
"
,
"
bucket
"
,
"
state
"
])
==
0
assert
_rwm_minconfig
([
"
storage-restore-state
"
,
"
bucket
"
,
"
bucket
"
,
"
state
"
,
"
version
"
])
==
0
# error handling
assert
rwm_main
([
"
--config
"
,
"
notexist
"
,
"
version
"
])
==
1
This diff is collapsed.
Click to expand it.
tests/test_rwm.py
+
5
−
2
View file @
79773811
...
...
@@ -392,12 +392,15 @@ def test_storage_restore_state_restic(tmpworkdir: str, radosuser_admin: rwm.Stor
assert
len
(
snapshots
)
==
2
assert
len
(
snapshot_files
)
==
1
assert
"
/testdatadir/testdata2.txt
"
==
snapshot_files
[
0
]
states
=
sorted
([
x
.
key
for
x
in
trwm
.
storage_manager
.
s3
.
Bucket
(
trwm
.
config
.
restic_bucket
).
object_versions
.
filter
(
Prefix
=
"
rwm/
"
)])
states
=
sorted
(
list
(
trwm
.
storage_manager
.
s3
.
Bucket
(
trwm
.
config
.
restic_bucket
).
object_versions
.
filter
(
Prefix
=
"
rwm/
"
)),
key
=
lambda
x
:
(
x
.
key
,
x
.
version_id
)
)
assert
len
(
states
)
==
2
# create restore bucket
restore_bucket_name
=
f
"
{
trwm
.
config
.
restic_bucket
}
-restore
"
trwm
.
storage_restore_state
(
trwm
.
config
.
restic_bucket
,
restore_bucket_name
,
states
[
0
])
trwm
.
storage_restore_state
(
trwm
.
config
.
restic_bucket
,
restore_bucket_name
,
states
[
0
]
.
key
,
states
[
0
].
version_id
)
# check restore bucket contents
trwm_restore
=
rwm
.
RWM
({
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment