From 754cb2501c51a0fde27abb5a5fc34ecb3766c21b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Radoslav=20Bod=C3=B3?= <bodik@cesnet.cz>
Date: Tue, 26 Mar 2024 17:34:41 +0100
Subject: [PATCH] general: update documentation and examples
---
README.md | 86 ++++++++++++++++-----------------------
examples/rwm-backups.conf | 63 ++++++++++++++++++++++++++++
examples/rwm-rclone.conf | 8 ++++
examples/rwm-restic.conf | 8 ++++
rwm.conf.example | 16 --------
rwm.py | 3 +-
6 files changed, 115 insertions(+), 69 deletions(-)
create mode 100644 examples/rwm-backups.conf
create mode 100644 examples/rwm-rclone.conf
create mode 100644 examples/rwm-restic.conf
delete mode 100644 rwm.conf.example
diff --git a/README.md b/README.md
index 7142e3f..6101db8 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,7 @@
# RWM - Restic WORM Manager
+## The story
+
Restic is a fast and secure backup program. Uses client-storage architecture to backup
local filesystem to the variety of backends including local and remote storages such
as S3. Given it's server-less nature, a backed-up resource has ability to manipulate it's
@@ -26,18 +28,21 @@ credentials for the managed bucket.
RWM can:
-* check if used bucket is configured for versioning
+* provide low-level S3 access for aws cli, rclone
+* rclone crypt over S3 backend
+* restic with S3 repository
+* configurable backup manager/executor
+
+
+todo:
+* check if used bucket is configured for versioning
* check if used access_key does not have administrator privileges to manipulate
with WORM policies
-
* generate and store current bucket state state-data
-
* recreate bucket contents on local filesystem (or remote bucket) acording to specified
state data
-
* ??? check completeness of the current state of the bucket
-
* prune all non-recent object versions to reclaim storage space
@@ -48,30 +53,29 @@ TBD:
## Usage
-Beside the WORM management features, `rwm` can also be used as simple wrapper to other S3 related
-tools such as `aws` CLI, `rclone` and `restic`.
-
-It can be also used as simple backup manager for intended usecases.
-
-
### Install
```
-git clone git@gitlab.flab.cesnet.cz:bodik/rwm.git /opt/rwm
+git clone git@gitlab.cesnet.cz:radoslav_bodo/rwm.git /opt/rwm
cd /opt/rwm
make install
```
-### Simple copy: rclone with crypt overlay
+### Low-level S3
```
-cat > rwm.conf <<__EOF__
-rwm_s3_endpoint_url: ""
-rwm_s3_access_key: ""
-rwm_s3_secret_key: ""
-rwm_rclone_crypt_bucket: "rwmcrypt"
-rwm_rclone_crypt_password: ""
-__EOF__
+cp examples/rwm-rclone.conf rwm.conf
+rwm aws s3 ls s3://
+rwm aws s3api list-buckets
+rwm rclone lsd rwmbe:/
+```
+
+### Simple copy: rclone with crypt overlay
+
+rclone_crypt defines single default remote named "rwmbe:/" pointed to `rwm_rclone_crypt_bucket` path.
+
+```
+cp examples/rwm-rclone.conf rwm.conf
rwm rclone_crypt sync /data rwmbe:/
rwm rclone_crypt lsl rwmbe:/
```
@@ -79,47 +83,25 @@ rwm rclone_crypt lsl rwmbe:/
### Restic: manual restic backup
```
-cat > rwm.conf <<__EOF__
-rwm_s3_endpoint_url: ""
-rwm_s3_access_key: ""
-rwm_s3_secret_key: ""
-rwm_restic_bucket: "rwmrestic"
-rwm_restic_password: ""
-__EOF__
-
+cp examples/rwm-restic.conf rwm.conf
rwm restic init
rwm restic backup /data
rwm restic snapshots
+rwm restic mount /mnt/restore
```
+note: executed tools stdout is buffered, mount does not have immediate output as normal `restic mount` would
+
+
### RWM: simple backups
-```
-cat > rwm.conf <<__EOF__
-rwm_s3_endpoint_url: ""
-rwm_s3_access_key: ""
-rwm_s3_secret_key: ""
-
-rwm_restic_bucket: "rwmrestic"
-rwm_restic_password: ""
-
-rwm_backups:
- backup1:
- filesdirs:
- - /etc
- - /data
- excludes:
- - *.cache
- extras:
- - --tag
- - mytag1
-
-rwm_retention:
- keep-daily: "14"
- keep-weekly: "20"
-__EOF__
+backups follows standard restic procedures, but adds profile like configuration to easily run in schedulers
+```
+cp examples/rwm-backups.conf rwm.conf
rwm backup_all
+rwm restic snapshots
+rwm restic mount /mnt/restore
```
diff --git a/examples/rwm-backups.conf b/examples/rwm-backups.conf
new file mode 100644
index 0000000..342f380
--- /dev/null
+++ b/examples/rwm-backups.conf
@@ -0,0 +1,63 @@
+# rwm aws, rclone, restic, backup, backup_all
+
+rwm_s3_endpoint_url: ""
+rwm_s3_access_key: ""
+rwm_s3_secret_key: ""
+
+rwm_restic_bucket: "rwmbackups"
+rwm_restic_password: ""
+
+rwm_backups:
+ linux1:
+ filesdirs:
+ - "/"
+ excludes:
+ #- "*.cache"
+ #- "*.log"
+ - "/dev/*"
+ - "/home/*/.cache/*"
+ - "/media/*"
+ - "/mnt/*"
+ - "/proc/*"
+ - "/sys/*"
+ - "/tmp/*"
+ - "/var/cache/*"
+ - "/var/lib/mysql"
+ - "/var/lib/postgresql"
+ - "/var/log/journal*"
+ - "/var/run/*"
+ - "/var/tmp/*"
+
+ simplefs:
+ filesdirs:
+ - "/"
+ exludes:
+ - "/mnt/*"
+ - "/tmp/*"
+ - "/var/cache/*"
+ - "/var/lib/mysql/*"
+ - "/var/lib/postgresql/*"
+ - "/var/run/*"
+ extras:
+ - --one-file-system
+
+ mysql:
+ filesdirs:
+ - /var/lib/rwm/database.tar.gz
+ extras: ["--tag", "mysql"]
+ prerun:
+ - dump_database
+ postrun:
+ - cleanup_dump_database
+
+ nas:
+ filesdirs:
+ - /mnt/nas
+ prerun:
+ - mount
+ postrun:
+ - unmount
+
+rwm_retention:
+ keep-daily: "14"
+ keep-weekly: "20"
diff --git a/examples/rwm-rclone.conf b/examples/rwm-rclone.conf
new file mode 100644
index 0000000..f287c8b
--- /dev/null
+++ b/examples/rwm-rclone.conf
@@ -0,0 +1,8 @@
+# rwm aws, rclone, rclone_crypt
+
+rwm_s3_endpoint_url: ""
+rwm_s3_access_key: ""
+rwm_s3_secret_key: ""
+
+rwm_rclone_crypt_bucket: "rwmcrypt"
+rwm_rclone_crypt_password: ""
diff --git a/examples/rwm-restic.conf b/examples/rwm-restic.conf
new file mode 100644
index 0000000..70a4d5a
--- /dev/null
+++ b/examples/rwm-restic.conf
@@ -0,0 +1,8 @@
+# rwm aws, rclone, restic
+
+rwm_s3_endpoint_url: ""
+rwm_s3_access_key: ""
+rwm_s3_secret_key: ""
+
+rwm_restic_bucket: "rwmrestic"
+rwm_restic_password: ""
diff --git a/rwm.conf.example b/rwm.conf.example
deleted file mode 100644
index 0931e9f..0000000
--- a/rwm.conf.example
+++ /dev/null
@@ -1,16 +0,0 @@
-# all commands
-rwm_s3_endpoint_url: ""
-rwm_s3_access_key: ""
-rwm_s3_secret_key: ""
-
-# rclone_crypt
-rwm_rclone_crypt_bucket: "rwmcrypt"
-rwm_rclone_crypt_password: ""
-
-# restic, backup
-rwm_restic_bucket: "rwmcrypt"
-rwm_restic_password: ""
-
-# backup
-rwm_backups: {}
-rwm_retention: {}
diff --git a/rwm.py b/rwm.py
index 239826b..a45f015 100755
--- a/rwm.py
+++ b/rwm.py
@@ -329,10 +329,11 @@ def main(argv=None):
if args.command == "backup":
ret = rwmi.backup_cmd(args.name).returncode
+ logger.info("rwm backup finished with %s (ret %d)", "success" if ret == 0 else "errors", ret)
if args.command == "backup_all":
ret = rwmi.backup_all_cmd()
+ logger.info("rwm backup_all finished with %s (ret %d)", "success" if ret == 0 else "errors", ret)
- logger.info("rwm finished with %s (ret %d)", "success" if ret == 0 else "errors", ret)
return ret
--
GitLab