general: update documentation and examples

README.md

 # RWM - Restic WORM Manager
 
+## The story
+
 Restic is a fast and secure backup program. Uses client-storage architecture to backup
 local filesystem to the variety of backends including local and remote storages such
 as S3. Given it's server-less nature, a backed-up resource has ability to manipulate it's
@@ -26,18 +28,21 @@ credentials for the managed bucket.
 
 RWM can:
 
-* check if used bucket is configured for versioning
+* provide low-level S3 access for aws cli, rclone
+* rclone crypt over S3 backend
+* restic with S3 repository
+* configurable backup manager/executor
+
+
+todo:
 
+* check if used bucket is configured for versioning
 * check if used access_key does not have administrator privileges to manipulate
   with WORM policies
-
 * generate and store current bucket state state-data
-
 * recreate bucket contents on local filesystem (or remote bucket) acording to specified
   state data
-
 * ??? check completeness of the current state of the bucket
-
 * prune all non-recent object versions to reclaim storage space
 
 
@@ -48,30 +53,29 @@ TBD:
 
 ## Usage
 
-Beside the WORM management features, `rwm` can also be used as simple wrapper to other S3 related
-tools such as `aws` CLI, `rclone` and `restic`.
-
-It can be also used as simple backup manager for intended usecases.
-
-
 ### Install
 ```
-git clone git@gitlab.flab.cesnet.cz:bodik/rwm.git /opt/rwm
+git clone git@gitlab.cesnet.cz:radoslav_bodo/rwm.git /opt/rwm
 cd /opt/rwm
 make install
 ```
 
-### Simple copy: rclone with crypt overlay
+### Low-level S3
 
 ```
-cat > rwm.conf <<__EOF__
-rwm_s3_endpoint_url: ""
-rwm_s3_access_key: ""
-rwm_s3_secret_key: ""
-rwm_rclone_crypt_bucket: "rwmcrypt"
-rwm_rclone_crypt_password: ""
-__EOF__
+cp examples/rwm-rclone.conf rwm.conf
+rwm aws s3 ls s3://
+rwm aws s3api list-buckets
+rwm rclone lsd rwmbe:/
+```
 
+
+### Simple copy: rclone with crypt overlay
+
+rclone_crypt defines single default remote named "rwmbe:/" pointed to `rwm_rclone_crypt_bucket` path.
+
+```
+cp examples/rwm-rclone.conf rwm.conf
 rwm rclone_crypt sync /data rwmbe:/
 rwm rclone_crypt lsl rwmbe:/
 ```
@@ -79,47 +83,25 @@ rwm rclone_crypt lsl rwmbe:/
 ### Restic: manual restic backup
 
 ```
-cat > rwm.conf <<__EOF__
-rwm_s3_endpoint_url: ""
-rwm_s3_access_key: ""
-rwm_s3_secret_key: ""
-rwm_restic_bucket: "rwmrestic"
-rwm_restic_password: ""
-__EOF__
-
+cp examples/rwm-restic.conf rwm.conf
 rwm restic init
 rwm restic backup /data
 rwm restic snapshots
+rwm restic mount /mnt/restore
 ```
 
+note: executed tools stdout is buffered, mount does not have immediate output as normal `restic mount` would
+
+
 ### RWM: simple backups
 
-```
-cat > rwm.conf <<__EOF__
-rwm_s3_endpoint_url: ""
-rwm_s3_access_key: ""
-rwm_s3_secret_key: ""
-
-rwm_restic_bucket: "rwmrestic"
-rwm_restic_password: ""
-
-rwm_backups:
-  backup1:
-    filesdirs:
-      - /etc
-      - /data
-    excludes:
-      - *.cache
-    extras:
-      - --tag
-      - mytag1
-
-rwm_retention:
-  keep-daily: "14"
-  keep-weekly: "20"
-__EOF__
+backups follows standard restic procedures, but adds profile like configuration to easily run in schedulers
 
+```
+cp examples/rwm-backups.conf rwm.conf
 rwm backup_all
+rwm restic snapshots
+rwm restic mount /mnt/restore
 ```
 
 

examples/rwm-backups.conf

+# rwm aws, rclone, restic, backup, backup_all
+
+rwm_s3_endpoint_url: ""
+rwm_s3_access_key: ""
+rwm_s3_secret_key: ""
+
+rwm_restic_bucket: "rwmbackups"
+rwm_restic_password: ""
+
+rwm_backups:
+  linux1:
+    filesdirs:
+      - "/"
+    excludes:
+      #- "*.cache"
+      #- "*.log"
+      - "/dev/*"
+      - "/home/*/.cache/*"
+      - "/media/*"
+      - "/mnt/*"
+      - "/proc/*"
+      - "/sys/*"
+      - "/tmp/*"
+      - "/var/cache/*"
+      - "/var/lib/mysql"
+      - "/var/lib/postgresql"
+      - "/var/log/journal*"
+      - "/var/run/*"
+      - "/var/tmp/*"
+
+  simplefs:
+    filesdirs:
+      - "/"
+    exludes:
+      - "/mnt/*"
+      - "/tmp/*"
+      - "/var/cache/*"
+      - "/var/lib/mysql/*"
+      - "/var/lib/postgresql/*"
+      - "/var/run/*"
+    extras:
+      - --one-file-system
+
+  mysql:
+    filesdirs:
+      - /var/lib/rwm/database.tar.gz
+    extras: ["--tag", "mysql"]
+    prerun:
+      - dump_database
+    postrun:
+      - cleanup_dump_database
+
+  nas:
+    filesdirs:
+        - /mnt/nas
+    prerun:
+      - mount
+    postrun:
+      - unmount
+
+rwm_retention:
+  keep-daily: "14"
+  keep-weekly: "20"

rwm.conf.example → examples/rwm-rclone.conf

-# all commands
+# rwm aws, rclone, rclone_crypt
+
 rwm_s3_endpoint_url: ""
 rwm_s3_access_key: ""
 rwm_s3_secret_key: ""
 
-# rclone_crypt
 rwm_rclone_crypt_bucket: "rwmcrypt"
 rwm_rclone_crypt_password: ""
-
-# restic, backup
-rwm_restic_bucket: "rwmcrypt"
-rwm_restic_password: ""
-
-# backup
-rwm_backups: {}
-rwm_retention: {}

examples/rwm-restic.conf

+# rwm aws, rclone, restic
+
+rwm_s3_endpoint_url: ""
+rwm_s3_access_key: ""
+rwm_s3_secret_key: ""
+
+rwm_restic_bucket: "rwmrestic"
+rwm_restic_password: ""

rwm.py

@@ -329,10 +329,11 @@ def main(argv=None):
 
     if args.command == "backup":
         ret = rwmi.backup_cmd(args.name).returncode
+        logger.info("rwm backup finished with %s (ret %d)", "success" if ret == 0 else "errors", ret)
     if args.command == "backup_all":
         ret = rwmi.backup_all_cmd()
+        logger.info("rwm backup_all finished with %s (ret %d)", "success" if ret == 0 else "errors", ret)
 
-    logger.info("rwm finished with %s (ret %d)", "success" if ret == 0 else "errors", ret)
     return ret