CHANGELOG.md

 # RWM changelog
 
+## 1.2 - Holodeck preserver
+
+* changed: cron, use .conf, propagate return code, prune logs
+* changed: rwm, require to select state version for storage-restore-state
+* changed: microceph, use bigger disk (improves cluster stability)
+* added: postgresql backup scripts (plain and docker version)
+
+
 ## 1.1 - Brahms tuning
 
 * added: storage-state command

Makefile

@@ -33,7 +33,7 @@ microceph-service:
 	snap install microceph
 	snap refresh --hold microceph
 	/snap/bin/microceph cluster bootstrap
-	/snap/bin/microceph disk add loop,1G,3
+	/snap/bin/microceph disk add loop,4G,3
 	/snap/bin/microceph enable rgw
 	while true; do /snap/bin/ceph status | grep "HEALTH_OK" && break; done
 	# required for gitlab runner shell executor which runs as non-privileged user

README.md

@@ -114,9 +114,9 @@ rwm restic mount /mnt/restore
 rwm --confg admin.conf storage-drop-versions bucket1
 
 # if storage gets corrupted, state can be restored to other bucket
-## select existing state file from storage-info
+## select existing state and version from storage-info
 rwm --confg admin.conf storage-info bucket1
-rwm --confg admin.conf storage-restore-state bucket1 bucket1-restore rwm/state_[timestamp].json.gz
+rwm --confg admin.conf storage-restore-state bucket1 bucket1-restore rwm/state_[timestamp].json.gz versionid
 ```
 
 
@@ -141,6 +141,15 @@ rwm restic snapshots
 rwm restic mount /mnt/restore
 ```
 
+### Restore PostgreSQL backup
+
+```
+# Using script for the whole archive
+scripts/restore_postgresql.sh postgresql.tar.gz
+
+# For individual dumps
+createdb database_name && gunzip -c database_name.sql.gz | psql -d database_name
+```
 
 ## Notes
 

examples/rwm-backups.conf

@@ -31,6 +31,16 @@ backups:
     postrun:
       - "/opt/rwm/scripts/backup_mysql.py cleanup"
 
+  postgresql:
+    filesdirs:
+      - /var/lib/rwm/postgresql.tar.gz
+    tags:
+      - "postgresql"
+    prerun:
+      - "/opt/rwm/scripts/backup_postgresql.py create"
+    postrun:
+      - "/opt/rwm/scripts/backup_postgresql.py cleanup"
+
 retention:
   keep-daily: "60"
   keep-within: "60d"

rwm.py

@@ -23,7 +23,7 @@ from pydantic import BaseModel, ConfigDict
 from tabulate import tabulate
 
 
-__version__ = "1.1"
+__version__ = "1.2"
 logger = logging.getLogger("rwm")
 logger.setLevel(logging.INFO)
 
@@ -427,9 +427,8 @@ class StorageManager:
                     result["old_size"] += obj["Size"]
             result["delete_markers"] += len(page.get("DeleteMarkers", []))
 
-        paginator = self.s3.meta.client.get_paginator('list_objects')
         for page in paginator.paginate(Bucket=bucket_name, Prefix="rwm/"):
-            result["saved_states"] += [x["Key"] for x in page.get("Contents", [])]
+            result["saved_states"] += [(x["Key"], x["VersionId"]) for x in page.get("Versions", [])]
 
         return result
 
@@ -512,11 +511,11 @@ class StorageManager:
 
         return 0
 
-    def storage_restore_state(self, source_bucket_name, target_bucket_name, state_object_key):
+    def storage_restore_state(self, source_bucket_name, target_bucket_name, state_object_key, state_version):
         """create new bucket, copy data by selected state_file"""
 
         target_bucket = self.storage_create(target_bucket_name, "dummy")
-        resp = self.s3.Bucket(source_bucket_name).Object(state_object_key).get()
+        resp = self.s3.Bucket(source_bucket_name).Object(state_object_key).get(VersionId=state_version)
         state = json.loads(gzip.decompress(resp['Body'].read()))
 
         for obj in state["versions"]:
@@ -764,7 +763,7 @@ class RWM:
         print(json.dumps(sinfo["policy"], indent=2))
         print("----------------------------------------")
         print("RWM saved states:")
-        print("\n".join(sorted(sinfo["saved_states"])))
+        print("\n".join([f"{key} {ver}" for key, ver in sorted(sinfo["saved_states"])]))
 
         return 0
 
@@ -783,10 +782,10 @@ class RWM:
 
         return self.storage_manager.storage_drop_versions(bucket_name)
 
-    def storage_restore_state(self, source_bucket, target_bucket, state_object_key) -> int:
+    def storage_restore_state(self, source_bucket, target_bucket, state_object_key, state_version) -> int:
         """storage restore state"""
 
-        return self.storage_manager.storage_restore_state(source_bucket, target_bucket, state_object_key)
+        return self.storage_manager.storage_restore_state(source_bucket, target_bucket, state_object_key, state_version)
 
 
 def configure_logging(debug):
@@ -849,6 +848,7 @@ def parse_arguments(argv):
     storage_restore_state_cmd_parser.add_argument("source_bucket", help="source_bucket")
     storage_restore_state_cmd_parser.add_argument("target_bucket", help="target_bucket; should not exist")
     storage_restore_state_cmd_parser.add_argument("state", help="state object key in source bucket")
+    storage_restore_state_cmd_parser.add_argument("version", help="state object version in source bucket")
 
     return parser.parse_args(argv)
 
@@ -916,7 +916,7 @@ def main(argv=None):  # pylint: disable=too-many-branches
         ret = rwmi.storage_drop_versions(args.bucket_name)
 
     if args.command == "storage-restore-state":
-        ret = rwmi.storage_restore_state(args.source_bucket, args.target_bucket, args.state)
+        ret = rwmi.storage_restore_state(args.source_bucket, args.target_bucket, args.state, args.version)
 
     logger.debug("finished with %s (ret %d)", "success" if ret == 0 else "errors", ret)
     return ret

scripts/backup_mysql.py

@@ -94,6 +94,7 @@ def cleanup():
     return os.unlink(ARCHIVE)
 
 
+# pylint: disable=duplicate-code
 def main():
     """main"""
 

scripts/backup_postgres_docker.sh

+#!/bin/bash
+# example dockerized backup of dockerized postgresql
+
+set -e
+umask 077
+
+# note: Alternatively, this can be placed in the rwm backup.prerun configuration field.
+docker exec -u postgres pgdocker pg_dumpall --clean > /var/backups/pgdocker.sql
+
+docker run \
+    --rm \
+    --pull always \
+    --volume "/etc/rwm.conf:/opt/rwm/rwm.conf:ro" \
+    --volume "/var/backups:/var/backups" \
+    --volume "/var/run:/var/run" \
+    --hostname "pgdocker-rwm-container" \
+    "gitlab-registry.cesnet.cz/radoslav_bodo/rwm:release-1.1" \
+    backup pgdocker
+
+# note: dtto
+rm -f /var/backups/pgdocker.sql
\ No newline at end of file

scripts/backup_postgresql.py

+#!/usr/bin/env python3
+"""rwm postgresql backup helper"""
+
+import os
+import shutil
+import subprocess
+import sys
+from argparse import ArgumentParser
+
+
+BASE = "/var/lib/rwm"
+BACKUPDIR = f"{BASE}/postgresql"
+ARCHIVE = f"{BASE}/postgresql.tar.gz"
+USERNAME = os.environ.get("PGUSER", "postgres")
+
+
+def list_databases():
+    """list postgresql databases"""
+
+    cmd = [
+        "su",
+        "-c",
+        'psql -q -A -t -c "SELECT datname FROM pg_database WHERE datistemplate = false;"',
+        USERNAME,
+    ]
+    proc = subprocess.run(cmd, check=True, stdout=subprocess.PIPE, text=True)
+    return proc.stdout.splitlines()
+
+
+def backup_database(database):
+    """backup single database"""
+
+    cmd = ["su", "-c", f"pg_dump --clean --create {database}", USERNAME]
+    try:
+        with open(f"{BACKUPDIR}/{database}.sql", "wb") as fd:
+            subprocess.run(cmd, stdout=fd, check=True)
+    except subprocess.CalledProcessError:
+        print(f"ERROR: cannot dump {database}", file=sys.stderr)
+        return 1
+
+    return 0
+
+
+def backup_global_data():
+    """backup global data"""
+
+    try:
+        cmd = ["su", "-c", "pg_dumpall --clean --globals-only", USERNAME]
+        with open(f"{BACKUPDIR}/_globals.sql", "wb") as fd:
+            subprocess.run(cmd, stdout=fd, check=True)
+    except subprocess.CalledProcessError:
+        print("ERROR: cannot dump database global data", file=sys.stderr)
+        return 1
+
+    return 0
+
+
+def create():
+    """dump database to archive"""
+
+    databases = 0
+    errors = 0
+
+    shutil.rmtree(BACKUPDIR, ignore_errors=True)
+    os.makedirs(BACKUPDIR, exist_ok=True)
+
+    for db in list_databases():
+        databases += 1
+        errors += backup_database(db)
+
+    errors += backup_global_data()
+
+    subprocess.run(["tar", "czf", ARCHIVE, BACKUPDIR], check=True)
+    shutil.rmtree(BACKUPDIR)
+    print("archive created:")
+    subprocess.run(["ls", "-l", ARCHIVE], check=True)
+
+    if databases == 0:
+        print("ERROR: no databases dumped", file=sys.stderr)
+        errors += 1
+
+    print(f"RESULT: errors={errors} databases={databases}")
+    return errors
+
+
+def cleanup():
+    """cleanup backup process"""
+
+    return os.unlink(ARCHIVE)
+
+
+# pylint: disable=duplicate-code
+def main():
+    """main"""
+
+    parser = ArgumentParser()
+    parser.add_argument("command", choices=["create", "cleanup"])
+    args = parser.parse_args()
+    os.umask(0o077)
+
+    if args.command == "create":
+        return create()
+    if args.command == "cleanup":
+        return cleanup()
+    return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/cron.sh

@@ -6,7 +6,7 @@ umask 077
 LOGFILE="/var/log/rwm/cron.log.$(date +'%Y-%m-%dT%H:%M:%S%z')"
 mkdir -p "$(dirname "$LOGFILE")"
 
-/opt/rwm/rwm.py --config /etc/rwm.yml backup-all 1>"$LOGFILE" 2>&1
+/opt/rwm/rwm.py --config /etc/rwm.conf backup-all 1>"$LOGFILE" 2>&1
 RET=$?
 
 if [ $RET = 0 ]; then
@@ -15,4 +15,8 @@ else
 	RESULT="ERROR"
 fi
 # shellcheck disable=SC2002
-cat "$LOGFILE" | mail -E -s "rwm backup-all $RESULT" $USER
+cat "$LOGFILE" | mail -E -s "rwm backup-all $RESULT" "$LOGNAME"
+
+find /var/log/rwm -type f -mtime +90 -exec rm {} \;
+
+exit $RET
\ No newline at end of file

scripts/restore_postgresql.sh

+#!/bin/bash
+#
+# Restores all databases (might throw errros, see pg docs).
+# Might require to regrant privileges or ownership of created objects.
+#
+# ```
+# \c database
+# GRANT ALL PRIVILEGES ON SCHEMA public TO y;
+# GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO y;
+# GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO y;
+# ```
+`
+set -ex
+umask 077
+
+tar_file="$1"
+
+temp_dir=$(mktemp -d)
+tar xzf "$tar_file" -C "$temp_dir"
+chown -R postgres "${temp_dir}"
+
+find "$temp_dir" -type f -name '*.sql' | while read -r dump_file; do
+    su -c "psql < '$dump_file'" postgres
+done
+
+rm -rf "$temp_dir"
\ No newline at end of file

tests/test_default.py

@@ -72,7 +72,7 @@ def test_main():
         assert _rwm_minconfig(["storage-drop-versions", "bucket"]) == 0
 
     with patch.object(rwm.RWM, "storage_restore_state", mock_ok):
-        assert _rwm_minconfig(["storage-restore-state", "bucket", "bucket", "state"]) == 0
+        assert _rwm_minconfig(["storage-restore-state", "bucket", "bucket", "state", "version"]) == 0
 
     # error handling
     assert rwm_main(["--config", "notexist", "version"]) == 1

tests/test_rwm.py

@@ -392,12 +392,15 @@ def test_storage_restore_state_restic(tmpworkdir: str, radosuser_admin: rwm.Stor
     assert len(snapshots) == 2
     assert len(snapshot_files) == 1
     assert "/testdatadir/testdata2.txt" == snapshot_files[0]
-    states = sorted([x.key for x in trwm.storage_manager.s3.Bucket(trwm.config.restic_bucket).object_versions.filter(Prefix="rwm/")])
+    states = sorted(
+        list(trwm.storage_manager.s3.Bucket(trwm.config.restic_bucket).object_versions.filter(Prefix="rwm/")),
+        key=lambda x: (x.key, x.version_id)
+    )
     assert len(states) == 2
 
     # create restore bucket
     restore_bucket_name = f"{trwm.config.restic_bucket}-restore"
-    trwm.storage_restore_state(trwm.config.restic_bucket, restore_bucket_name, states[0])
+    trwm.storage_restore_state(trwm.config.restic_bucket, restore_bucket_name, states[0].key, states[0].version_id)
 
     # check restore bucket contents
     trwm_restore = rwm.RWM({