From d9264e1c33c4aa01415d6c810ceb1fe0d8bc00a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franti=C5=A1ek=20Dvo=C5=99=C3=A1k?= <valtri@civ.zcu.cz> Date: Thu, 22 May 2025 13:56:55 +0000 Subject: [PATCH] Backup and recover --- README.md | 29 ++++++++ common/playbooks/backup.yaml | 50 +++++++++++++ common/playbooks/recover.yaml | 30 ++++++++ common/playbooks/templates/backup.yaml | 11 +++ common/playbooks/templates/recover.yaml | 99 +++++++++++++++++++++++++ 5 files changed, 219 insertions(+) create mode 100644 common/playbooks/backup.yaml create mode 100644 common/playbooks/recover.yaml create mode 100644 common/playbooks/templates/backup.yaml create mode 100644 common/playbooks/templates/recover.yaml diff --git a/README.md b/README.md index fa97a0c..85e990b 100644 --- a/README.md +++ b/README.md @@ -34,3 +34,32 @@ Used parameters in ansible recipes: * *mail_local*: disable e-mail (only local delivery) * *site\_name*: site identifier * *vault\_mount\_point:*: path to secrets in the Vault + +## Backup and Restore + +Backup: + + # initialize repository + # restic -v init <repository> + + read -r PASSWORD AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY + ansible-playbook \ + --extra-vars "backup_password=$PASSWORD" \ + --extra-vars "s3_access_key=$AWS_ACCESS_KEY_ID" \ + --extra-vars "s3_secret_key=$AWS_SECRET_ACCESS_KEY" \ + playbooks/backup.yaml + +Restore (on admin machine, config only): + + read -r PASSWORD AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY + ansible-playbook \ + --extra-vars "backup_password=$PASSWORD" \ + --extra-vars "s3_access_key=$AWS_ACCESS_KEY_ID" \ + --extra-vars "s3_secret_key=$AWS_SECRET_ACCESS_KEY" \ + playbooks/recover.yaml + +Restore (on k8s master): + + kubectl label node k8s-nfs nfs-server=true + kubectl create namespace recover + kubectl apply -f /tmp/recover.yaml diff --git a/common/playbooks/backup.yaml b/common/playbooks/backup.yaml new file mode 100644 index 0000000..c45869e --- /dev/null +++ b/common/playbooks/backup.yaml @@ -0,0 +1,50 @@ +# +# K8s launcher file to backup EGI notebooks +# +# See https://github.com/EGI-Federation/egi-notebooks-backup. +# +# Required variables: +# +# * s3_access_key +# * s3_secret_key +# * backup_password +# * backup_repostory +# +# Launch backup immediately: +# +# kubectl create job -n backup backup-manual --from cronjob/notebooks-backup +# kubectl delete job -n backup backup-manual +# +# Launch snapshots rotation immediately: +# +# kubectl create job -n backup backup-culler-manual --from cronjob/notebooks-backup-culler +# kubectl delete job -n backup backup-culler-manual +# +--- +- name: Setup periodic backup + hosts: master[0] + become: true + tasks: + - name: Backup configuration + template: + dest: /tmp/backup.yaml + src: templates/backup.yaml + mode: 0600 + - name: Backup + vars: + config: >- + --version=0.0.1-0.dev.git.17.h6dc29d8 + -f /tmp/backup.yaml + url: oci://registry.egi.eu/vo.notebooks.egi.eu/notebooks-backup + shell: |- + helm status --namespace backup backup + if [ $? -ne 0 ]; then + kubectl create ns backup 2>/dev/null || true + helm install --namespace backup {{ config }} backup {{ url }} + else + helm upgrade --namespace backup {{ config }} backup {{ url }} + fi + environment: + KUBECONFIG: /etc/kubernetes/admin.conf + PATH: /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin + when: true diff --git a/common/playbooks/recover.yaml b/common/playbooks/recover.yaml new file mode 100644 index 0000000..5c9799c --- /dev/null +++ b/common/playbooks/recover.yaml @@ -0,0 +1,30 @@ +# +# Helper for EGI notebooks recover from backup +# +# See https://github.com/EGI-Federation/egi-notebooks-backup. +# +# 1. apply playbook (only for k8s template) +# +# read -r PASSWORD AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY +# ansible-playbook \ +# --extra-vars "backup_password=$PASSWORD" \ +# --extra-vars "s3_access_key=$AWS_ACCESS_KEY_ID" \ +# --extra-vars "s3_secret_key=$AWS_SECRET_ACCESS_KEY" \ +# playbooks/recover.yaml +# +# 2. launch on the k8s master +# +# kubectl label node k8s-nfs nfs-server=true +# kubectl create namespace recover +# kubectl apply -f /tmp/recover.yaml +# +--- +- name: Recover batch job + hosts: master[0] + become: true + tasks: + - name: Configuration for recover + template: + src: templates/recover.yaml + dest: /tmp/recover.yaml + mode: 0600 diff --git a/common/playbooks/templates/backup.yaml b/common/playbooks/templates/backup.yaml new file mode 100644 index 0000000..6d6e7e8 --- /dev/null +++ b/common/playbooks/templates/backup.yaml @@ -0,0 +1,11 @@ +--- +backup: + password: {{ backup_password }} + repository: {{ backup_repository }} + args: + - "--exclude=/exports/archived-*" + - "--exclude=/exports/nexus-*" + - "--exclude=/exports/prometheus-*" + env: + AWS_ACCESS_KEY_ID: {{ s3_access_key}} + AWS_SECRET_ACCESS_KEY: {{ s3_secret_key }} diff --git a/common/playbooks/templates/recover.yaml b/common/playbooks/templates/recover.yaml new file mode 100644 index 0000000..7df0012 --- /dev/null +++ b/common/playbooks/templates/recover.yaml @@ -0,0 +1,99 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: restic-env + namespace: recover +type: Opaque +data: + AWS_ACCESS_KEY_ID: {{ s3_access_key | b64encode }} + AWS_SECRET_ACCESS_KEY: {{ s3_secret_key | b64encode }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: restic-password + namespace: recover +type: Opaque +data: + password: "{{ backup_password | b64encode }}" +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: notebooks-backup-recover + namespace: recover +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: notebooks-backup-recover +rules: + - apiGroups: [""] + resources: ["persistentvolumeclaims", "persistentvolumes"] + verbs: ["get", "list", "watch", "create"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: notebooks-backup-recover +subjects: + - kind: ServiceAccount + name: notebooks-backup-recover + namespace: recover +roleRef: + kind: ClusterRole + name: notebooks-backup-recover + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: notebooks-backup-recover + namespace: recover +spec: + template: + spec: + serviceAccountName: notebooks-backup-recover + restartPolicy: Never + containers: + - name: recover + args: + - /usr/local/bin/recover.sh + # - "--overwrite" + image: registry.egi.eu/vo.notebooks.egi.eu/svc-backup:0.0.1-0.dev.git.17.h6dc29d8 + imagePullPolicy: Always + volumeMounts: + - mountPath: /exports + name: dest + - mountPath: /restic-secret/ + name: restic-password + env: + - name: NFS_PATH + value: /exports + - name: NAMESPACE + value: hub + - name: TARGET_NAMESPACE + value: hub + - name: RESTIC_REPOSITORY + value: "{{ backup_repository }}" + - name: RESTIC_PASSWORD_FILE + value: /restic-secret/password + envFrom: + - secretRef: + name: restic-env + nodeSelector: + nfs-server: "true" + volumes: + - hostPath: + path: /exports + type: DirectoryOrCreate + name: dest + - secret: + secretName: restic-ssh + defaultMode: 256 + name: restic-ssh + - secret: + secretName: restic-password + defaultMode: 256 + name: restic-password -- GitLab