Backup and recover

README.md

@@ -34,3 +34,32 @@ Used parameters in ansible recipes:
 * *mail_local*: disable e-mail (only local delivery)
 * *site\_name*: site identifier
 * *vault\_mount\_point:*: path to secrets in the Vault
+
+## Backup and Restore
+
+Backup:
+
+    # initialize repository
+    # restic -v init <repository>
+
+    read -r PASSWORD AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY
+    ansible-playbook  \
+      --extra-vars "backup_password=$PASSWORD" \
+      --extra-vars "s3_access_key=$AWS_ACCESS_KEY_ID" \
+      --extra-vars "s3_secret_key=$AWS_SECRET_ACCESS_KEY" \
+      playbooks/backup.yaml
+
+Restore (on admin machine, config only):
+
+    read -r PASSWORD AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY
+    ansible-playbook \
+      --extra-vars "backup_password=$PASSWORD" \
+      --extra-vars "s3_access_key=$AWS_ACCESS_KEY_ID" \
+      --extra-vars "s3_secret_key=$AWS_SECRET_ACCESS_KEY" \
+      playbooks/recover.yaml
+
+Restore (on k8s master):
+
+    kubectl label node k8s-nfs nfs-server=true
+    kubectl create namespace recover
+    kubectl apply -f /tmp/recover.yaml

common/playbooks/backup.yaml

+#
+# K8s launcher file to backup EGI notebooks
+#
+# See https://github.com/EGI-Federation/egi-notebooks-backup.
+#
+# Required variables:
+#
+# * s3_access_key
+# * s3_secret_key
+# * backup_password
+# * backup_repostory
+#
+# Launch backup immediately:
+#
+# kubectl create job -n backup backup-manual --from cronjob/notebooks-backup
+# kubectl delete job -n backup backup-manual
+#
+# Launch snapshots rotation immediately:
+#
+# kubectl create job -n backup backup-culler-manual --from cronjob/notebooks-backup-culler
+# kubectl delete job -n backup backup-culler-manual
+#
+---
+- name: Setup periodic backup
+  hosts: master[0]
+  become: true
+  tasks:
+    - name: Backup configuration
+      template:
+        dest: /tmp/backup.yaml
+        src: templates/backup.yaml
+        mode: 0600
+    - name: Backup
+      vars:
+        config: >-
+          --version=0.0.1-0.dev.git.17.h6dc29d8
+          -f /tmp/backup.yaml
+        url: oci://registry.egi.eu/vo.notebooks.egi.eu/notebooks-backup
+      shell: |-
+        helm status --namespace backup backup
+        if [ $? -ne 0 ]; then
+            kubectl create ns backup 2>/dev/null || true
+            helm install --namespace backup {{ config }} backup {{ url }}
+        else
+            helm upgrade --namespace backup {{ config }} backup {{ url }}
+        fi
+      environment:
+        KUBECONFIG: /etc/kubernetes/admin.conf
+        PATH: /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin
+      when: true

common/playbooks/recover.yaml

+#
+# Helper for EGI notebooks recover from backup
+#
+# See https://github.com/EGI-Federation/egi-notebooks-backup.
+#
+# 1. apply playbook (only for k8s template)
+#
+#    read -r PASSWORD AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY
+#    ansible-playbook \
+#      --extra-vars "backup_password=$PASSWORD" \
+#      --extra-vars "s3_access_key=$AWS_ACCESS_KEY_ID" \
+#      --extra-vars "s3_secret_key=$AWS_SECRET_ACCESS_KEY" \
+#      playbooks/recover.yaml
+#
+# 2. launch on the k8s master
+#
+#   kubectl label node k8s-nfs nfs-server=true
+#   kubectl create namespace recover
+#   kubectl apply -f /tmp/recover.yaml
+#
+---
+- name: Recover batch job
+  hosts: master[0]
+  become: true
+  tasks:
+    - name: Configuration for recover
+      template:
+        src: templates/recover.yaml
+        dest: /tmp/recover.yaml
+        mode: 0600

common/playbooks/templates/backup.yaml

+---
+backup:
+  password: {{ backup_password }}
+  repository: {{ backup_repository }}
+  args:
+    - "--exclude=/exports/archived-*"
+    - "--exclude=/exports/nexus-*"
+    - "--exclude=/exports/prometheus-*"
+  env:
+    AWS_ACCESS_KEY_ID: {{ s3_access_key}}
+    AWS_SECRET_ACCESS_KEY: {{ s3_secret_key }}

common/playbooks/templates/recover.yaml

+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: restic-env
+  namespace: recover
+type: Opaque
+data:
+  AWS_ACCESS_KEY_ID: {{ s3_access_key | b64encode }}
+  AWS_SECRET_ACCESS_KEY: {{ s3_secret_key | b64encode }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: restic-password
+  namespace: recover
+type: Opaque
+data:
+  password: "{{ backup_password | b64encode }}"
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: notebooks-backup-recover
+  namespace: recover
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: notebooks-backup-recover
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims", "persistentvolumes"]
+    verbs: ["get", "list", "watch", "create"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: notebooks-backup-recover
+subjects:
+  - kind: ServiceAccount
+    name: notebooks-backup-recover
+    namespace: recover
+roleRef:
+  kind: ClusterRole
+  name: notebooks-backup-recover
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: notebooks-backup-recover
+  namespace: recover
+spec:
+  template:
+    spec:
+      serviceAccountName: notebooks-backup-recover
+      restartPolicy: Never
+      containers:
+        - name: recover
+          args:
+            - /usr/local/bin/recover.sh
+            # - "--overwrite"
+          image: registry.egi.eu/vo.notebooks.egi.eu/svc-backup:0.0.1-0.dev.git.17.h6dc29d8
+          imagePullPolicy: Always
+          volumeMounts:
+            - mountPath: /exports
+              name: dest
+            - mountPath: /restic-secret/
+              name: restic-password
+          env:
+            - name: NFS_PATH
+              value: /exports
+            - name: NAMESPACE
+              value: hub
+            - name: TARGET_NAMESPACE
+              value: hub
+            - name: RESTIC_REPOSITORY
+              value: "{{ backup_repository }}"
+            - name: RESTIC_PASSWORD_FILE
+              value: /restic-secret/password
+          envFrom:
+            - secretRef:
+                name: restic-env
+      nodeSelector:
+        nfs-server: "true"
+      volumes:
+        - hostPath:
+            path: /exports
+            type: DirectoryOrCreate
+          name: dest
+        - secret:
+            secretName: restic-ssh
+            defaultMode: 256
+          name: restic-ssh
+        - secret:
+            secretName: restic-password
+            defaultMode: 256
+          name: restic-password