Staging deployment (WIP)

cesnet-central/playbooks/squid.yaml

@@ -5,7 +5,7 @@
   tasks:
     - name: Gather facts on the node
       debug:
-        msg: "IPv4: {{ ansible_default_ipv4.address }}, IPv6: {{ ansible_default_ipv6.address }}"
+        msg: "IPv4: {{ ansible_default_ipv4.address | default('') }}, IPv6: {{ ansible_default_ipv6.address | default('') }}"
 - name: Squid proxy deployment
   hosts: ingress[0]
   become: true

staging/ansible.cfg

+[defaults]
+inventory=inventory
+
+[diff]
+always=true

staging/deploy.sh

+#! /bin/bash -xe
+
+#
+# Deploy EOSC staging instance
+#
+
+cd terraform && terraform init && terraform apply
+cd -
+cp -pv terraform/inventory.yaml inventory/1-psnc.yaml
+
+# dynamic DNS
+ip="$(head -n 1 < terraform/fip.txt)"
+# shellstate=$(shopt -po xtrace)
+# set +o xtrace
+# # https://nsupdate.fedcloud.eu
+# vault_prefix=secrets/users/e1662e20-e34b-468c-b0ce-d899bc878364@egi.eu/eosc-staging
+# FEDCLOUD_DYNAMIC_DNS=$(vault read -field data $vault_prefix/FEDCLOUD_DYNAMIC_DNS | grep ^map | head -n 1 | sed 's/map\[\(.*\)\]/\1/')
+# for auth in $FEDCLOUD_DYNAMIC_DNS; do
+# 	echo "curl -i -X GET -u $(echo "$auth" | cut -d: -f1):XXX https://nsupdate.fedcloud.eu/nic/update?myip=$ip"
+# 	curl -i -X GET -u "$auth" https://nsupdate.fedcloud.eu/nic/update?myip="$ip"
+# done
+# eval "$shellstate"
+echo "Terraform finished. Continue? (CTRL-C to quit)"
+read -r _
+
+# wait for ping and ssh
+for ip in $(cat terraform/hosts.txt); do
+	while ! ping -c 1 "$ip"; do sleep 5; done
+	ssh-keygen -R "$ip"
+	while ! ssh egi@"$ip" -o ConnectTimeout=10 -o PreferredAuthentications=publickey -o StrictHostKeyChecking=no :; do sleep 10; done
+done
+
+# check ssh access
+ansible -m command -a 'uname -a' allnodes
+
+# wait cloud-init
+ansible -m shell -a 'while ! test -f /var/lib/cloud/instance/boot-finished; do sleep 2; done' allnodes
+
+# setup volumes
+ansible -m copy -a 'src=terraform/nfs-volume.sh dest=/root/ mode=preserve' nfs
+ansible -m command -a '/root/nfs-volume.sh' nfs
+ansible -m copy -a 'src=terraform/squid-volume.sh dest=/root/ mode=preserve' 'ingress[0]'
+ansible -m command -a '/root/squid-volume.sh' 'ingress[0]'
+
+# k8s + notebooks
+ansible-playbook playbooks/k8s.yaml
+# docker runtime directory after Kubernetes deployment (problem with unmounts)
+ansible -m copy -a 'src=terraform/docker-volume.sh dest=/root/ mode=preserve' 'ingress nfs worker'
+ansible -m command -a '/root/docker-volume.sh' 'ingress nfs worker'
+
+# ansible-playbook playbooks/squid.yaml
+# ansible-playbook playbooks/cvmfs.yaml
+
+while ansible -i ./inventory -m command -a 'kubectl get pods --all-namespaces' master | tail -n +3 | grep -v ' Running '; do sleep 5; done

staging/extra

+../cesnet-central/extra
\ No newline at end of file

staging/inventory/1-psnc.yaml

+---
+fip:
+  hosts:
+    62.3.174.45:
+
+master:
+  hosts:
+    192.168.3.27:
+      # must be IPv4 address or hostname
+      kube_server: 192.168.3.27
+
+ingress:
+  hosts:
+    192.168.1.221:
+
+nfs:
+  hosts:
+    192.168.0.64:
+
+worker:
+  hosts:
+    192.168.1.24:
+
+gpu:
+  hosts:
+
+# using public IP of kube_server for ansible delegate_to
+kube_server:
+  hosts:
+    192.168.3.27:
+      ansible_host: 192.168.3.27

staging/inventory/99-all.yaml

+---
+allnodes:
+  children:
+    master:
+    ingress:
+    nfs:
+    worker:
+
+all:
+  vars:
+    ansible_become: yes
+    ansible_user: egi
+    ansible_ssh_common_args: '-o ProxyCommand="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -W %h:%p -q egi@{{ groups["fip"][0] }}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'
+
+    site_name: psnc-staging
+    vault_mount_point: secrets/users/e1662e20-e34b-468c-b0ce-d899bc878364@egi.eu/eosc-staging
+
+    grafana_hostname: grafana-stg.cloud.cesnet.cz

staging/playbooks/cvmfs.yaml

+../../cesnet-central/playbooks/cvmfs.yaml
\ No newline at end of file

staging/playbooks/files/calico.yaml

+../../../cesnet-central/playbooks/files/calico.yaml
\ No newline at end of file

staging/playbooks/files/helm_repos.fact

+../../../cesnet-central/playbooks/files/helm_repos.fact
\ No newline at end of file

staging/playbooks/files/k8s-cheats.sh

+../../../cesnet-central/playbooks/files/k8s-cheats.sh
\ No newline at end of file

staging/playbooks/files/xfs-quotas.sh

+../../../cesnet-central/playbooks/files/xfs-quotas.sh
\ No newline at end of file

staging/playbooks/k8s.yaml

+../../cesnet-central/playbooks/k8s.yaml
\ No newline at end of file

staging/playbooks/public_keys

+../../cesnet-central/playbooks/public_keys
\ No newline at end of file

staging/playbooks/squid.yaml

+../../cesnet-central/playbooks/squid.yaml
\ No newline at end of file

staging/playbooks/templates/etc/exports

+# export the NFS directory to all the cluster members
+/exports {% for host in groups['allnodes'] -%}{{ hostvars[host].ansible_default_ipv4.address }}(rw,async,no_root_squash,no_subtree_check) {% endfor -%}

staging/playbooks/templates/etc/mailutils.conf

+../../../../cesnet-central/playbooks/templates/etc/mailutils.conf
\ No newline at end of file

staging/playbooks/templates/etc/squid/conf.d/allcluster.conf

+{% for host in groups['ingress'] + groups['nfs'] + groups['worker'] + groups['gpu'] -%}
+acl allcluster src {{ hostvars[host].ansible_default_ipv4.address }}
+{% endfor -%}
+http_access allow allcluster

staging/playbooks/upgrade.yaml

+/home/valtri/notebooks-operations.eosc/cesnet-central/playbooks/upgrade.yaml
\ No newline at end of file

staging/terraform/.gitignore

+/.terraform/
+/.terraform.lock.hcl
+/fip.txt
+/hosts.txt
+/inventory.yaml
+/terraform.tfstate
+/terraform.tfstate.backup
+/*-volume.sh

staging/terraform/cloud-init.yaml

+#cloud-config
+---
+package_update: true
+
+package_upgrade: true
+
+users:
+  - default
+  - name: egi
+    gecos: EGI
+    primary_group: egi
+    groups: users
+    shell: /bin/bash
+    sudo: ALL=(ALL) NOPASSWD:ALL
+    ssh_import_id:
+      - gh:enolfc
+      - gh:andrea-manzi
+      - gh:valtri