From 9594881b3e3acccd50de84e63817644b9c6b2cd9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Franti=C5=A1ek=20Dvo=C5=99=C3=A1k?= <valtri@civ.zcu.cz>
Date: Fri, 14 Jun 2024 09:23:46 +0000
Subject: [PATCH] Try to fix a race-condition during docker runtime volume
 setup

---
 staging/deploy.sh        | 1 +
 staging/terraform/vms.tf | 4 +++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/staging/deploy.sh b/staging/deploy.sh
index 00c6352..06fc8bd 100755
--- a/staging/deploy.sh
+++ b/staging/deploy.sh
@@ -44,6 +44,7 @@ ansible -m command -a '/root/squid-volume.sh' 'ingress[0]'
 
 # k8s + notebooks
 ansible-playbook playbooks/k8s.yaml
+while ansible -i ./inventory -m command -a 'kubectl get pods --all-namespaces' master | tail -n +3 | grep -v ' Running '; do sleep 5; done
 # docker runtime directory after Kubernetes deployment (problem with unmounts)
 ansible -m copy -a 'src=terraform/docker-volume.sh dest=/root/ mode=preserve' 'ingress nfs worker'
 ansible -m command -a '/root/docker-volume.sh' 'ingress nfs worker'
diff --git a/staging/terraform/vms.tf b/staging/terraform/vms.tf
index c1889c0..c3a30e0 100644
--- a/staging/terraform/vms.tf
+++ b/staging/terraform/vms.tf
@@ -237,13 +237,15 @@ if ! dumpe2fs -h "$device" >/dev/null 2>&1; then
 	mkfs.ext4 -L DOCKER "$device"
 	grep -q 'LABEL=DOCKER' /etc/fstab || /bin/echo -e "LABEL=DOCKER\t/var/lib/docker/overlay2\text4\tdefaults,x-systemd.before=local-fs.target\t0\t0" | tee -a /etc/fstab
 	mkdir -p /var/lib/docker/overlay2 2>/dev/null || true
-	service docker stop >/dev/null 2>&1 || true
+	systemctl stop docker kubelet >/dev/null 2>&1 || true
 	sleep 10
+	systemctl stop docker kubelet >/dev/null 2>&1 || true
 	umount /var/lib/docker/overlay2 2>&1 || true
 	mount "$device" /mnt
 	mv /var/lib/docker/overlay2/* /mnt >/dev/null 2>&1 || true
 	umount /mnt
 	mount -a
+	systemctl start docker kubelet >/dev/null 2>&1 || true
 fi
 EOT
 }
-- 
GitLab