From 6a58a802b75c2233736eb05e55c716ba2c6f75cd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Franti=C5=A1ek=20Dvo=C5=99=C3=A1k?= <valtri@civ.zcu.cz>
Date: Mon, 10 Feb 2025 13:17:31 +0000
Subject: [PATCH] EGI devel deployment
* OpenStack infrastructure
* Kubernetes
* Jupyter Notebooks
* Sonatype Nexus repository
* tune email config
* Matlab
* local accounting
* backup and recover
---
common/playbooks/k8s.yaml | 4 +-
.../accounting-config.yaml | 27 ++
egi-devel/ansible.cfg | 5 +
egi-devel/deploy.sh | 58 +++
egi-devel/deployments/hub.yaml | 433 ++++++++++++++++++
egi-devel/extra | 1 +
egi-devel/inventory/1-cesnet.yaml | 31 ++
egi-devel/inventory/99-all.yaml | 27 ++
egi-devel/playbooks/accounting.yaml | 1 +
egi-devel/playbooks/backup.yaml | 1 +
egi-devel/playbooks/cvmfs.yaml | 1 +
egi-devel/playbooks/files/calico.yaml | 1 +
egi-devel/playbooks/files/etc | 1 +
egi-devel/playbooks/files/jupyterhub-jwt.yaml | 1 +
egi-devel/playbooks/files/usr | 1 +
egi-devel/playbooks/k8s.yaml | 1 +
egi-devel/playbooks/notebooks.yaml | 1 +
egi-devel/playbooks/public_keys | 1 +
egi-devel/playbooks/recover.yaml | 1 +
egi-devel/playbooks/repository-nexus.yaml | 1 +
egi-devel/playbooks/squid.yaml | 1 +
egi-devel/playbooks/subtasks | 1 +
egi-devel/playbooks/templates/backup.yaml | 1 +
egi-devel/playbooks/templates/etc/exports | 1 +
.../playbooks/templates/etc/mailutils.conf | 1 +
egi-devel/playbooks/templates/etc/squid | 1 +
egi-devel/playbooks/templates/nexus | 1 +
egi-devel/playbooks/templates/nexus.yaml | 1 +
egi-devel/playbooks/templates/recover.yaml | 1 +
egi-devel/playbooks/upgrade.yaml | 1 +
egi-devel/terraform/cloud-init.yaml | 1 +
egi-devel/terraform/firewall.tf | 1 +
egi-devel/terraform/terraform.tfvars | 34 ++
egi-devel/terraform/vars.tf | 1 +
egi-devel/terraform/versions.tf | 1 +
egi-devel/terraform/vms.tf | 245 ++++++++++
36 files changed, 889 insertions(+), 2 deletions(-)
create mode 100644 egi-devel/accounting_deployments/accounting-config.yaml
create mode 100644 egi-devel/ansible.cfg
create mode 100755 egi-devel/deploy.sh
create mode 100644 egi-devel/deployments/hub.yaml
create mode 120000 egi-devel/extra
create mode 100644 egi-devel/inventory/1-cesnet.yaml
create mode 100644 egi-devel/inventory/99-all.yaml
create mode 120000 egi-devel/playbooks/accounting.yaml
create mode 120000 egi-devel/playbooks/backup.yaml
create mode 120000 egi-devel/playbooks/cvmfs.yaml
create mode 120000 egi-devel/playbooks/files/calico.yaml
create mode 120000 egi-devel/playbooks/files/etc
create mode 120000 egi-devel/playbooks/files/jupyterhub-jwt.yaml
create mode 120000 egi-devel/playbooks/files/usr
create mode 120000 egi-devel/playbooks/k8s.yaml
create mode 120000 egi-devel/playbooks/notebooks.yaml
create mode 120000 egi-devel/playbooks/public_keys
create mode 120000 egi-devel/playbooks/recover.yaml
create mode 120000 egi-devel/playbooks/repository-nexus.yaml
create mode 120000 egi-devel/playbooks/squid.yaml
create mode 120000 egi-devel/playbooks/subtasks
create mode 120000 egi-devel/playbooks/templates/backup.yaml
create mode 120000 egi-devel/playbooks/templates/etc/exports
create mode 120000 egi-devel/playbooks/templates/etc/mailutils.conf
create mode 120000 egi-devel/playbooks/templates/etc/squid
create mode 120000 egi-devel/playbooks/templates/nexus
create mode 120000 egi-devel/playbooks/templates/nexus.yaml
create mode 120000 egi-devel/playbooks/templates/recover.yaml
create mode 120000 egi-devel/playbooks/upgrade.yaml
create mode 120000 egi-devel/terraform/cloud-init.yaml
create mode 120000 egi-devel/terraform/firewall.tf
create mode 100644 egi-devel/terraform/terraform.tfvars
create mode 120000 egi-devel/terraform/vars.tf
create mode 120000 egi-devel/terraform/versions.tf
create mode 100644 egi-devel/terraform/vms.tf
diff --git a/common/playbooks/k8s.yaml b/common/playbooks/k8s.yaml
index c68aa1d..cec8cdc 100644
--- a/common/playbooks/k8s.yaml
+++ b/common/playbooks/k8s.yaml
@@ -67,7 +67,7 @@
inet_protocols: ipv4
set_fact:
main: '{{ main | combine(main_cesnet) }}'
- when: site_name == "cesnet-testing" or site_name == "cesnet-mcc"
+ when: site_name is regex('^(cesnet|egi)-')
- name: Site-specific postfix settings - mail_fromdomain
set_fact:
main: '{{ main | combine({"myhostname": mail_fromdomain}) }}'
@@ -90,7 +90,7 @@
src: templates/etc/mailutils.conf
dest: /etc/mailutils.conf
mode: 0644
- when: (site_name == "cesnet-testing" or site_name == "cesnet-mcc" or mail_fromdomain is defined) and not (mail_local | default(false))
+ when: (site_name is regex('^(cesnet|egi)-') or mail_fromdomain is defined) and not (mail_local | default(false))
rescue:
- name: Mail Settings Fail
fail:
diff --git a/egi-devel/accounting_deployments/accounting-config.yaml b/egi-devel/accounting_deployments/accounting-config.yaml
new file mode 100644
index 0000000..c4311dd
--- /dev/null
+++ b/egi-devel/accounting_deployments/accounting-config.yaml
@@ -0,0 +1,27 @@
+accounting:
+ # schedule: 23 */6 * * *
+ sitename: EGI-NOTEBOOKS-DEVEL2
+ default_fqan: vo.notebooks.egi.eu
+ # fqan_key: primary_group
+ fqan:
+ auger: urn:mace:egi.eu:group:auger:role=member#aai.egi.eu
+ biomed: urn:mace:egi.eu:group:biomed:role=member#aai.egi.eu
+ eiscat.se: urn:mace:egi.eu:group:eiscat.se:Dev:role=member#aai.egi.eu,urn:mace:egi.eu:group:eiscat.se:Hub:role=member#aai.egi.eu,urn:mace:egi.eu:group:cc-eiscat3d#sso.egi.eu
+ eval.c-scale.eu: urn:mace:egi.eu:group:eval.c-scale.eu:role=member#aai.egi.eu
+ vo.cessda.eduteams.org: urn:mace:egi.eu:group:vo.cessda.eduteams.org:role=member#aai.egi.eu
+ vo.environmental.egi.eu: urn:mace:egi.eu:group:vo.environmental.egi.eu:role=member#aai.egi.eu
+ vo.lethe-project.eu: urn:mace:egi.eu:group:vo.lethe-project.eu:role=member#aai.egi.eu,urn:mace:egi.eu:group:vo.lethe-project.eu:lethe-notebooks:role=member#aai.egi.eu
+ vo.panosc.eu: urn:mace:egi.eu:group:vo.panosc.eu:role=vm_operator#aai.egi.eu
+ vo.reliance-project.eu: urn:mace:egi.eu:group:vo.reliance-project.eu:role=member#aai.egi.eu
+ vo.access.egi.eu: urn:mace:egi.eu:group:vo.access.egi.eu:role=member#aai.egi.eu
+ vo.notebooks.egi.eu: urn:mace:egi.eu:group:vo.notebooks.egi.eu:role=member#aai.egi.eu
+ # used in devel
+ fedcloud.egi.eu: urn:mace:egi.eu:www.egi.eu:fedcloud-users:member@egi.eu,urn:mace:egi.eu:group:fedcloud-users#sso.egi.eu,urn:egi.eu:group:fedcloud-users
+
+debug: true
+
+prometheus:
+ filter: "pod=~'jupyter-.*'"
+
+storage:
+ apelSpool: /accounting/ssm
diff --git a/egi-devel/ansible.cfg b/egi-devel/ansible.cfg
new file mode 100644
index 0000000..c3a73be
--- /dev/null
+++ b/egi-devel/ansible.cfg
@@ -0,0 +1,5 @@
+[defaults]
+inventory=inventory
+
+[diff]
+always=true
diff --git a/egi-devel/deploy.sh b/egi-devel/deploy.sh
new file mode 100755
index 0000000..b9f1de0
--- /dev/null
+++ b/egi-devel/deploy.sh
@@ -0,0 +1,58 @@
+#! /bin/bash -xe
+
+#
+# Deploy EGI devel instance
+#
+
+cd terraform && terraform init && terraform apply
+cd -
+cp -pv terraform/inventory.yaml inventory/1-cesnet.yaml
+
+# dynamic DNS
+ip="$(head -n 1 <terraform/fip.txt)"
+shellstate=$(shopt -po xtrace)
+set +o xtrace
+# https://nsupdate.fedcloud.eu
+vault_prefix=secrets/users/e1662e20-e34b-468c-b0ce-d899bc878364@egi.eu/egi-devel
+FEDCLOUD_DYNAMIC_DNS=$(vault read -field data $vault_prefix/FEDCLOUD_DYNAMIC_DNS | grep ^map | head -n 1 | sed 's/map\[\(.*\)\]/\1/')
+for auth in $FEDCLOUD_DYNAMIC_DNS; do
+ echo "curl -i -X GET -u $(echo "$auth" | cut -d: -f1):XXX https://nsupdate.fedcloud.eu/nic/update?myip=$ip"
+ curl -i -X GET -u "$auth" https://nsupdate.fedcloud.eu/nic/update?myip="$ip"
+done
+eval "$shellstate"
+echo "Terraform finished. Check terraform/docker-volume.sh. Continue? (CTRL-C to quit)"
+read -r _
+
+# wait for ping and ssh
+while read -r ip; do
+ while ! ping -c 1 "$ip"; do sleep 5; done
+ ssh-keygen -R "$ip"
+ while ! ssh egi@"$ip" -o ConnectTimeout=10 -o PreferredAuthentications=publickey -o StrictHostKeyChecking=no :; do sleep 10; done
+done <terraform/fip.txt
+
+# check ssh access
+ansible -m command -a 'uname -a' allnodes
+
+# wait cloud-init
+ansible -m shell -a 'while ! test -f /var/lib/cloud/instance/boot-finished; do sleep 2; done' allnodes
+
+# setup volumes
+ansible -m copy -a 'src=terraform/nfs-volume.sh dest=/root/ mode=preserve' nfs
+ansible -m command -a '/root/nfs-volume.sh' nfs
+ansible -m copy -a 'src=terraform/squid-volume.sh dest=/root/ mode=preserve' 'ingress[0]'
+ansible -m command -a '/root/squid-volume.sh' 'ingress[0]'
+
+# kubernetes
+ansible-playbook playbooks/k8s.yaml
+while ansible -m command -a 'kubectl get pods --all-namespaces' master | tail -n +3 | grep -Ev ' (Running|Completed) '; do sleep 5; done
+# docker runtime directory after Kubernetes deployment (problem with unmounts)
+ansible -m copy -a 'src=terraform/docker-volume.sh dest=/root/ mode=preserve' 'ingress nfs worker gpu'
+ansible -m command -a '/root/docker-volume.sh' 'ingress nfs worker gpu'
+ansible-playbook playbooks/squid.yaml
+ansible-playbook playbooks/cvmfs.yaml
+
+# image repository
+ansible-playbook playbooks/repository-nexus.yaml
+
+# wait for finish
+while ansible -m command -a 'kubectl get pods --all-namespaces' master | tail -n +3 | grep -Ev ' (Running|Completed) '; do sleep 5; done
diff --git a/egi-devel/deployments/hub.yaml b/egi-devel/deployments/hub.yaml
new file mode 100644
index 0000000..5d7ec86
--- /dev/null
+++ b/egi-devel/deployments/hub.yaml
@@ -0,0 +1,433 @@
+---
+proxy:
+ service:
+ type: NodePort
+
+ingress:
+ enabled: true
+ annotations:
+ kubernetes.io/ingress.class: "nginx"
+ kubernetes.io/tls-acme: "true"
+ hosts:
+ - "{{ notebooks_hostname }}"
+ tls:
+ - hosts:
+ - "{{ notebooks_hostname }}"
+ secretName: acme-tls-hub
+
+imagePullSecret:
+ create: true
+ registry: "{{ registry_notebooks_hostname }}"
+ username: notebooks-reader
+ password: "{{ nexus_secrets['notebooks-reader'] }}"
+
+singleuser:
+ # keep resource limits in sync with:
+ # - profileList
+ # - EGI web-page https://www.egi.eu/service/notebooks/
+ storage:
+ capacity: 10Gi
+ dynamic:
+ pvcNameTemplate: claim-{userid}{servername}
+ volumeNameTemplate: vol-{userid}{servername}
+ storageAccessModes: ["ReadWriteMany"]
+ extraVolumes:
+ - name: cvmfs-host
+ hostPath:
+ path: /cvmfs
+ type: Directory
+ - name: b2drop
+ empty_dir:
+ extraVolumeMounts:
+ - name: cvmfs-host
+ mountPath: "/cvmfs:shared"
+ - name: b2drop
+ mountPath: '/mnt/b2drop:shared'
+ lifecycleHooks:
+ postStart:
+ exec:
+ command:
+ - "/bin/sh"
+ - "-c"
+ - |
+ ln -snf /mnt/oneclient $HOME/datahub
+ ln -snf /mnt/b2drop $HOME/b2drop
+ ln -snf /cvmfs $HOME/cvmfs
+ mkdir -p /home/jovyan/.notebookCheckpoints
+ memory:
+ limit: 6G
+ guarantee: 128M
+ cpu:
+ limit: 2
+ guarantee: .02
+ defaultUrl: "/lab"
+ networkPolicy:
+ egress:
+ - to:
+ - namespaceSelector:
+ matchLabels:
+ kubernetes.io/metadata.name: kube-system
+ podSelector:
+ matchLabels:
+ app.kubernetes.io/instance: cluster-ingress
+ image:
+ name: eginotebooks/single-user
+ tag: "sha-0e47d79"
+ profileList:
+ - display_name: Default EGI environment - 6 GB RAM / 2 core
+ description: >
+ The Default notebook environment includes Python, R, Julia and Octave kernels.
+ default: true
+ kubespawner_override:
+ args:
+ - "--CondaKernelSpecManager.env_filter='/opt/conda$'"
+ - display_name: Default EGI environment with Elyra and AI tools - 6 GB RAM / 2 core
+ description: >
+ The Default notebook environment includes Python, R, Julia and Octave kernels extended with Elyra and AI tools.
+ kubespawner_override:
+ args:
+ - "--CondaKernelSpecManager.env_filter='/opt/conda$'"
+ image: "eginotebooks/single-user-ai:sha-0e47d79"
+ - display_name: RELIANCE project environment - 12 GB RAM / 2 core
+ description: >
+ Notebook environment for RELIANCE project includes Python, R, Julia and Octave kernels
+ kubespawner_override:
+ args:
+ - "--CondaKernelSpecManager.env_filter='/opt/conda$'"
+ mem_guarantee: 2G
+ mem_limit: 12G
+ vo_claims:
+ - urn:mace:egi.eu:group:notebooks-support#sso.egi.eu
+ - urn:mace:egi.eu:group:vo.reliance-project.eu:role=member#aai.egi.eu
+ - urn:mace:egi.eu:www.egi.eu:notebooks-support:member@egi.eu
+ - urn:mace:egi.eu:group:notebooks-support#sso.egi.eu
+ - display_name: MATLAB Environment (Basic) - 4GB RAM / 4 cores
+ description: >
+ The MATLAB environment 24.1.0.2537033 (R2024a) (requires a
+ <a href="https://github.com/mathworks/jupyter-matlab-proxy/blob/main/MATLAB-Licensing-Info.md">
+ valid license</a>), includes Python and MATLAB kernels
+ kubespawner_override:
+ cpu_guarantee: 2
+ cpu_limit: 4
+ mem_guarantee: 2G
+ mem_limit: 4G
+ image: "{{ registry_notebooks_hostname }}/matlab:r2024a-notebook"
+ - display_name: MATLAB Environment (Full) - 4GB RAM / 4 cores
+ description: >
+ The MATLAB environment 24.1.0.2537033 (R2024a) with toolboxes (requires a
+ <a href="https://github.com/mathworks/jupyter-matlab-proxy/blob/main/MATLAB-Licensing-Info.md">
+ valid license</a>), includes Python and MATLAB kernels
+ kubespawner_override:
+ cpu_guarantee: 2
+ cpu_limit: 4
+ mem_guarantee: 2G
+ mem_limit: 4G
+ image: "{{ registry_notebooks_hostname }}/matlab:r2024a-full"
+ - display_name: EISCAT environment - 4 GB RAM / 2 cores
+ description: >
+ The EISCAT environment.
+ kubespawner_override:
+ image: "ingemarh/guisdap"
+ imagePullPolicy: Always
+ vo_claims:
+ - urn:mace:egi.eu:group:cc-eiscat3d#sso.egi.eu
+ - urn:mace:egi.eu:group:eiscat.se:Hub:role=member#aai.egi.eu
+ - urn:mace:egi.eu:group:notebooks-support#sso.egi.eu
+ - urn:mace:egi.eu:www.egi.eu:notebooks-support:member@egi.eu
+ - urn:mace:egi.eu:group:notebooks-support#sso.egi.eu
+
+hub:
+ services:
+ status:
+ url: "http://status-web/"
+ admin: true
+ # recommended to keep in sync with common/playbooks/files/jupyterhub-jwt.yaml
+ # keep k8s-hub version in sync with ../playbooks/notebooks.yaml
+ image:
+ name: eginotebooks/hub
+ # k8s-hub 4.0.0
+ tag: "sha-aef23d2"
+ config:
+ Authenticator:
+ enable_auth_state: true
+ admin_users:
+ - 529a87e5ce04cd5ddd7161734d02df0e2199a11452430803e714cb1309cc3907@egi.eu
+ - 025166931789a0f57793a6092726c2ad89387a4cc167e7c63c5d85fc91021d18@egi.eu
+ - 7ce47695f1e7fc91a1156e672f4a47576559938cdbe840355e2429e3a05b4ff8@egi.eu
+ # fdvorak2 @ aai.egi.eu
+ - 52cc7599bd1553c9d63e34e4c90b7e84d44967490c28bb4c53fe97b0c881d677@egi.eu
+ # fdvorak2 @ aai-dev.egi.eu
+ - c481e0a85e1ae0a5a1480a63e62295ca2f9ac652244947995bd4a0210fbcb77c@egi.eu
+ # jhradil3 @ aai-dev.egi.eu
+ - 240c0594fe34ac26cffd82fd0ad85f29d9ad9dfbb46febb05ed42db0bff594d1@egi.eu
+ # keep in sync with:
+ # - cesnet/playbooks/templates/binder.yaml
+ # - documentation/content/en/users/dev-env/notebooks/_index.md
+ allowed_groups:
+ # EISCAT Dirac testing
+ - urn:mace:egi.eu:group:cc-eiscat3d#sso.egi.eu
+ - urn:mace:egi.eu:group:auger:role=member#aai.egi.eu
+ - urn:mace:egi.eu:group:biomed:role=member#aai.egi.eu
+ - urn:mace:egi.eu:group:vo.cessda.eduteams.org:role=member#aai.egi.eu
+ - urn:mace:egi.eu:group:eiscat.se:Hub:role=member#aai.egi.eu
+ - urn:mace:egi.eu:group:eval.c-scale.eu:role=member#aai.egi.eu
+ - urn:mace:egi.eu:group:vo.environmental.egi.eu:role=member#aai.egi.eu
+ - urn:mace:egi.eu:group:vo.lethe-project.eu:lethe-notebooks:role=member#aai.egi.eu
+ - urn:mace:egi.eu:group:vo.panosc.eu:role=vm_operator#aai.egi.eu
+ - urn:mace:egi.eu:group:vo.reliance-project.eu:role=member#aai.egi.eu
+ - urn:mace:egi.eu:group:vo.access.egi.eu:role=member#aai.egi.eu
+ - urn:mace:egi.eu:group:vo.notebooks.egi.eu:role=member#aai.egi.eu
+ - urn:mace:egi.eu:group:vo.bioexcel.eu:role=member#aai.egi.eu
+ - urn:mace:egi.eu:group:vo.egu2024.egi.eu:role=member#aai.egi.eu
+ - urn:mace:egi.eu:www.egi.eu:fedcloud-users:member@egi.eu
+ - urn:mace:egi.eu:www.egi.eu:techsolutions:member@egi.eu
+ # changed 2022-10
+ - urn:mace:egi.eu:group:fedcloud-users#sso.egi.eu
+ - urn:mace:egi.eu:group:supplier-notebooks#sso.egi.eu
+ - urn:mace:egi.eu:group:techsolutions#sso.egi.eu
+ - urn:mace:egi.eu:group:notebooks-support#sso.egi.eu
+ # changed 2025-03
+ - urn:egi.eu:group:supplier-notebooks
+ - urn:egi.eu:group:notebooks-support
+ - urn:egi.eu:group:fedcloud-users
+ - urn:egi.eu:group:egi-ace-all
+ auto_login: true
+ claim_groups_key: "entitlements"
+ OnedataAuthenticator:
+ oneprovider_host: "cesnet-oneprovider-01.datahub.egi.eu"
+ authorize_url: "https://{{ secrets['checkin_host'] }}/auth/realms/egi/protocol/openid-connect/auth"
+ token_url: "https://{{ secrets['checkin_host'] }}/auth/realms/egi/protocol/openid-connect/token"
+ userdata_url: "https://{{ secrets['checkin_host'] }}/auth/realms/egi/protocol/openid-connect/userinfo"
+ client_id: "{{ secrets['client_id'] }}"
+ client_secret: "{{ secrets['client_secret'] }}"
+ oauth_callback_url: "https://{{ notebooks_hostname }}/hub/oauth_callback"
+ scope: ["openid", "profile", "email", "offline_access", "eduperson_scoped_affiliation", "eduperson_entitlement"]
+ username_key: "sub"
+ OnedataSpawner:
+ sidecar_image: "eginotebooks/oneclient-sidecar:sha-9789b9a"
+ force_direct_io: true
+ http_timeout: 60
+ args:
+ - "--FileCheckpoints.checkpoint_dir='/home/jovyan/.notebookCheckpoints'"
+ JupyterHub:
+ admin_access: true
+ authenticate_prometheus: false
+ authenticator_class: egi_notebooks_hub.onedata.OnedataAuthenticator
+ # spawner_class: (in egi-notebooks-b2drop)
+ extraConfig:
+ egi-notebooks-welcome: |-
+ from egi_notebooks_hub.welcome import WelcomeHandler
+ c.JupyterHub.default_url = "/welcome"
+ c.JupyterHub.extra_handlers = [(r'/welcome', WelcomeHandler)]
+ egi-notebooks-b2drop: |-
+{%- raw %}
+ import base64
+ from jinja2 import BaseLoader
+ from jinja2 import Environment
+ from egi_notebooks_hub.onedata import OnedataSpawner
+ from kubernetes_asyncio.client.rest import ApiException
+
+ class B2DropSpawner(OnedataSpawner):
+ async def auth_state_hook(self, spawner, auth_state):
+ await super().auth_state_hook(spawner, auth_state)
+ self.b2drop_ready = False
+ self.b2drop_user = ""
+ self.b2drop_pwd = ""
+ try:
+ secret = await self.api.read_namespaced_secret(self.token_secret_name, self.namespace)
+ except ApiException:
+ return
+ if secret and secret.data:
+ self.b2drop_user = base64.b64decode(secret.data.get("b2drop-user", "")).decode()
+ self.b2drop_pwd = base64.b64decode(secret.data.get("b2drop-pwd", "")).decode()
+ self.b2drop_ready = (self.b2drop_user and self.b2drop_pwd)
+
+ def _render_options_form(self, profile_list):
+ # old hub: self._profile_list = self._init_profile_list(profile_list)
+ self._profile_list = self._get_initialized_profile_list(profile_list)
+ profile_form_template = Environment(loader=BaseLoader).from_string(
+ self.profile_form_template
+ )
+ return profile_form_template.render(profile_list=self._profile_list, b2drop_ready=self.b2drop_ready, b2drop_user=self.b2drop_user, b2drop_pwd=self.b2drop_pwd)
+
+ async def pre_spawn_hook(self, spawner):
+ await super(B2DropSpawner, self).pre_spawn_hook(spawner)
+ b2drop_user = self.user_options.get("b2drop-user", "")
+ b2drop_pwd = self.user_options.get("b2drop-pwd", "")
+ b2drop_remember = self.user_options.get("b2drop-remember", None)
+ if not (b2drop_user and b2drop_pwd):
+ secret = await self.api.read_namespaced_secret(self.token_secret_name, self.namespace)
+ if secret and secret.data:
+ b2drop_user = base64.b64decode(secret.data.get("b2drop-user", "")).decode()
+ b2drop_pwd = base64.b64decode(secret.data.get("b2drop-pwd", "")).decode()
+ if b2drop_user and b2drop_pwd:
+ volume_mounts = [
+ {"mountPath": "/b2drop:shared", "name": "b2drop"},
+ ]
+ spawner.extra_containers.append(
+ {
+ "name": "b2drop",
+ "image": "eginotebooks/webdav-sidecar:sha-e5e8df2",
+ "env": [
+ {"name": "WEBDAV_URL", "value": "https://b2drop.eudat.eu/remote.php/webdav"},
+ {"name": "WEBDAV_PWD", "value": b2drop_pwd},
+ {"name": "WEBDAV_USER", "value": b2drop_user},
+ {"name": "MOUNT_PATH", "value": "/b2drop"},
+ ],
+ "resources": self.sidecar_resources,
+ # "command": cmd,
+ "securityContext": {
+ "runAsUser": 0,
+ "privileged": True,
+ "capabilities": {"add": ["SYS_ADMIN"]},
+ },
+ "volumeMounts": volume_mounts,
+ "lifecycle": {
+ "preStop": {
+ "exec": {"command": ["umount", "-l", "/b2drop"]}
+ },
+ },
+ }
+ )
+ if b2drop_remember:
+ await self._update_secret({"b2drop-user": b2drop_user,
+ "b2drop-pwd": b2drop_pwd})
+ else:
+ await self._update_secret({"b2drop-user": "", "b2drop-pwd": ""})
+
+ def options_from_form(self, formdata):
+ data = super(B2DropSpawner, self)._options_from_form(formdata)
+ data.update({'b2drop-user': formdata.get('b2drop-user', [None])[0],
+ 'b2drop-remember': formdata.get('b2drop-remember', [None])[0],
+ 'b2drop-pwd': formdata.get('b2drop-pwd', [None])[0]})
+ return data
+
+ c.JupyterHub.spawner_class = B2DropSpawner
+ c.B2DropSpawner.http_timeout = 60
+ c.B2DropSpawner.args = ["--FileCheckpoints.checkpoint_dir='/home/jovyan/.notebookCheckpoints'"]
+ c.B2DropSpawner.profile_form_template = """
+ <style>
+ /*
+ .profile divs holds two div tags: one for a radio button, and one
+ for the profile's content.
+ */
+ #kubespawner-profiles-list .profile {
+ display: flex;
+ flex-direction: row;
+ font-weight: normal;
+ border-bottom: 1px solid #ccc;
+ padding-bottom: 12px;
+ }
+
+ #kubespawner-profiles-list .profile .radio {
+ padding: 12px;
+ }
+
+ /* .option divs holds a label and a select tag */
+ #kubespawner-profiles-list .profile .option {
+ display: flex;
+ flex-direction: row;
+ align-items: center;
+ padding-bottom: 12px;
+ }
+
+ #kubespawner-profiles-list .profile .option label {
+ font-weight: normal;
+ margin-right: 8px;
+ min-width: 96px;
+ }
+ </style>
+
+ <div class='form-group' id='kubespawner-profiles-list'>
+ {%- for profile in profile_list %}
+ {#- Wrap everything in a <label> so clicking anywhere selects the option #}
+ <label for='profile-item-{{ profile.slug }}' class='profile'>
+ <div class='radio'>
+ <input type='radio' name='profile' id='profile-item-{{ profile.slug }}'
+ value='{{ profile.slug }}' {% if profile.default %}checked{% endif %} />
+ </div>
+ <div>
+ <h3>{{ profile.display_name }}</h3>
+
+ {%- if profile.description %}
+ <p>{{ profile.description }}</p>
+ {%- endif %}
+
+ {%- if profile.profile_options %}
+ <div>
+ {%- for k, option in profile.profile_options.items() %}
+ <div class='option'>
+ <label for='profile-option-{{profile.slug}}-{{k}}'>{{option.display_name}}</label>
+ <select name="profile-option-{{profile.slug}}-{{k}}" class="form-control">
+ {%- for k, choice in option['choices'].items() %}
+ <option value="{{ k }}" {% if choice.default %}selected{%endif %}>{{ choice.display_name }}</option>
+ {%- endfor %}
+ </select>
+ </div>
+ {%- endfor %}
+ </div>
+ {%- endif %}
+ </div>
+ </label>
+ {%- endfor %}
+ <div class="panel-group" id="accordion" role="tablist" aria-multiselectable="true">
+ <div class="panel panel-default">
+ <div class="panel-heading" role="tab" id="headingOne">
+ <h4 class="panel-title">
+ <a class="collapsed" role="button" data-bs-toggle="collapse" data-parent="#accordion" href="#collapseOne" aria-expanded="false"
+ aria-controls="collapseOne">
+ B2DROP connection
+ </a>
+ {%if b2drop_ready %}<span class="label label-success">Already configured!</span>{% endif %}
+ </h4>
+ </div>
+ <div id="collapseOne" class="panel-collapse collapse" role="tabpanel" aria-labelledby="headingOne">
+ <div class="panel-body">
+ <div class='form-group'>
+ <label for="b2drop-user" class="form-label">B2DROP app Username</label>
+ <input type="text" class="form-control" name="b2drop-user" id="b2drop-user" aria-describedby="b2drop-user-help" value="{{ b2drop_user }}">
+ <div id="b2drop-user-help" class="form-text">Create new app password at <a href="https://b2drop.eudat.eu/settings/user/security">
+ B2DROP security configuration</a></div>
+ </div>
+ <div class='form-group'>
+ <label for="b2drop-pwd" class="form-label">B2DROP app Password</label>
+ <input type="password" class="form-control" name="b2drop-pwd" id="b2drop-pwd" value="{{ b2drop_pwd }}">
+ </div>
+ <div class='form-group'>
+ <input type="checkbox" id="b2drop-remember" name="b2drop-remember" {%if b2drop_ready %}checked{% endif %}>
+ <label class="form-check-label" for="from-check-input">Remember B2DROP credentials</label>
+ </div>
+ </div>
+ </div>
+ </div>
+ </div>
+ """
+{% endraw %}
+ templatePaths:
+ - /egi-notebooks-hub/templates
+ extraFiles:
+ login.html:
+ mountPath: /egi-notebooks-hub/templates/login.html
+ stringData: |-
+{%- raw %}
+ {% extends "egi-login.html" %}
+ {% block main_intro %}
+ <h1><img alt="Notebooks Logo" src="{{ static_url('images/egi-icon-notebooks.svg') }}"
+ height="100">Notebooks</h1>
+ <p>
+ Notebooks is an environment based on <a href="http://jupyter.org/">Jupyter</a> and
+ the <a href="https://www.egi.eu/services/cloud-compute/">EGI cloud service</a> that
+ offers a browser-based, scalable tool for interactive data analysis. The Notebooks
+ environment provides users with notebooks where they can combine text, mathematics,
+ computations and rich media output.
+ </p>
+ <p>
+ Access requires a valid <a href="https://docs.egi.eu/users/check-in/signup">EGI account</a>
+ and <a href="https://docs.egi.eu/users/dev-env/notebooks/#notebooks-for-researchers">
+ enrolling to one of the supported VOs</a>.
+ </p>
+ <p>
+ Default environment provides 4 vCPU cores, 6 GB RAM and 10GB of personal storage space per user.
+ </p>
+ {% endblock main_intro %}
+{% endraw %}
diff --git a/egi-devel/extra b/egi-devel/extra
new file mode 120000
index 0000000..440decb
--- /dev/null
+++ b/egi-devel/extra
@@ -0,0 +1 @@
+../common/extra
\ No newline at end of file
diff --git a/egi-devel/inventory/1-cesnet.yaml b/egi-devel/inventory/1-cesnet.yaml
new file mode 100644
index 0000000..0238e7f
--- /dev/null
+++ b/egi-devel/inventory/1-cesnet.yaml
@@ -0,0 +1,31 @@
+---
+fip:
+ hosts:
+ 147.251.245.198:
+
+master:
+ hosts:
+ 192.168.0.77:
+ # must be IPv4 address or hostname
+ kube_server: 192.168.0.77
+
+ingress:
+ hosts:
+ 192.168.0.118:
+
+nfs:
+ hosts:
+ 192.168.0.219:
+
+worker:
+ hosts:
+ 192.168.0.119:
+
+gpu:
+ hosts:
+
+# using public IP of kube_server for ansible delegate_to
+kube_server:
+ hosts:
+ 192.168.0.77:
+ ansible_host: 192.168.0.77
diff --git a/egi-devel/inventory/99-all.yaml b/egi-devel/inventory/99-all.yaml
new file mode 100644
index 0000000..bda844c
--- /dev/null
+++ b/egi-devel/inventory/99-all.yaml
@@ -0,0 +1,27 @@
+---
+allnodes:
+ children:
+ master:
+ ingress:
+ nfs:
+ worker:
+ gpu:
+
+all:
+ vars:
+ ansible_become: true
+ ansible_user: egi
+ ansible_ssh_common_args: >-
+ -o ProxyCommand="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -W %h:%p -q egi@{{ groups["fip"][0] }}"
+ -o StrictHostKeyChecking=no
+ -o UserKnownHostsFile=/dev/null
+
+ site_name: egi-devel
+ vault_mount_point: secrets/users/e1662e20-e34b-468c-b0ce-d899bc878364@egi.eu/egi-devel
+ backup_repository: s3:s3.cl2.du.cesnet.cz/notebooks-devel-g2
+
+ notebooks_hostname: notebooks-dev.egi.zcu.cz
+ grafana_hostname: grafana.notebooks-dev.egi.zcu.cz
+ nexus_hostname: nexus.notebooks-dev.egi.zcu.cz
+ registry_binder_hostname: registry.binder-dev.egi.zcu.cz
+ registry_notebooks_hostname: registry.notebooks-dev.egi.zcu.cz
diff --git a/egi-devel/playbooks/accounting.yaml b/egi-devel/playbooks/accounting.yaml
new file mode 120000
index 0000000..8928484
--- /dev/null
+++ b/egi-devel/playbooks/accounting.yaml
@@ -0,0 +1 @@
+../../common/playbooks/accounting.yaml
\ No newline at end of file
diff --git a/egi-devel/playbooks/backup.yaml b/egi-devel/playbooks/backup.yaml
new file mode 120000
index 0000000..7456d1f
--- /dev/null
+++ b/egi-devel/playbooks/backup.yaml
@@ -0,0 +1 @@
+../../common/playbooks/backup.yaml
\ No newline at end of file
diff --git a/egi-devel/playbooks/cvmfs.yaml b/egi-devel/playbooks/cvmfs.yaml
new file mode 120000
index 0000000..2e82cca
--- /dev/null
+++ b/egi-devel/playbooks/cvmfs.yaml
@@ -0,0 +1 @@
+../../common/playbooks/cvmfs.yaml
\ No newline at end of file
diff --git a/egi-devel/playbooks/files/calico.yaml b/egi-devel/playbooks/files/calico.yaml
new file mode 120000
index 0000000..732c864
--- /dev/null
+++ b/egi-devel/playbooks/files/calico.yaml
@@ -0,0 +1 @@
+../../../common/playbooks/files/calico.yaml
\ No newline at end of file
diff --git a/egi-devel/playbooks/files/etc b/egi-devel/playbooks/files/etc
new file mode 120000
index 0000000..ed53b87
--- /dev/null
+++ b/egi-devel/playbooks/files/etc
@@ -0,0 +1 @@
+../../../common/playbooks/files/etc
\ No newline at end of file
diff --git a/egi-devel/playbooks/files/jupyterhub-jwt.yaml b/egi-devel/playbooks/files/jupyterhub-jwt.yaml
new file mode 120000
index 0000000..59f9ac2
--- /dev/null
+++ b/egi-devel/playbooks/files/jupyterhub-jwt.yaml
@@ -0,0 +1 @@
+../../../common/playbooks/files/jupyterhub-jwt.yaml
\ No newline at end of file
diff --git a/egi-devel/playbooks/files/usr b/egi-devel/playbooks/files/usr
new file mode 120000
index 0000000..b034223
--- /dev/null
+++ b/egi-devel/playbooks/files/usr
@@ -0,0 +1 @@
+../../../common/playbooks/files/usr
\ No newline at end of file
diff --git a/egi-devel/playbooks/k8s.yaml b/egi-devel/playbooks/k8s.yaml
new file mode 120000
index 0000000..117aed6
--- /dev/null
+++ b/egi-devel/playbooks/k8s.yaml
@@ -0,0 +1 @@
+../../common/playbooks/k8s.yaml
\ No newline at end of file
diff --git a/egi-devel/playbooks/notebooks.yaml b/egi-devel/playbooks/notebooks.yaml
new file mode 120000
index 0000000..3f1a33f
--- /dev/null
+++ b/egi-devel/playbooks/notebooks.yaml
@@ -0,0 +1 @@
+../../common/playbooks/notebooks.yaml
\ No newline at end of file
diff --git a/egi-devel/playbooks/public_keys b/egi-devel/playbooks/public_keys
new file mode 120000
index 0000000..6ef4918
--- /dev/null
+++ b/egi-devel/playbooks/public_keys
@@ -0,0 +1 @@
+../../common/playbooks/public_keys
\ No newline at end of file
diff --git a/egi-devel/playbooks/recover.yaml b/egi-devel/playbooks/recover.yaml
new file mode 120000
index 0000000..cb9970b
--- /dev/null
+++ b/egi-devel/playbooks/recover.yaml
@@ -0,0 +1 @@
+../../common/playbooks/recover.yaml
\ No newline at end of file
diff --git a/egi-devel/playbooks/repository-nexus.yaml b/egi-devel/playbooks/repository-nexus.yaml
new file mode 120000
index 0000000..e96cf70
--- /dev/null
+++ b/egi-devel/playbooks/repository-nexus.yaml
@@ -0,0 +1 @@
+../../common/playbooks/repository-nexus.yaml
\ No newline at end of file
diff --git a/egi-devel/playbooks/squid.yaml b/egi-devel/playbooks/squid.yaml
new file mode 120000
index 0000000..114c327
--- /dev/null
+++ b/egi-devel/playbooks/squid.yaml
@@ -0,0 +1 @@
+../../common/playbooks/squid.yaml
\ No newline at end of file
diff --git a/egi-devel/playbooks/subtasks b/egi-devel/playbooks/subtasks
new file mode 120000
index 0000000..a6b9b95
--- /dev/null
+++ b/egi-devel/playbooks/subtasks
@@ -0,0 +1 @@
+../../common/playbooks/subtasks
\ No newline at end of file
diff --git a/egi-devel/playbooks/templates/backup.yaml b/egi-devel/playbooks/templates/backup.yaml
new file mode 120000
index 0000000..df474ef
--- /dev/null
+++ b/egi-devel/playbooks/templates/backup.yaml
@@ -0,0 +1 @@
+../../../common/playbooks/templates/backup.yaml
\ No newline at end of file
diff --git a/egi-devel/playbooks/templates/etc/exports b/egi-devel/playbooks/templates/etc/exports
new file mode 120000
index 0000000..a743a02
--- /dev/null
+++ b/egi-devel/playbooks/templates/etc/exports
@@ -0,0 +1 @@
+../../../../common/playbooks/templates/etc/exports.ipv4
\ No newline at end of file
diff --git a/egi-devel/playbooks/templates/etc/mailutils.conf b/egi-devel/playbooks/templates/etc/mailutils.conf
new file mode 120000
index 0000000..dbd8a1f
--- /dev/null
+++ b/egi-devel/playbooks/templates/etc/mailutils.conf
@@ -0,0 +1 @@
+../../../../common/playbooks/templates/etc/mailutils.conf
\ No newline at end of file
diff --git a/egi-devel/playbooks/templates/etc/squid b/egi-devel/playbooks/templates/etc/squid
new file mode 120000
index 0000000..352b598
--- /dev/null
+++ b/egi-devel/playbooks/templates/etc/squid
@@ -0,0 +1 @@
+../../../../common/playbooks/templates/etc/squid
\ No newline at end of file
diff --git a/egi-devel/playbooks/templates/nexus b/egi-devel/playbooks/templates/nexus
new file mode 120000
index 0000000..bc36c2a
--- /dev/null
+++ b/egi-devel/playbooks/templates/nexus
@@ -0,0 +1 @@
+../../../common/playbooks/templates/nexus
\ No newline at end of file
diff --git a/egi-devel/playbooks/templates/nexus.yaml b/egi-devel/playbooks/templates/nexus.yaml
new file mode 120000
index 0000000..ff8f7a8
--- /dev/null
+++ b/egi-devel/playbooks/templates/nexus.yaml
@@ -0,0 +1 @@
+../../../common/playbooks/templates/nexus.yaml
\ No newline at end of file
diff --git a/egi-devel/playbooks/templates/recover.yaml b/egi-devel/playbooks/templates/recover.yaml
new file mode 120000
index 0000000..646a5ae
--- /dev/null
+++ b/egi-devel/playbooks/templates/recover.yaml
@@ -0,0 +1 @@
+../../../common/playbooks/templates/recover.yaml
\ No newline at end of file
diff --git a/egi-devel/playbooks/upgrade.yaml b/egi-devel/playbooks/upgrade.yaml
new file mode 120000
index 0000000..0f9e3f4
--- /dev/null
+++ b/egi-devel/playbooks/upgrade.yaml
@@ -0,0 +1 @@
+../../common/playbooks/upgrade.yaml
\ No newline at end of file
diff --git a/egi-devel/terraform/cloud-init.yaml b/egi-devel/terraform/cloud-init.yaml
new file mode 120000
index 0000000..f315687
--- /dev/null
+++ b/egi-devel/terraform/cloud-init.yaml
@@ -0,0 +1 @@
+../../common/terraform/cloud-init.yaml
\ No newline at end of file
diff --git a/egi-devel/terraform/firewall.tf b/egi-devel/terraform/firewall.tf
new file mode 120000
index 0000000..0088c12
--- /dev/null
+++ b/egi-devel/terraform/firewall.tf
@@ -0,0 +1 @@
+../../common/terraform/firewall.tf
\ No newline at end of file
diff --git a/egi-devel/terraform/terraform.tfvars b/egi-devel/terraform/terraform.tfvars
new file mode 100644
index 0000000..28c4e43
--- /dev/null
+++ b/egi-devel/terraform/terraform.tfvars
@@ -0,0 +1,34 @@
+# These need to be defined for things to work
+ip_pool = "external-ipv4-general-public"
+net_name = "group-project-network"
+net6_name = ""
+site_name = "egi-devel"
+
+# These may need some adjustment for your provider
+master_flavor_name = "e1.small"
+worker_flavor_name = "e1.large"
+gpu_flavor_name = "a3.30core-240ram-nvidia-t4"
+
+# Number of extra workers
+extra_workers = 1
+
+# Number of GPU workers
+gpu_workers = 0
+
+# volumes for docker
+docker_volumes_size = 384
+
+# NFS volume
+nfs_volume_size = 256
+
+# scratch volume
+scratch_volumes_size = 0
+
+# squid volume
+squid_volume_size = 128
+
+# global firewall rules - public and admin access
+security_public_cidr = {
+ "0.0.0.0/0": "Public access",
+ "::/0": "Public access",
+}
diff --git a/egi-devel/terraform/vars.tf b/egi-devel/terraform/vars.tf
new file mode 120000
index 0000000..00c4e3a
--- /dev/null
+++ b/egi-devel/terraform/vars.tf
@@ -0,0 +1 @@
+../../common/terraform/vars.tf
\ No newline at end of file
diff --git a/egi-devel/terraform/versions.tf b/egi-devel/terraform/versions.tf
new file mode 120000
index 0000000..b4eea0e
--- /dev/null
+++ b/egi-devel/terraform/versions.tf
@@ -0,0 +1 @@
+../../common/terraform/versions.tf
\ No newline at end of file
diff --git a/egi-devel/terraform/vms.tf b/egi-devel/terraform/vms.tf
new file mode 100644
index 0000000..fc518b5
--- /dev/null
+++ b/egi-devel/terraform/vms.tf
@@ -0,0 +1,245 @@
+locals {
+ nodes = concat([
+ openstack_compute_instance_v2.ingress,
+ openstack_compute_instance_v2.nfs,
+ ], openstack_compute_instance_v2.worker[*], openstack_compute_instance_v2.gpu[*])
+ master_ip = openstack_compute_instance_v2.master.network[0].fixed_ip_v4
+ ingress_ip = openstack_compute_instance_v2.ingress.network[0].fixed_ip_v4
+ nfs_ip = openstack_compute_instance_v2.nfs.network[0].fixed_ip_v4
+ worker_ips = [for s in openstack_compute_instance_v2.worker[*].network[0].fixed_ip_v4 : s]
+ gpu_ips = [for s in openstack_compute_instance_v2.gpu[*].network[0].fixed_ip_v4 : s]
+}
+
+resource "openstack_networking_floatingip_v2" "public_ip" {
+ pool = var.ip_pool
+}
+
+data "openstack_images_image_v2" "ubuntu" {
+ name = "ubuntu-noble-x86_64"
+}
+
+data "openstack_compute_flavor_v2" "master-flavor" {
+ name = var.master_flavor_name
+}
+
+data "openstack_compute_flavor_v2" "worker-flavor" {
+ name = var.worker_flavor_name
+}
+
+data "openstack_compute_flavor_v2" "gpu-flavor" {
+ name = var.gpu_flavor_name
+}
+
+resource "openstack_compute_instance_v2" "master" {
+ name = "k8s-${var.site_name}-master"
+ image_id = data.openstack_images_image_v2.ubuntu.id
+ flavor_id = data.openstack_compute_flavor_v2.master-flavor.id
+ security_groups = ["default", openstack_networking_secgroup_v2.ping.name, openstack_networking_secgroup_v2.ssh.name]
+ user_data = file("cloud-init.yaml")
+ tags = ["master"]
+ network {
+ name = var.net_name
+ }
+}
+
+resource "openstack_compute_instance_v2" "nfs" {
+ name = "k8s-${var.site_name}-nfs"
+ image_id = data.openstack_images_image_v2.ubuntu.id
+ flavor_id = data.openstack_compute_flavor_v2.worker-flavor.id
+ security_groups = ["default", openstack_networking_secgroup_v2.ping.name, openstack_networking_secgroup_v2.ssh.name]
+ user_data = file("cloud-init.yaml")
+ tags = ["worker"]
+ network {
+ name = var.net_name
+ }
+}
+
+resource "openstack_compute_instance_v2" "ingress" {
+ name = "k8s-${var.site_name}-w-ingress"
+ image_id = data.openstack_images_image_v2.ubuntu.id
+ flavor_id = data.openstack_compute_flavor_v2.worker-flavor.id
+ security_groups = ["default", openstack_networking_secgroup_v2.ping.name, openstack_networking_secgroup_v2.ssh.name, openstack_networking_secgroup_v2.http.name]
+ user_data = file("cloud-init.yaml")
+ tags = ["worker"]
+ network {
+ name = var.net_name
+ }
+}
+
+resource "openstack_compute_instance_v2" "worker" {
+ count = var.extra_workers
+ name = "k8s-${var.site_name}-worker-${count.index}"
+ image_id = data.openstack_images_image_v2.ubuntu.id
+ flavor_id = data.openstack_compute_flavor_v2.worker-flavor.id
+ security_groups = ["default", openstack_networking_secgroup_v2.ping.name, openstack_networking_secgroup_v2.ssh.name]
+ user_data = file("cloud-init.yaml")
+ tags = ["worker"]
+ network {
+ name = var.net_name
+ }
+}
+
+resource "openstack_compute_instance_v2" "gpu" {
+ count = var.gpu_workers
+ name = "k8s-${var.site_name}-gpu-${count.index}"
+ image_id = data.openstack_images_image_v2.ubuntu.id
+ flavor_id = data.openstack_compute_flavor_v2.gpu-flavor.id
+ security_groups = ["default", openstack_networking_secgroup_v2.ping.name, openstack_networking_secgroup_v2.ssh.name]
+ user_data = file("cloud-init.yaml")
+ tags = ["worker"]
+ network {
+ name = var.net_name
+ }
+}
+
+resource "openstack_compute_floatingip_associate_v2" "fip" {
+ floating_ip = openstack_networking_floatingip_v2.public_ip.address
+ instance_id = openstack_compute_instance_v2.ingress.id
+}
+
+resource "openstack_blockstorage_volume_v3" "nfs-volume" {
+ name = "nfs"
+ size = var.nfs_volume_size
+}
+
+resource "openstack_compute_volume_attach_v2" "nfs-volume-attach" {
+ instance_id = openstack_compute_instance_v2.nfs.id
+ volume_id = openstack_blockstorage_volume_v3.nfs-volume.id
+}
+
+resource "local_file" "volume-script" {
+ filename = "nfs-volume.sh"
+ file_permission = "0755"
+ content = <<EOT
+#! /bin/bash -xe
+if ! dpkg-query -s xfsprogs >/dev/null 2>&1; then
+ apt-get update
+ apt-get install -y xfsprogs
+fi
+device="${openstack_compute_volume_attach_v2.nfs-volume-attach.device}"
+mkfs.xfs -L NFS "$device" || true
+grep -q 'LABEL=NFS' /etc/fstab || /bin/echo -e "LABEL=NFS\t/exports\txfs\tdefaults,uquota,pquota\t0\t0" | tee -a /etc/fstab
+mkdir /exports 2>/dev/null || true
+mount -a
+EOT
+}
+
+resource "openstack_blockstorage_volume_v3" "docker-volume" {
+ count = var.extra_workers + var.gpu_workers + 2
+ name = format("docker-%s", local.nodes[count.index].name)
+ size = var.docker_volumes_size
+}
+
+resource "openstack_compute_volume_attach_v2" "docker-volume-attach" {
+ count = var.extra_workers + var.gpu_workers + 2
+ instance_id = local.nodes[count.index].id
+ volume_id = openstack_blockstorage_volume_v3.docker-volume[count.index].id
+}
+
+resource "local_file" "docker-volume-script" {
+ filename = "docker-volume.sh"
+ file_permission = "0755"
+ content = <<EOT
+#! /bin/bash -xe
+volumes="${join("\n", [for n, d in zipmap(tolist(local.nodes[*].name), tolist(openstack_compute_volume_attach_v2.docker-volume-attach[*].device)) : format("%s:%s", n, d)])}"
+volume=$(echo "$volumes" | grep "$(hostname):")
+device=$(echo "$volume" | cut -d: -f2)
+if ! dumpe2fs -h "$device" >/dev/null 2>&1; then
+ mkfs.ext4 -L DOCKER "$device"
+ grep -q 'LABEL=DOCKER' /etc/fstab || /bin/echo -e "LABEL=DOCKER\t/var/lib/docker/overlay2\text4\tdefaults,x-systemd.before=local-fs.target\t0\t0" | tee -a /etc/fstab
+ mkdir -p /var/lib/docker/overlay2 2>/dev/null || true
+ systemctl stop docker kubelet >/dev/null 2>&1 || true
+ sleep 10
+ systemctl stop docker kubelet >/dev/null 2>&1 || true
+ umount /var/lib/docker/overlay2 2>&1 || true
+ mount "$device" /mnt
+ mv /var/lib/docker/overlay2/* /mnt >/dev/null 2>&1 || true
+ umount /mnt
+ mount -a
+ systemctl start docker kubelet >/dev/null 2>&1 || true
+fi
+EOT
+}
+
+resource "openstack_blockstorage_volume_v3" "squid-volume" {
+ name = "squid"
+ size = var.squid_volume_size
+}
+
+resource "openstack_compute_volume_attach_v2" "squid-volume-attach" {
+ instance_id = openstack_compute_instance_v2.ingress.id
+ volume_id = openstack_blockstorage_volume_v3.squid-volume.id
+}
+
+resource "local_file" "squid-volume-script" {
+ filename = "squid-volume.sh"
+ file_permission = "0755"
+ content = <<EOT
+#! /bin/bash -xe
+device="${openstack_compute_volume_attach_v2.squid-volume-attach.device}"
+if ! dumpe2fs -h "$device" >/dev/null 2>&1; then
+ mkfs.ext4 -L SQUID "$device"
+fi
+grep -q 'LABEL=SQUID' /etc/fstab || /bin/echo -e "LABEL=SQUID\t/var/spool/squid\text4\tdefaults,x-systemd.before=local-fs.target\t0\t0" | tee -a /etc/fstab
+mkdir /var/spool/squid 2>/dev/null || true
+mount -a
+EOT
+}
+
+resource "local_file" "inventory" {
+ filename = "inventory.yaml"
+ file_permission = "0644"
+ content = <<EOT
+---
+fip:
+ hosts:
+ ${openstack_networking_floatingip_v2.public_ip.address}:
+
+master:
+ hosts:
+ ${local.master_ip}:
+ # must be IPv4 address or hostname
+ kube_server: ${openstack_compute_instance_v2.master.network[0].fixed_ip_v4}
+
+ingress:
+ hosts:
+ ${local.ingress_ip}:
+
+nfs:
+ hosts:
+ ${local.nfs_ip}:
+
+worker:
+ hosts:
+ ${join("\n ", [for s in local.worker_ips : "${s}:"])}
+
+gpu:
+ hosts:
+ ${join("\n ", [for s in local.gpu_ips : "${s}:"])}
+
+# using public IP of kube_server for ansible delegate_to
+kube_server:
+ hosts:
+ ${openstack_compute_instance_v2.master.network[0].fixed_ip_v4}:
+ ansible_host: ${local.master_ip}
+EOT
+}
+
+resource "local_file" "fip" {
+ filename = "fip.txt"
+ file_permission = "0644"
+ content = <<EOT
+${openstack_networking_floatingip_v2.public_ip.address}
+EOT
+}
+
+resource "local_file" "hosts" {
+ filename = "hosts.txt"
+ file_permission = "0644"
+ content = <<EOT
+${local.master_ip}
+${local.ingress_ip}
+${local.nfs_ip}
+${join("\n", concat(local.worker_ips, local.gpu_ips))}
+EOT
+}
--
GitLab