From 9069fdcd87c2b72c3eec0aa579b8492a334a6ece Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Franti=C5=A1ek=20Dvo=C5=99=C3=A1k?= <valtri@civ.zcu.cz>
Date: Mon, 17 Jun 2024 14:56:13 +0000
Subject: [PATCH] EOSC LOT3 testing deployment based on the demo
---
cesnet-central/deployments/central.yaml | 99 -----------
cesnet-central/deployments/fullhub.yaml | 5 +-
cesnet-central/playbooks/notebooks.yaml | 34 ++--
demo/inventory/1-cesnet.yaml | 32 ----
demo/playbooks/notebooks.yaml | 135 ---------------
staging/deploy.sh | 12 +-
staging/terraform/vms.tf | 10 ++
{demo => testing}/ansible.cfg | 0
{demo => testing}/deploy.sh | 12 +-
{demo => testing}/deployments/hub.yaml | 32 +++-
{demo => testing}/extra | 0
testing/inventory/1-cesnet.yaml | 31 ++++
{demo => testing}/inventory/99-all.yaml | 2 +-
{demo => testing}/playbooks/cvmfs.yaml | 0
{demo => testing}/playbooks/files/calico.yaml | 0
.../playbooks/files/helm_repos.fact | 0
.../playbooks/files/k8s-cheats.sh | 0
.../playbooks/files/xfs-quotas.sh | 0
{demo => testing}/playbooks/k8s.yaml | 0
testing/playbooks/notebooks.yaml | 1 +
{demo => testing}/playbooks/public_keys | 0
{demo => testing}/playbooks/squid.yaml | 0
.../playbooks/templates/etc/exports | 0
.../playbooks/templates/etc/mailutils.conf | 0
.../playbooks/templates/etc/squid | 0
{demo => testing}/terraform/.gitignore | 0
{demo => testing}/terraform/cloud-init.yaml | 0
{demo => testing}/terraform/terraform.tfvars | 4 +-
{demo => testing}/terraform/vars.tf | 0
{demo => testing}/terraform/versions.tf | 3 +
{demo => testing}/terraform/vms.tf | 154 +++++++++++-------
31 files changed, 198 insertions(+), 368 deletions(-)
delete mode 100644 cesnet-central/deployments/central.yaml
delete mode 100644 demo/inventory/1-cesnet.yaml
delete mode 100644 demo/playbooks/notebooks.yaml
rename {demo => testing}/ansible.cfg (100%)
rename {demo => testing}/deploy.sh (85%)
rename {demo => testing}/deployments/hub.yaml (95%)
rename {demo => testing}/extra (100%)
create mode 100644 testing/inventory/1-cesnet.yaml
rename {demo => testing}/inventory/99-all.yaml (91%)
rename {demo => testing}/playbooks/cvmfs.yaml (100%)
rename {demo => testing}/playbooks/files/calico.yaml (100%)
rename {demo => testing}/playbooks/files/helm_repos.fact (100%)
rename {demo => testing}/playbooks/files/k8s-cheats.sh (100%)
rename {demo => testing}/playbooks/files/xfs-quotas.sh (100%)
rename {demo => testing}/playbooks/k8s.yaml (100%)
create mode 120000 testing/playbooks/notebooks.yaml
rename {demo => testing}/playbooks/public_keys (100%)
rename {demo => testing}/playbooks/squid.yaml (100%)
rename {demo => testing}/playbooks/templates/etc/exports (100%)
rename {demo => testing}/playbooks/templates/etc/mailutils.conf (100%)
rename {demo => testing}/playbooks/templates/etc/squid (100%)
rename {demo => testing}/terraform/.gitignore (100%)
rename {demo => testing}/terraform/cloud-init.yaml (100%)
rename {demo => testing}/terraform/terraform.tfvars (94%)
rename {demo => testing}/terraform/vars.tf (100%)
rename {demo => testing}/terraform/versions.tf (88%)
rename {demo => testing}/terraform/vms.tf (72%)
diff --git a/cesnet-central/deployments/central.yaml b/cesnet-central/deployments/central.yaml
deleted file mode 100644
index 2f19796..0000000
--- a/cesnet-central/deployments/central.yaml
+++ /dev/null
@@ -1,99 +0,0 @@
----
-proxy:
- service:
- type: NodePort
-
-ingress:
- enabled: true
- annotations:
- kubernetes.io/ingress.class: "nginx"
- kubernetes.io/tls-acme: "true"
- hosts:
- - eosc.zcu.cz
- tls:
- - hosts:
- - eosc.zcu.cz
- secretName: acme-tls-central
-
-singleuser:
- storage:
- capacity: 20Gi
- dynamic:
- pvcNameTemplate: claim-{userid}{servername}
- volumeNameTemplate: vol-{userid}{servername}
- storageAccessModes: ["ReadWriteMany"]
- extraVolumes:
- - name: cvmfs-host
- hostPath:
- path: /cvmfs
- type: Directory
- extraVolumeMounts:
- - name: cvmfs-host
- mountPath: "/cvmfs:shared"
- lifecycleHooks:
- postStart:
- exec: { "command": ["/bin/sh", "-c", "ln -snf /cvmfs $HOME/cvmfs; mkdir -p /home/jovyan/.notebookCheckpoints"] }
- memory:
- limit: 6G
- guarantee: 128M
- cpu:
- limit: 2
- guarantee: .02
- defaultUrl: "/lab"
- # requires single-user image compatible with enterprise-gateway
- image:
- # https://github.com/valtri/egi-notebooks-images/tree/jupyter-4.x/single-user-gateway
- # (not compatible)
- # name: valtri/single-user-gateway
- # tag: "jupyter-4b"
-
- # name: elyra/nb2kg
- # tag: dev
-
- # hub-4.0.2 julia-1.10.0 lab-4.1.0 notebook-7.0.7 python-3.11.7
- name: quay.io/jupyter/datascience-notebook
- tag: 2024-02-06
- extraEnv:
- KG_REQUEST_TIMEOUT: "60"
- JUPYTER_GATEWAY_AUTH_TOKEN: "{{ gateways_token['cesnet-mcc'] }}"
- JUPYTER_GATEWAY_URL: "https://gateway-cesnet.eosc.zcu.cz"
-
-hub:
- # services:
- # status:
- # url: "http://status-web/"
- # admin: true
- image:
- name: valtri/hub
- tag: "3.x-eosc2" # EOSC, jupyter 3.1.0
- config:
- Authenticator:
- enable_auth_state: true
- admin_users:
- # valtri@civ.zcu.cz
- - c36b18fe-e03a-4a22-ab14-5965e0171410@eosc-federation.eu
- allowed_groups:
- - urn:geant:eosc-federation.eu:testing:group:eosc#testing.eosc-federation.eu
- auto_login: true
- claim_groups_key: "entitlements"
- EGICheckinAuthenticator:
- checkin_host: "{{ secret['checkin_host'] }}"
- authorize_url: "https://{{ secret['checkin_host'] }}/OIDC/authorization"
- token_url: "https://{{ secret['checkin_host'] }}/OIDC/token"
- userdata_url: "https://{{ secret['checkin_host'] }}/OIDC/userinfo"
- client_id: "{{ secret['client_id'] }}"
- client_secret: "{{ secret['client_secret'] }}"
- oauth_callback_url: "https://eosc.zcu.cz/hub/oauth_callback"
- scope: ["openid", "profile", "email", "offline_access", "entitlements"]
- username_key: "sub"
- extra_authorize_params:
- prompt: consent
- JupyterHub:
- admin_access: true
- authenticate_prometheus: false
- authenticator_class: egi_notebooks_hub.egiauthenticator.EGICheckinAuthenticator
- # spawner_class: kubespawner.KubeSpawner
- spawner_class: egi_notebooks_hub.egispawner.EGISpawner
- # c.B2DropSpawner.args = ["--FileCheckpoints.checkpoint_dir='/home/jovyan/.notebookCheckpoints'"]
- templatePaths:
- - /egi-notebooks-hub/templates
diff --git a/cesnet-central/deployments/fullhub.yaml b/cesnet-central/deployments/fullhub.yaml
index 0b925a6..eb82a46 100644
--- a/cesnet-central/deployments/fullhub.yaml
+++ b/cesnet-central/deployments/fullhub.yaml
@@ -54,7 +54,7 @@ singleuser:
# - name: scratch
# mountPath: '/scratch'
memory:
- limit: 6G
+ limit: 4G
guarantee: 128M
cpu:
limit: 2
@@ -395,8 +395,5 @@ hub:
and <a href="https://docs.egi.eu/users/dev-env/notebooks/#notebooks-for-researchers">
enrolling to one of the supported VOs</a>.
</p>
- <p>
- Default environment provides 2 CPU cores, 6 GB RAM and 20GB of personal storage space per user
- </p>
{% endblock main_intro %}
{% endraw %}
diff --git a/cesnet-central/playbooks/notebooks.yaml b/cesnet-central/playbooks/notebooks.yaml
index 8a20d3f..89c48ab 100644
--- a/cesnet-central/playbooks/notebooks.yaml
+++ b/cesnet-central/playbooks/notebooks.yaml
@@ -18,25 +18,14 @@
token_validate=false)}) }}"
with_fileglob:
- "../deployments/*.yaml"
- - name: Get Secrets from Vault for gateway
- set_fact:
- gateways_token: "{{ {item: lookup('community.hashi_vault.hashi_vault', vault_mount_point + '/gateway-' + item + ':authtoken',
- token_validate=false)} }}"
- loop:
- - cesnet-mcc
- # - name: Debug Deployments Secrets
- # debug:
- # msg: "{{ item.key }} = {{ item.value }}"
- # loop: "{{ secrets | dict2items }}"
- # - name: Debug Gateway Secrets
- # debug:
- # msg: "{{ item.key }} = {{ item.value }}"
- # loop: "{{ gateways_token | dict2items }}"
+ - name: Debug Deployments Secrets
+ debug:
+ msg: "{{ item.key }} = {{ item.value }}"
+ loop: "{{ secrets | dict2items }}"
- name: Copy config file to master
vars:
name: "{{ item | basename | splitext | first }}"
secret: "{{ secrets[name] }}"
- gateways_token: "{{ gateways_token }}"
template:
src: "{{ item }}"
dest: "/tmp/{{ item | basename }}"
@@ -129,3 +118,18 @@
with_fileglob:
- "../extra/*.yaml"
when: true
+- hosts: nfs
+ become: true
+ tasks:
+ - name: Quota settings
+ vars:
+ name: "{{ item | basename | splitext | first }}"
+ cron:
+ cron_file: notebook-quotas
+ name: "{{ name }} quotas"
+ minute: "0"
+ hour: "*/2"
+ job: "/usr/local/bin/xfs-quotas.sh --include ^/exports/{{ name }}- --exclude ^/exports/{{ name }}-hub-db-dir-"
+ user: root
+ with_fileglob:
+ - "../deployments/*.yaml"
diff --git a/demo/inventory/1-cesnet.yaml b/demo/inventory/1-cesnet.yaml
deleted file mode 100644
index 7706101..0000000
--- a/demo/inventory/1-cesnet.yaml
+++ /dev/null
@@ -1,32 +0,0 @@
----
-fip:
- hosts:
- 147.251.124.105:
-
-master:
- hosts:
- 2001:718:801:432:f816:3eff:fe33:7a22:
- # must be IPv4 address or hostname
- kube_server: 192.168.0.231
-
-ingress:
- hosts:
- 2001:718:801:432:f816:3eff:fece:3b38:
-
-nfs:
- hosts:
- 2001:718:801:432:f816:3eff:fe97:4bbd:
-
-worker:
- hosts:
- 2001:718:801:432:f816:3eff:fe82:628a:
-
-gpu:
- hosts:
- 2001:718:801:432:f816:3eff:fe4e:c5ab:
-
-# using public IP of kube_server for ansible delegate_to
-kube_server:
- hosts:
- 192.168.0.231:
- ansible_host: 2001:718:801:432:f816:3eff:fe33:7a22
diff --git a/demo/playbooks/notebooks.yaml b/demo/playbooks/notebooks.yaml
deleted file mode 100644
index 89c48ab..0000000
--- a/demo/playbooks/notebooks.yaml
+++ /dev/null
@@ -1,135 +0,0 @@
----
-- name: Notebooks deployments
- hosts: master
- become: true
- tasks:
- - name: Configure helm repo
- shell: |-
- helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/
- helm repo add eginotebooks https://egi-federation.github.io/egi-notebooks-chart/
- helm repo update
- when: "'jupyterhub' not in ansible_local.helm_repos | map(attribute='name') | list or
- 'eginotebooks' not in ansible_local.helm_repos | map(attribute='name') | list"
- - name: Get Secrets from Vault for notebooks
- vars:
- name: "{{ item | basename | splitext | first }}"
- set_fact:
- secrets: "{{ secrets|default({}) | combine({name: lookup('community.hashi_vault.hashi_vault', vault_mount_point + '/deployment-' + name,
- token_validate=false)}) }}"
- with_fileglob:
- - "../deployments/*.yaml"
- - name: Debug Deployments Secrets
- debug:
- msg: "{{ item.key }} = {{ item.value }}"
- loop: "{{ secrets | dict2items }}"
- - name: Copy config file to master
- vars:
- name: "{{ item | basename | splitext | first }}"
- secret: "{{ secrets[name] }}"
- template:
- src: "{{ item }}"
- dest: "/tmp/{{ item | basename }}"
- mode: 0600
- with_fileglob:
- - "../deployments/*.yaml"
- - name: Deploy/upgrade notebook instance
- vars:
- name: "{{ item | basename | splitext | first }}"
- version: "3.2.1" # app 4.0.2 (2023-11-27)
- monitor_version: "0.3.1"
- shell: |-
- helm status --namespace {{ name }} {{ name }}
- if [ $? -ne 0 ]; then
- helm install --create-namespace --namespace {{ name }} \
- -f /tmp/{{ item | basename }} --version {{ version }} --timeout 2h \
- {{ name }} jupyterhub/jupyterhub
- else
- helm upgrade --version {{ version }} -f /tmp/{{ item | basename }} --timeout 2h \
- --namespace {{ name }} {{ name }} jupyterhub/jupyterhub
- fi
- helm status --namespace {{ name }} {{ name }}-monitor
- if [ $? -ne 0 ]; then
- helm install --namespace {{ name }} \
- -f /tmp/{{ item | basename }} --version {{ monitor_version }} \
- {{ name }}-monitor eginotebooks/notebooks-monitor
- else
- helm upgrade --version {{ monitor_version }} \
- -f /tmp/{{ item | basename }} --namespace {{ name }} \
- {{ name }}-monitor eginotebooks/notebooks-monitor
- fi
- environment:
- KUBECONFIG: /etc/kubernetes/admin.conf
- PATH: /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin
- when: true
- with_fileglob:
- - "../deployments/*.yaml"
-
- - name: Configure secrets management for the hub
- vars:
- name: "{{ item | basename | splitext | first }}"
- shell: |-
- kubectl apply -f - << EOF
- ---
- kind: Role
- apiVersion: rbac.authorization.k8s.io/v1
- metadata:
- name: hub-secrets
- namespace: {{ name }}
- rules:
- - apiGroups: [""] # "" indicates the core API group
- resources: ["secrets"]
- verbs: ["get", "watch", "list", "create", "delete", "patch", "update"]
- ---
- kind: RoleBinding
- apiVersion: rbac.authorization.k8s.io/v1
- metadata:
- name: hub-secrets
- namespace: {{ name }}
- subjects:
- - kind: ServiceAccount
- name: hub
- namespace: {{ name }}
- roleRef:
- kind: Role
- name: hub-secrets
- apiGroup: rbac.authorization.k8s.io
- EOF
- environment:
- KUBECONFIG: /etc/kubernetes/admin.conf
- PATH: /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin
- when: true
- with_fileglob:
- - "../deployments/*.yaml"
- # do the extra bits of configuration
- # here we should have all the namespaces, pre-requirements in place
- # XXX: this won't remove things that are delete from the directory
- - name: Copy extra configuration files
- copy:
- src: "{{ item }}"
- dest: "/tmp/{{ item | basename }}"
- mode: 0600
- with_fileglob:
- - "../extra/*.yaml"
- - name: Extra configuration
- command: |-
- kubectl apply -f /tmp/{{ item | basename }}
- environment:
- KUBECONFIG: /etc/kubernetes/admin.conf
- with_fileglob:
- - "../extra/*.yaml"
- when: true
-- hosts: nfs
- become: true
- tasks:
- - name: Quota settings
- vars:
- name: "{{ item | basename | splitext | first }}"
- cron:
- cron_file: notebook-quotas
- name: "{{ name }} quotas"
- minute: "0"
- hour: "*/2"
- job: "/usr/local/bin/xfs-quotas.sh --include ^/exports/{{ name }}- --exclude ^/exports/{{ name }}-hub-db-dir-"
- user: root
- with_fileglob:
- - "../deployments/*.yaml"
diff --git a/staging/deploy.sh b/staging/deploy.sh
index aeeb651..b02846d 100755
--- a/staging/deploy.sh
+++ b/staging/deploy.sh
@@ -46,10 +46,10 @@ ansible -m command -a '/root/squid-volume.sh' 'ingress[0]'
ansible-playbook playbooks/k8s.yaml
while ansible -i ./inventory -m command -a 'kubectl get pods --all-namespaces' master | tail -n +3 | grep -v ' Running '; do sleep 5; done
# docker runtime directory after Kubernetes deployment (problem with unmounts)
-ansible -m copy -a 'src=terraform/docker-volume.sh dest=/root/ mode=preserve' 'ingress nfs worker'
-ansible -m command -a '/root/docker-volume.sh' 'ingress nfs worker'
+ansible -m copy -a 'src=terraform/docker-volume.sh dest=/root/ mode=preserve' 'ingress nfs worker gpu'
+ansible -m command -a '/root/docker-volume.sh' 'ingress nfs worker gpu'
+ansible-playbook playbooks/squid.yaml
+ansible-playbook playbooks/cvmfs.yaml
-# ansible-playbook playbooks/squid.yaml
-# ansible-playbook playbooks/cvmfs.yaml
-
-while ansible -i ./inventory -m command -a 'kubectl get pods --all-namespaces' master | tail -n +3 | grep -v ' Running '; do sleep 5; done
+# wait for finish
+while ansible -m command -a 'kubectl get pods --all-namespaces' master | tail -n +3 | grep -v ' Running '; do sleep 5; done
diff --git a/staging/terraform/vms.tf b/staging/terraform/vms.tf
index c3a30e0..c3db9ec 100644
--- a/staging/terraform/vms.tf
+++ b/staging/terraform/vms.tf
@@ -37,6 +37,16 @@ resource "openstack_networking_secgroup_rule_v2" "ping4" {
security_group_id = openstack_networking_secgroup_v2.ping.id
}
+resource "openstack_networking_secgroup_rule_v2" "ping6" {
+ direction = "ingress"
+ ethertype = "IPv6"
+ port_range_min = 128
+ port_range_max = 0
+ protocol = "icmp"
+ remote_ip_prefix = "::/0"
+ security_group_id = openstack_networking_secgroup_v2.ping.id
+}
+
resource "openstack_networking_secgroup_rule_v2" "ssh4" {
direction = "ingress"
ethertype = "IPv4"
diff --git a/demo/ansible.cfg b/testing/ansible.cfg
similarity index 100%
rename from demo/ansible.cfg
rename to testing/ansible.cfg
diff --git a/demo/deploy.sh b/testing/deploy.sh
similarity index 85%
rename from demo/deploy.sh
rename to testing/deploy.sh
index d1f60bd..b44f552 100755
--- a/demo/deploy.sh
+++ b/testing/deploy.sh
@@ -1,7 +1,7 @@
#! /bin/bash -xe
#
-# EOSC LOT3 Jupyter Notebooks Demo
+# EOSC LOT3 Jupyter Notebooks testing instance
#
cd terraform && terraform init && terraform apply
@@ -13,7 +13,7 @@ ip="$(head -n 1 < terraform/fip.txt)"
shellstate=$(shopt -po xtrace)
set +o xtrace
# https://nsupdate.fedcloud.eu
-vault_prefix=secrets/users/e1662e20-e34b-468c-b0ce-d899bc878364@egi.eu/eosc-demo
+vault_prefix=secrets/users/e1662e20-e34b-468c-b0ce-d899bc878364@egi.eu/eosc-testing
FEDCLOUD_DYNAMIC_DNS=$(vault read -field data $vault_prefix/FEDCLOUD_DYNAMIC_DNS | grep ^map | head -n 1 | sed 's/map\[\(.*\)\]/\1/')
for auth in $FEDCLOUD_DYNAMIC_DNS; do
echo "curl -i -X GET -u $(echo "$auth" | cut -d: -f1):XXX https://nsupdate.fedcloud.eu/nic/update?myip=$ip"
@@ -22,7 +22,7 @@ done
eval "$shellstate"
# wait for ping and ssh
-for ip in $(cat terraform/hosts.txt); do
+for ip in $(cat terraform/hosts.txt) $(cat terraform/fip.txt); do
while ! ping -c 1 "$ip"; do sleep 5; done
ssh-keygen -R "$ip"
while ! ssh egi@"$ip" -o ConnectTimeout=10 -o PreferredAuthentications=publickey -o StrictHostKeyChecking=no :; do sleep 10; done
@@ -39,13 +39,15 @@ ansible -m copy -a 'src=terraform/nfs-volume.sh dest=/root/ mode=preserve' nfs
ansible -m command -a '/root/nfs-volume.sh' nfs
ansible -m copy -a 'src=terraform/squid-volume.sh dest=/root/ mode=preserve' 'ingress[0]'
ansible -m command -a '/root/squid-volume.sh' 'ingress[0]'
-ansible -m copy -a 'src=terraform/docker-volume.sh dest=/root/ mode=preserve' 'ingress nfs worker gpu'
-ansible -m command -a '/root/docker-volume.sh' 'ingress nfs worker gpu'
ansible -m copy -a 'src=terraform/scratch-volume.sh dest=/root/ mode=preserve' 'ingress nfs worker gpu'
ansible -m command -a '/root/scratch-volume.sh' 'ingress nfs worker gpu'
# k8s + notebooks
ansible-playbook playbooks/k8s.yaml
+while ansible -i ./inventory -m command -a 'kubectl get pods --all-namespaces' master | tail -n +3 | grep -v ' Running '; do sleep 5; done
+# docker runtime directory after Kubernetes deployment (problem with unmounts)
+ansible -m copy -a 'src=terraform/docker-volume.sh dest=/root/ mode=preserve' 'ingress nfs worker gpu'
+ansible -m command -a '/root/docker-volume.sh' 'ingress nfs worker gpu'
ansible-playbook playbooks/squid.yaml
ansible-playbook playbooks/cvmfs.yaml
diff --git a/demo/deployments/hub.yaml b/testing/deployments/hub.yaml
similarity index 95%
rename from demo/deployments/hub.yaml
rename to testing/deployments/hub.yaml
index a1b7fe2..3cb326c 100644
--- a/demo/deployments/hub.yaml
+++ b/testing/deployments/hub.yaml
@@ -17,8 +17,6 @@ ingress:
singleuser:
# keep resource limits in sync with:
- # - documentation/content/en/users/dev-env/notebooks/_index.md
- # - documentation/content/en/users/dev-env/notebooks/data/_index.md
# - profileList
storage:
capacity: 20Gi
@@ -102,6 +100,31 @@ singleuser:
nvidia.com/gpu: 1
extra_resource_limits:
nvidia.com/gpu: 1
+ cmd: jupyterhub-singleuser-webdav-wrapper
+ extraFiles:
+ wait-owncloud.sh:
+ mode: 0755
+ mountPath: /usr/local/bin/jupyterhub-singleuser-webdav-wrapper
+ stringData: |-
+ #! /bin/sh
+
+ #
+ # Dirty hack to make remote mount on home directory working properly:
+ #
+ # 1) wait for webdav sidecar image to kick in
+ # 2) change directory to the mounted version of itself
+ # 3) launch notebook server
+ #
+ i=0
+ while ! grep '^webdav-fs: /mnt/owncloud ' /proc/mounts && test $i -lt 30; do
+ echo 'Waiting for ownClound mount...'
+ sleep 0.5
+ i=$((i+1))
+ done
+
+ cd .
+
+ exec jupyterhub-singleuser "$@"
hub:
services:
@@ -110,7 +133,7 @@ hub:
admin: true
image:
name: valtri/hub
- tag: "3.x-eosc4" # EOSC, jupyter 3.1.0
+ tag: "eosc7-jwt"
config:
Authenticator:
enable_auth_state: true
@@ -393,8 +416,5 @@ hub:
and <a href="https://docs.egi.eu/users/dev-env/notebooks/#notebooks-for-researchers">
enrolling to one of the supported VOs</a>.
</p>
- <p>
- Default environment provides 2 CPU cores, 4 GB RAM and 20GB of personal storage space per user
- </p>
{% endblock main_intro %}
{% endraw %}
diff --git a/demo/extra b/testing/extra
similarity index 100%
rename from demo/extra
rename to testing/extra
diff --git a/testing/inventory/1-cesnet.yaml b/testing/inventory/1-cesnet.yaml
new file mode 100644
index 0000000..8cbf53f
--- /dev/null
+++ b/testing/inventory/1-cesnet.yaml
@@ -0,0 +1,31 @@
+---
+fip:
+ hosts:
+ 147.251.124.105:
+
+master:
+ hosts:
+ 2001:718:801:432:f816:3eff:fe36:5620:
+ # must be IPv4 address or hostname
+ kube_server: 192.168.0.115
+
+ingress:
+ hosts:
+ 2001:718:801:432:f816:3eff:fe41:a4b4:
+
+nfs:
+ hosts:
+ 2001:718:801:432:f816:3eff:fee1:9385:
+
+worker:
+ hosts:
+ 2001:718:801:432:f816:3eff:fef7:c707:
+
+gpu:
+ hosts:
+
+# using public IP of kube_server for ansible delegate_to
+kube_server:
+ hosts:
+ 192.168.0.115:
+ ansible_host: 2001:718:801:432:f816:3eff:fe36:5620
diff --git a/demo/inventory/99-all.yaml b/testing/inventory/99-all.yaml
similarity index 91%
rename from demo/inventory/99-all.yaml
rename to testing/inventory/99-all.yaml
index b35f043..f93b09e 100644
--- a/demo/inventory/99-all.yaml
+++ b/testing/inventory/99-all.yaml
@@ -12,7 +12,7 @@ all:
ansible_become: yes
ansible_user: egi
site_name: cesnet
- vault_mount_point: secrets/users/e1662e20-e34b-468c-b0ce-d899bc878364@egi.eu/eosc-demo
+ vault_mount_point: secrets/users/e1662e20-e34b-468c-b0ce-d899bc878364@egi.eu/eosc-testing
notebooks_hostname: eosc-notebooks.vm.fedcloud.eu
grafana_hostname: grafana-eosc-notebooks.vm.fedcloud.eu
diff --git a/demo/playbooks/cvmfs.yaml b/testing/playbooks/cvmfs.yaml
similarity index 100%
rename from demo/playbooks/cvmfs.yaml
rename to testing/playbooks/cvmfs.yaml
diff --git a/demo/playbooks/files/calico.yaml b/testing/playbooks/files/calico.yaml
similarity index 100%
rename from demo/playbooks/files/calico.yaml
rename to testing/playbooks/files/calico.yaml
diff --git a/demo/playbooks/files/helm_repos.fact b/testing/playbooks/files/helm_repos.fact
similarity index 100%
rename from demo/playbooks/files/helm_repos.fact
rename to testing/playbooks/files/helm_repos.fact
diff --git a/demo/playbooks/files/k8s-cheats.sh b/testing/playbooks/files/k8s-cheats.sh
similarity index 100%
rename from demo/playbooks/files/k8s-cheats.sh
rename to testing/playbooks/files/k8s-cheats.sh
diff --git a/demo/playbooks/files/xfs-quotas.sh b/testing/playbooks/files/xfs-quotas.sh
similarity index 100%
rename from demo/playbooks/files/xfs-quotas.sh
rename to testing/playbooks/files/xfs-quotas.sh
diff --git a/demo/playbooks/k8s.yaml b/testing/playbooks/k8s.yaml
similarity index 100%
rename from demo/playbooks/k8s.yaml
rename to testing/playbooks/k8s.yaml
diff --git a/testing/playbooks/notebooks.yaml b/testing/playbooks/notebooks.yaml
new file mode 120000
index 0000000..7c6d7a3
--- /dev/null
+++ b/testing/playbooks/notebooks.yaml
@@ -0,0 +1 @@
+../../cesnet-central/playbooks/notebooks.yaml
\ No newline at end of file
diff --git a/demo/playbooks/public_keys b/testing/playbooks/public_keys
similarity index 100%
rename from demo/playbooks/public_keys
rename to testing/playbooks/public_keys
diff --git a/demo/playbooks/squid.yaml b/testing/playbooks/squid.yaml
similarity index 100%
rename from demo/playbooks/squid.yaml
rename to testing/playbooks/squid.yaml
diff --git a/demo/playbooks/templates/etc/exports b/testing/playbooks/templates/etc/exports
similarity index 100%
rename from demo/playbooks/templates/etc/exports
rename to testing/playbooks/templates/etc/exports
diff --git a/demo/playbooks/templates/etc/mailutils.conf b/testing/playbooks/templates/etc/mailutils.conf
similarity index 100%
rename from demo/playbooks/templates/etc/mailutils.conf
rename to testing/playbooks/templates/etc/mailutils.conf
diff --git a/demo/playbooks/templates/etc/squid b/testing/playbooks/templates/etc/squid
similarity index 100%
rename from demo/playbooks/templates/etc/squid
rename to testing/playbooks/templates/etc/squid
diff --git a/demo/terraform/.gitignore b/testing/terraform/.gitignore
similarity index 100%
rename from demo/terraform/.gitignore
rename to testing/terraform/.gitignore
diff --git a/demo/terraform/cloud-init.yaml b/testing/terraform/cloud-init.yaml
similarity index 100%
rename from demo/terraform/cloud-init.yaml
rename to testing/terraform/cloud-init.yaml
diff --git a/demo/terraform/terraform.tfvars b/testing/terraform/terraform.tfvars
similarity index 94%
rename from demo/terraform/terraform.tfvars
rename to testing/terraform/terraform.tfvars
index c2231f2..6932d36 100644
--- a/demo/terraform/terraform.tfvars
+++ b/testing/terraform/terraform.tfvars
@@ -2,7 +2,7 @@
ip_pool = "public-muni-147-251-124-GROUP"
net_name = "group-project-network"
net6_name = "public-muni-v6-432"
-site_name = "demo"
+site_name = "testing"
# These may need some adjustment for your provider
master_cpus = 2 # 2 CPUs to match existing flavours
@@ -15,7 +15,7 @@ gpu_flavor_name = "a3.32core-240ram-1t4"
extra_workers = 1
# Number of GPU workers
-gpu_workers = 1
+gpu_workers = 0
# volumes for docker
docker_volumes_size = 384
diff --git a/demo/terraform/vars.tf b/testing/terraform/vars.tf
similarity index 100%
rename from demo/terraform/vars.tf
rename to testing/terraform/vars.tf
diff --git a/demo/terraform/versions.tf b/testing/terraform/versions.tf
similarity index 88%
rename from demo/terraform/versions.tf
rename to testing/terraform/versions.tf
index 43054dd..ff6f75b 100644
--- a/demo/terraform/versions.tf
+++ b/testing/terraform/versions.tf
@@ -1,3 +1,6 @@
+provider "openstack" {
+}
+
terraform {
required_providers {
local = "~> 2.0"
diff --git a/demo/terraform/vms.tf b/testing/terraform/vms.tf
similarity index 72%
rename from demo/terraform/vms.tf
rename to testing/terraform/vms.tf
index 4a32e2f..3628212 100644
--- a/demo/terraform/vms.tf
+++ b/testing/terraform/vms.tf
@@ -1,6 +1,3 @@
-provider "openstack" {
-}
-
locals {
nodes = concat([
openstack_compute_instance_v2.ingress,
@@ -15,71 +12,99 @@ locals {
# Security groups
-resource "openstack_compute_secgroup_v2" "ping" {
+resource "openstack_networking_secgroup_v2" "ping" {
name = "ping"
description = "ICMP for ping"
-
- rule {
- from_port = 8
- to_port = 0
- ip_protocol = "icmp"
- cidr = "0.0.0.0/0"
- }
- rule {
- from_port = 128
- to_port = 0
- # initial installation (bug in terraform): ip_protocol = "icmp"
- ip_protocol = "ipv6-icmp"
- cidr = "::/0"
- }
}
-resource "openstack_compute_secgroup_v2" "ssh" {
+resource "openstack_networking_secgroup_v2" "ssh" {
name = "ssh"
description = "ssh connection"
-
- rule {
- from_port = 22
- to_port = 22
- ip_protocol = "tcp"
- cidr = "0.0.0.0/0"
- }
- rule {
- from_port = 22
- to_port = 22
- ip_protocol = "tcp"
- cidr = "::/0"
- }
}
-resource "openstack_compute_secgroup_v2" "http" {
+resource "openstack_networking_secgroup_v2" "http" {
name = "http"
description = "http/https"
+}
- rule {
- from_port = 80
- to_port = 80
- ip_protocol = "tcp"
- cidr = "0.0.0.0/0"
- }
- rule {
- from_port = 80
- to_port = 80
- ip_protocol = "tcp"
- cidr = "::/0"
- }
- rule {
- from_port = 443
- to_port = 443
- ip_protocol = "tcp"
- cidr = "0.0.0.0/0"
- }
- rule {
- from_port = 443
- to_port = 443
- ip_protocol = "tcp"
- cidr = "::/0"
- }
+resource "openstack_networking_secgroup_rule_v2" "ping4" {
+ direction = "ingress"
+ ethertype = "IPv4"
+ port_range_min = 8
+ port_range_max = 0
+ protocol = "icmp"
+ remote_ip_prefix = "0.0.0.0/0"
+ security_group_id = openstack_networking_secgroup_v2.ping.id
+}
+
+resource "openstack_networking_secgroup_rule_v2" "ping6" {
+ direction = "ingress"
+ ethertype = "IPv6"
+ port_range_min = 128
+ port_range_max = 0
+ protocol = "icmp"
+ remote_ip_prefix = "::/0"
+ security_group_id = openstack_networking_secgroup_v2.ping.id
+}
+
+resource "openstack_networking_secgroup_rule_v2" "ssh4" {
+ direction = "ingress"
+ ethertype = "IPv4"
+ port_range_min = 22
+ port_range_max = 22
+ protocol = "tcp"
+ remote_ip_prefix = "0.0.0.0/0"
+ security_group_id = openstack_networking_secgroup_v2.ssh.id
+}
+
+resource "openstack_networking_secgroup_rule_v2" "ssh6" {
+ direction = "ingress"
+ ethertype = "IPv6"
+ port_range_min = 22
+ port_range_max = 22
+ protocol = "tcp"
+ remote_ip_prefix = "::/0"
+ security_group_id = openstack_networking_secgroup_v2.ssh.id
+}
+
+resource "openstack_networking_secgroup_rule_v2" "http4" {
+ direction = "ingress"
+ ethertype = "IPv4"
+ port_range_min = 80
+ port_range_max = 80
+ protocol = "tcp"
+ remote_ip_prefix = "0.0.0.0/0"
+ security_group_id = openstack_networking_secgroup_v2.http.id
+}
+
+resource "openstack_networking_secgroup_rule_v2" "http6" {
+ direction = "ingress"
+ ethertype = "IPv6"
+ port_range_min = 80
+ port_range_max = 80
+ protocol = "tcp"
+ remote_ip_prefix = "::/0"
+ security_group_id = openstack_networking_secgroup_v2.http.id
+}
+
+resource "openstack_networking_secgroup_rule_v2" "https4" {
+ direction = "ingress"
+ ethertype = "IPv4"
+ port_range_min = 443
+ port_range_max = 443
+ protocol = "tcp"
+ remote_ip_prefix = "0.0.0.0/0"
+ security_group_id = openstack_networking_secgroup_v2.http.id
+}
+
+resource "openstack_networking_secgroup_rule_v2" "https6" {
+ direction = "ingress"
+ ethertype = "IPv6"
+ port_range_min = 443
+ port_range_max = 443
+ protocol = "tcp"
+ remote_ip_prefix = "::/0"
+ security_group_id = openstack_networking_secgroup_v2.http.id
}
resource "openstack_networking_floatingip_v2" "public_ip" {
@@ -108,7 +133,7 @@ resource "openstack_compute_instance_v2" "master" {
name = "k8s-${var.site_name}-master"
image_id = data.openstack_images_image_v2.ubuntu.id
flavor_id = data.openstack_compute_flavor_v2.master-flavor.id
- security_groups = ["default", openstack_compute_secgroup_v2.ping.name, openstack_compute_secgroup_v2.ssh.name]
+ security_groups = ["default", openstack_networking_secgroup_v2.ping.name, openstack_networking_secgroup_v2.ssh.name]
user_data = file("cloud-init.yaml")
tags = ["master"]
network {
@@ -123,7 +148,7 @@ resource "openstack_compute_instance_v2" "nfs" {
name = "k8s-${var.site_name}-nfs"
image_id = data.openstack_images_image_v2.ubuntu.id
flavor_id = data.openstack_compute_flavor_v2.worker-flavor.id
- security_groups = ["default", openstack_compute_secgroup_v2.ping.name, openstack_compute_secgroup_v2.ssh.name]
+ security_groups = ["default", openstack_networking_secgroup_v2.ping.name, openstack_networking_secgroup_v2.ssh.name]
user_data = file("cloud-init.yaml")
tags = ["worker"]
network {
@@ -138,7 +163,7 @@ resource "openstack_compute_instance_v2" "ingress" {
name = "k8s-${var.site_name}-w-ingress"
image_id = data.openstack_images_image_v2.ubuntu.id
flavor_id = data.openstack_compute_flavor_v2.worker-flavor.id
- security_groups = ["default", openstack_compute_secgroup_v2.ping.name, openstack_compute_secgroup_v2.ssh.name, openstack_compute_secgroup_v2.http.name]
+ security_groups = ["default", openstack_networking_secgroup_v2.ping.name, openstack_networking_secgroup_v2.ssh.name, openstack_networking_secgroup_v2.http.name]
user_data = file("cloud-init.yaml")
tags = ["worker"]
network {
@@ -154,7 +179,7 @@ resource "openstack_compute_instance_v2" "worker" {
name = "k8s-${var.site_name}-worker-${count.index}"
image_id = data.openstack_images_image_v2.ubuntu.id
flavor_id = data.openstack_compute_flavor_v2.worker-flavor.id
- security_groups = ["default", openstack_compute_secgroup_v2.ping.name, openstack_compute_secgroup_v2.ssh.name]
+ security_groups = ["default", openstack_networking_secgroup_v2.ping.name, openstack_networking_secgroup_v2.ssh.name]
user_data = file("cloud-init.yaml")
tags = ["worker"]
network {
@@ -170,7 +195,7 @@ resource "openstack_compute_instance_v2" "gpu" {
name = "k8s-${var.site_name}-gpu-${count.index}"
image_id = data.openstack_images_image_v2.ubuntu.id
flavor_id = data.openstack_compute_flavor_v2.gpu-flavor.id
- security_groups = ["default", openstack_compute_secgroup_v2.ping.name, openstack_compute_secgroup_v2.ssh.name]
+ security_groups = ["default", openstack_networking_secgroup_v2.ping.name, openstack_networking_secgroup_v2.ssh.name]
user_data = file("cloud-init.yaml")
tags = ["worker"]
network {
@@ -237,12 +262,15 @@ if ! dumpe2fs -h "$device" >/dev/null 2>&1; then
mkfs.ext4 -L DOCKER "$device"
grep -q 'LABEL=DOCKER' /etc/fstab || /bin/echo -e "LABEL=DOCKER\t/var/lib/docker/overlay2\text4\tdefaults,x-systemd.before=local-fs.target\t0\t0" | tee -a /etc/fstab
mkdir -p /var/lib/docker/overlay2 2>/dev/null || true
- service docker stop >/dev/null 2>&1 || true
+ systemctl stop docker kubelet >/dev/null 2>&1 || true
sleep 10
+ systemctl stop docker kubelet >/dev/null 2>&1 || true
+ umount /var/lib/docker/overlay2 2>&1 || true
mount "$device" /mnt
mv /var/lib/docker/overlay2/* /mnt >/dev/null 2>&1 || true
umount /mnt
mount -a
+ systemctl start docker kubelet >/dev/null 2>&1 || true
fi
EOT
}
--
GitLab