EOSC LOT3 testing deployment based on the demo

cesnet-central/deployments/central.yaml

----
-proxy:
-  service:
-    type: NodePort
-
-ingress:
-  enabled: true
-  annotations:
-    kubernetes.io/ingress.class: "nginx"
-    kubernetes.io/tls-acme: "true"
-  hosts:
-    - eosc.zcu.cz
-  tls:
-    - hosts:
-        - eosc.zcu.cz
-      secretName: acme-tls-central
-
-singleuser:
-  storage:
-    capacity: 20Gi
-    dynamic:
-      pvcNameTemplate: claim-{userid}{servername}
-      volumeNameTemplate: vol-{userid}{servername}
-      storageAccessModes: ["ReadWriteMany"]
-    extraVolumes:
-      - name: cvmfs-host
-        hostPath:
-          path: /cvmfs
-          type: Directory
-    extraVolumeMounts:
-      - name: cvmfs-host
-        mountPath: "/cvmfs:shared"
-  lifecycleHooks:
-    postStart:
-      exec: { "command": ["/bin/sh", "-c", "ln -snf /cvmfs $HOME/cvmfs; mkdir -p /home/jovyan/.notebookCheckpoints"] }
-  memory:
-    limit: 6G
-    guarantee: 128M
-  cpu:
-    limit: 2
-    guarantee: .02
-  defaultUrl: "/lab"
-  # requires single-user image compatible with enterprise-gateway
-  image:
-    # https://github.com/valtri/egi-notebooks-images/tree/jupyter-4.x/single-user-gateway
-    # (not compatible)
-    # name: valtri/single-user-gateway
-    # tag: "jupyter-4b"
-
-    # name: elyra/nb2kg
-    # tag: dev
-
-    # hub-4.0.2 julia-1.10.0 lab-4.1.0 notebook-7.0.7 python-3.11.7
-    name: quay.io/jupyter/datascience-notebook
-    tag: 2024-02-06
-  extraEnv:
-    KG_REQUEST_TIMEOUT: "60"
-    JUPYTER_GATEWAY_AUTH_TOKEN: "{{ gateways_token['cesnet-mcc'] }}"
-    JUPYTER_GATEWAY_URL: "https://gateway-cesnet.eosc.zcu.cz"
-
-hub:
-  # services:
-  #   status:
-  #     url: "http://status-web/"
-  #     admin: true
-  image:
-    name: valtri/hub
-    tag: "3.x-eosc2" # EOSC, jupyter 3.1.0
-  config:
-    Authenticator:
-      enable_auth_state: true
-      admin_users:
-        # valtri@civ.zcu.cz
-        - c36b18fe-e03a-4a22-ab14-5965e0171410@eosc-federation.eu
-      allowed_groups:
-        - urn:geant:eosc-federation.eu:testing:group:eosc#testing.eosc-federation.eu
-      auto_login: true
-      claim_groups_key: "entitlements"
-    EGICheckinAuthenticator:
-      checkin_host: "{{ secret['checkin_host'] }}"
-      authorize_url: "https://{{ secret['checkin_host'] }}/OIDC/authorization"
-      token_url: "https://{{ secret['checkin_host'] }}/OIDC/token"
-      userdata_url: "https://{{ secret['checkin_host'] }}/OIDC/userinfo"
-      client_id: "{{ secret['client_id'] }}"
-      client_secret: "{{ secret['client_secret'] }}"
-      oauth_callback_url: "https://eosc.zcu.cz/hub/oauth_callback"
-      scope: ["openid", "profile", "email", "offline_access", "entitlements"]
-      username_key: "sub"
-      extra_authorize_params:
-        prompt: consent
-    JupyterHub:
-      admin_access: true
-      authenticate_prometheus: false
-      authenticator_class: egi_notebooks_hub.egiauthenticator.EGICheckinAuthenticator
-      # spawner_class: kubespawner.KubeSpawner
-      spawner_class: egi_notebooks_hub.egispawner.EGISpawner
-      # c.B2DropSpawner.args = ["--FileCheckpoints.checkpoint_dir='/home/jovyan/.notebookCheckpoints'"]
-  templatePaths:
-    - /egi-notebooks-hub/templates

cesnet-central/deployments/fullhub.yaml

@@ -54,7 +54,7 @@ singleuser:
       # - name: scratch
       #   mountPath: '/scratch'
   memory:
-    limit: 6G
+    limit: 4G
     guarantee: 128M
   cpu:
     limit: 2
@@ -395,8 +395,5 @@ hub:
         and <a href="https://docs.egi.eu/users/dev-env/notebooks/#notebooks-for-researchers">
         enrolling to one of the supported VOs</a>.
         </p>
-        <p>
-        Default environment provides 2 CPU cores, 6 GB RAM and 20GB of personal storage space per user
-        </p>
         {% endblock main_intro %}
 {% endraw %}

cesnet-central/playbooks/notebooks.yaml

@@ -18,25 +18,14 @@
           token_validate=false)}) }}"
       with_fileglob:
         - "../deployments/*.yaml"
-    - name: Get Secrets from Vault for gateway
-      set_fact:
-        gateways_token: "{{ {item: lookup('community.hashi_vault.hashi_vault', vault_mount_point + '/gateway-' + item + ':authtoken',
-          token_validate=false)} }}"
-      loop:
-        - cesnet-mcc
-    # - name: Debug Deployments Secrets
-    #   debug:
-    #     msg: "{{ item.key }} = {{ item.value }}"
-    #   loop: "{{ secrets | dict2items }}"
-    # - name: Debug Gateway Secrets
-    #   debug:
-    #     msg: "{{ item.key }} = {{ item.value }}"
-    #   loop: "{{ gateways_token | dict2items }}"
+    - name: Debug Deployments Secrets
+      debug:
+        msg: "{{ item.key }} = {{ item.value }}"
+      loop: "{{ secrets | dict2items }}"
     - name: Copy config file to master
       vars:
         name: "{{ item | basename | splitext | first }}"
         secret: "{{ secrets[name] }}"
-        gateways_token: "{{ gateways_token }}"
       template:
         src: "{{ item }}"
         dest: "/tmp/{{ item | basename }}"
@@ -129,3 +118,18 @@
       with_fileglob:
         - "../extra/*.yaml"
       when: true
+- hosts: nfs
+  become: true
+  tasks:
+    - name: Quota settings
+      vars:
+        name: "{{ item | basename | splitext | first }}"
+      cron:
+        cron_file: notebook-quotas
+        name: "{{ name }} quotas"
+        minute: "0"
+        hour: "*/2"
+        job: "/usr/local/bin/xfs-quotas.sh --include ^/exports/{{ name }}- --exclude ^/exports/{{ name }}-hub-db-dir-"
+        user: root
+      with_fileglob:
+        - "../deployments/*.yaml"

demo/playbooks/notebooks.yaml

----
-- name: Notebooks deployments
-  hosts: master
-  become: true
-  tasks:
-    - name: Configure helm repo
-      shell: |-
-        helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/
-        helm repo add eginotebooks https://egi-federation.github.io/egi-notebooks-chart/
-        helm repo update
-      when: "'jupyterhub' not in ansible_local.helm_repos | map(attribute='name') | list or
-             'eginotebooks' not in ansible_local.helm_repos | map(attribute='name') | list"
-    - name: Get Secrets from Vault for notebooks
-      vars:
-        name: "{{ item | basename | splitext | first }}"
-      set_fact:
-        secrets: "{{ secrets|default({}) | combine({name: lookup('community.hashi_vault.hashi_vault', vault_mount_point + '/deployment-' + name,
-          token_validate=false)}) }}"
-      with_fileglob:
-        - "../deployments/*.yaml"
-    - name: Debug Deployments Secrets
-      debug:
-        msg: "{{ item.key }} = {{ item.value }}"
-      loop: "{{ secrets | dict2items }}"
-    - name: Copy config file to master
-      vars:
-        name: "{{ item | basename | splitext | first }}"
-        secret: "{{ secrets[name] }}"
-      template:
-        src: "{{ item }}"
-        dest: "/tmp/{{ item | basename }}"
-        mode: 0600
-      with_fileglob:
-        - "../deployments/*.yaml"
-    - name: Deploy/upgrade notebook instance
-      vars:
-        name: "{{ item | basename | splitext | first }}"
-        version: "3.2.1" # app 4.0.2 (2023-11-27)
-        monitor_version: "0.3.1"
-      shell: |-
-        helm status --namespace {{ name }} {{ name }}
-        if [ $? -ne 0 ]; then
-            helm install --create-namespace --namespace {{ name }} \
-                -f /tmp/{{ item | basename }} --version {{ version }} --timeout 2h \
-                 {{ name }} jupyterhub/jupyterhub
-        else
-            helm upgrade --version {{ version }} -f /tmp/{{ item | basename }} --timeout 2h \
-                --namespace {{ name }} {{ name }} jupyterhub/jupyterhub
-        fi
-        helm status --namespace {{ name }} {{ name }}-monitor
-        if [ $? -ne 0 ]; then
-            helm install --namespace {{ name }} \
-                -f /tmp/{{ item | basename }} --version {{ monitor_version }} \
-                {{ name }}-monitor eginotebooks/notebooks-monitor
-        else
-            helm upgrade --version {{ monitor_version }} \
-                -f /tmp/{{ item | basename }} --namespace {{ name }} \
-                {{ name }}-monitor eginotebooks/notebooks-monitor
-        fi
-      environment:
-        KUBECONFIG: /etc/kubernetes/admin.conf
-        PATH: /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin
-      when: true
-      with_fileglob:
-        - "../deployments/*.yaml"
-
-    - name: Configure secrets management for the hub
-      vars:
-        name: "{{ item | basename | splitext | first }}"
-      shell: |-
-        kubectl apply -f - << EOF
-        ---
-        kind: Role
-        apiVersion: rbac.authorization.k8s.io/v1
-        metadata:
-          name: hub-secrets
-          namespace: {{ name }}
-        rules:
-          - apiGroups: [""]       # "" indicates the core API group
-            resources: ["secrets"]
-            verbs: ["get", "watch", "list", "create", "delete", "patch", "update"]
-        ---
-        kind: RoleBinding
-        apiVersion: rbac.authorization.k8s.io/v1
-        metadata:
-          name: hub-secrets
-          namespace: {{ name }}
-        subjects:
-          - kind: ServiceAccount
-            name: hub
-            namespace: {{ name }}
-        roleRef:
-          kind: Role
-          name: hub-secrets
-          apiGroup: rbac.authorization.k8s.io
-        EOF
-      environment:
-        KUBECONFIG: /etc/kubernetes/admin.conf
-        PATH: /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin
-      when: true
-      with_fileglob:
-        - "../deployments/*.yaml"
-    # do the extra bits of configuration
-    # here we should have all the namespaces, pre-requirements in place
-    # XXX: this won't remove things that are delete from the directory
-    - name: Copy extra configuration files
-      copy:
-        src: "{{ item }}"
-        dest: "/tmp/{{ item | basename }}"
-        mode: 0600
-      with_fileglob:
-        - "../extra/*.yaml"
-    - name: Extra configuration
-      command: |-
-          kubectl apply -f /tmp/{{ item | basename }}
-      environment:
-        KUBECONFIG: /etc/kubernetes/admin.conf
-      with_fileglob:
-        - "../extra/*.yaml"
-      when: true
-- hosts: nfs
-  become: true
-  tasks:
-    - name: Quota settings
-      vars:
-        name: "{{ item | basename | splitext | first }}"
-      cron:
-        cron_file: notebook-quotas
-        name: "{{ name }} quotas"
-        minute: "0"
-        hour: "*/2"
-        job: "/usr/local/bin/xfs-quotas.sh --include ^/exports/{{ name }}- --exclude ^/exports/{{ name }}-hub-db-dir-"
-        user: root
-      with_fileglob:
-        - "../deployments/*.yaml"

staging/deploy.sh

@@ -46,10 +46,10 @@ ansible -m command -a '/root/squid-volume.sh' 'ingress[0]'
 ansible-playbook playbooks/k8s.yaml
 while ansible -i ./inventory -m command -a 'kubectl get pods --all-namespaces' master | tail -n +3 | grep -v ' Running '; do sleep 5; done
 # docker runtime directory after Kubernetes deployment (problem with unmounts)
-ansible -m copy -a 'src=terraform/docker-volume.sh dest=/root/ mode=preserve' 'ingress nfs worker'
-ansible -m command -a '/root/docker-volume.sh' 'ingress nfs worker'
+ansible -m copy -a 'src=terraform/docker-volume.sh dest=/root/ mode=preserve' 'ingress nfs worker gpu'
+ansible -m command -a '/root/docker-volume.sh' 'ingress nfs worker gpu'
+ansible-playbook playbooks/squid.yaml
+ansible-playbook playbooks/cvmfs.yaml
 
-# ansible-playbook playbooks/squid.yaml
-# ansible-playbook playbooks/cvmfs.yaml
-
-while ansible -i ./inventory -m command -a 'kubectl get pods --all-namespaces' master | tail -n +3 | grep -v ' Running '; do sleep 5; done
+# wait for finish
+while ansible -m command -a 'kubectl get pods --all-namespaces' master | tail -n +3 | grep -v ' Running '; do sleep 5; done

staging/terraform/vms.tf

@@ -37,6 +37,16 @@ resource "openstack_networking_secgroup_rule_v2" "ping4" {
   security_group_id = openstack_networking_secgroup_v2.ping.id
 }
 
+resource "openstack_networking_secgroup_rule_v2" "ping6" {
+  direction         = "ingress"
+  ethertype         = "IPv6"
+  port_range_min    = 128
+  port_range_max    = 0
+  protocol          = "icmp"
+  remote_ip_prefix  = "::/0"
+  security_group_id = openstack_networking_secgroup_v2.ping.id
+}
+
 resource "openstack_networking_secgroup_rule_v2" "ssh4" {
   direction         = "ingress"
   ethertype         = "IPv4"

demo/deploy.sh → testing/deploy.sh

 #! /bin/bash -xe
 
 #
-# EOSC LOT3 Jupyter Notebooks Demo
+# EOSC LOT3 Jupyter Notebooks testing instance
 #
 
 cd terraform && terraform init && terraform apply
@@ -13,7 +13,7 @@ ip="$(head -n 1 < terraform/fip.txt)"
 shellstate=$(shopt -po xtrace)
 set +o xtrace
 # https://nsupdate.fedcloud.eu
-vault_prefix=secrets/users/e1662e20-e34b-468c-b0ce-d899bc878364@egi.eu/eosc-demo
+vault_prefix=secrets/users/e1662e20-e34b-468c-b0ce-d899bc878364@egi.eu/eosc-testing
 FEDCLOUD_DYNAMIC_DNS=$(vault read -field data $vault_prefix/FEDCLOUD_DYNAMIC_DNS | grep ^map | head -n 1 | sed 's/map\[\(.*\)\]/\1/')
 for auth in $FEDCLOUD_DYNAMIC_DNS; do
 	echo "curl -i -X GET -u $(echo "$auth" | cut -d: -f1):XXX https://nsupdate.fedcloud.eu/nic/update?myip=$ip"
@@ -22,7 +22,7 @@ done
 eval "$shellstate"
 
 # wait for ping and ssh
-for ip in $(cat terraform/hosts.txt); do
+for ip in $(cat terraform/hosts.txt) $(cat terraform/fip.txt); do
 	while ! ping -c 1 "$ip"; do sleep 5; done
 	ssh-keygen -R "$ip"
 	while ! ssh egi@"$ip" -o ConnectTimeout=10 -o PreferredAuthentications=publickey -o StrictHostKeyChecking=no :; do sleep 10; done
@@ -39,13 +39,15 @@ ansible -m copy -a 'src=terraform/nfs-volume.sh dest=/root/ mode=preserve' nfs
 ansible -m command -a '/root/nfs-volume.sh' nfs
 ansible -m copy -a 'src=terraform/squid-volume.sh dest=/root/ mode=preserve' 'ingress[0]'
 ansible -m command -a '/root/squid-volume.sh' 'ingress[0]'
-ansible -m copy -a 'src=terraform/docker-volume.sh dest=/root/ mode=preserve' 'ingress nfs worker gpu'
-ansible -m command -a '/root/docker-volume.sh' 'ingress nfs worker gpu'
 ansible -m copy -a 'src=terraform/scratch-volume.sh dest=/root/ mode=preserve' 'ingress nfs worker gpu'
 ansible -m command -a '/root/scratch-volume.sh' 'ingress nfs worker gpu'
 
 # k8s + notebooks
 ansible-playbook playbooks/k8s.yaml
+while ansible -i ./inventory -m command -a 'kubectl get pods --all-namespaces' master | tail -n +3 | grep -v ' Running '; do sleep 5; done
+# docker runtime directory after Kubernetes deployment (problem with unmounts)
+ansible -m copy -a 'src=terraform/docker-volume.sh dest=/root/ mode=preserve' 'ingress nfs worker gpu'
+ansible -m command -a '/root/docker-volume.sh' 'ingress nfs worker gpu'
 ansible-playbook playbooks/squid.yaml
 ansible-playbook playbooks/cvmfs.yaml
 

demo/deployments/hub.yaml → testing/deployments/hub.yaml

@@ -17,8 +17,6 @@ ingress:
 
 singleuser:
   # keep resource limits in sync with:
-  # - documentation/content/en/users/dev-env/notebooks/_index.md
-  # - documentation/content/en/users/dev-env/notebooks/data/_index.md
   # - profileList
   storage:
     capacity: 20Gi
@@ -102,6 +100,31 @@ singleuser:
           nvidia.com/gpu: 1
         extra_resource_limits:
           nvidia.com/gpu: 1
+  cmd: jupyterhub-singleuser-webdav-wrapper
+  extraFiles:
+    wait-owncloud.sh:
+      mode: 0755
+      mountPath: /usr/local/bin/jupyterhub-singleuser-webdav-wrapper
+      stringData: |-
+        #! /bin/sh
+
+        #
+        # Dirty hack to make remote mount on home directory working properly:
+        #
+        # 1) wait for webdav sidecar image to kick in
+        # 2) change directory to the mounted version of itself
+        # 3) launch notebook server
+        #
+        i=0
+        while ! grep '^webdav-fs: /mnt/owncloud ' /proc/mounts && test $i -lt 30; do
+          echo 'Waiting for ownClound mount...'
+          sleep 0.5
+          i=$((i+1))
+        done
+
+        cd .
+
+        exec jupyterhub-singleuser "$@"
 
 hub:
   services:
@@ -110,7 +133,7 @@ hub:
       admin: true
   image:
     name: valtri/hub
-    tag: "3.x-eosc4" # EOSC, jupyter 3.1.0
+    tag: "eosc7-jwt"
   config:
     Authenticator:
       enable_auth_state: true
@@ -393,8 +416,5 @@ hub:
         and <a href="https://docs.egi.eu/users/dev-env/notebooks/#notebooks-for-researchers">
         enrolling to one of the supported VOs</a>.
         </p>
-        <p>
-        Default environment provides 2 CPU cores, 4 GB RAM and 20GB of personal storage space per user
-        </p>
         {% endblock main_intro %}
 {% endraw %}

demo/inventory/1-cesnet.yaml → testing/inventory/1-cesnet.yaml

@@ -5,28 +5,27 @@ fip:
 
 master:
   hosts:
-    2001:718:801:432:f816:3eff:fe33:7a22:
+    2001:718:801:432:f816:3eff:fe36:5620:
       # must be IPv4 address or hostname
-      kube_server: 192.168.0.231
+      kube_server: 192.168.0.115
 
 ingress:
   hosts:
-    2001:718:801:432:f816:3eff:fece:3b38:
+    2001:718:801:432:f816:3eff:fe41:a4b4:
 
 nfs:
   hosts:
-    2001:718:801:432:f816:3eff:fe97:4bbd:
+    2001:718:801:432:f816:3eff:fee1:9385:
 
 worker:
   hosts:
-    2001:718:801:432:f816:3eff:fe82:628a:
+    2001:718:801:432:f816:3eff:fef7:c707:
 
 gpu:
   hosts:
-    2001:718:801:432:f816:3eff:fe4e:c5ab:
 
 # using public IP of kube_server for ansible delegate_to
 kube_server:
   hosts:
-    192.168.0.231:
-      ansible_host: 2001:718:801:432:f816:3eff:fe33:7a22
+    192.168.0.115:
+      ansible_host: 2001:718:801:432:f816:3eff:fe36:5620

demo/inventory/99-all.yaml → testing/inventory/99-all.yaml

@@ -12,7 +12,7 @@ all:
     ansible_become: yes
     ansible_user: egi
     site_name: cesnet
-    vault_mount_point: secrets/users/e1662e20-e34b-468c-b0ce-d899bc878364@egi.eu/eosc-demo
+    vault_mount_point: secrets/users/e1662e20-e34b-468c-b0ce-d899bc878364@egi.eu/eosc-testing
 
     notebooks_hostname: eosc-notebooks.vm.fedcloud.eu
     grafana_hostname: grafana-eosc-notebooks.vm.fedcloud.eu

testing/playbooks/notebooks.yaml

+../../cesnet-central/playbooks/notebooks.yaml
\ No newline at end of file