New production2 deployment @ SafeSpring

common/deployments/hub-production.yaml

+---
+proxy:
+  service:
+    type: NodePort
+
+ingress:
+  enabled: true
+  annotations:
+    kubernetes.io/ingress.class: "nginx"
+    kubernetes.io/tls-acme: "true"
+  hosts:
+    - "{{ notebooks_hostname }}"
+  tls:
+    - hosts:
+        - "{{ notebooks_hostname }}"
+      secretName: acme-tls-hub
+
+singleuser:
+  # keep resource limits in sync with:
+  # - profileList
+  storage:
+    type: none
+    extraVolumes:
+      - name: cvmfs-host
+        hostPath:
+          path: /cvmfs
+          type: Directory
+      - name: owncloud-home
+        empty_dir:
+      # - name: scratch
+      #   ephemeral:
+      #     volumeClaimTemplate:
+      #       spec:
+      #         accessModes: [ "ReadWriteOnce" ]
+      #         storageClassName: local-path
+      #         resources:
+      #           requests:
+      #             storage: "10Gi"
+    extraVolumeMounts:
+      - name: cvmfs-host
+        mountPath: "/cvmfs:shared"
+      - name: owncloud-home
+        mountPath: '/home/jovyan:shared'
+      # - name: scratch
+      #   mountPath: '/scratch'
+  memory:
+    limit: 4G
+    guarantee: 512M
+  cpu:
+    limit: 2
+    guarantee: .2
+  defaultUrl: "/lab"
+  image:
+    name: eginotebooks/single-user
+    tag: "sha-6d48e61"
+  profileList:
+    - display_name: Small Environment - 2 vCPU / 4 GB RAM
+      description: >
+        The notebook environment includes Python, R, Julia and Octave kernels.
+      default: true
+      kubespawner_override:
+        args:
+          - "--CondaKernelSpecManager.env_filter='/opt/conda$'"
+        extra_annotations:
+          "egi.eu/flavor": "small-environment-2-vcpu-4-gb-ram"
+      vo_claims:
+        - urn:geant:eosc-federation.eu:res:notebooks.open-science-cloud.ec.europa.eu:2-vcpu-4-gb-ram:act:ppa
+    - display_name: Medium Environment - 4 vCPU / 8 GB RAM
+      description: >
+        The notebook environment includes Python, R, Julia and Octave kernels.
+      kubespawner_override:
+        args:
+          - "--CondaKernelSpecManager.env_filter='/opt/conda$'"
+        extra_annotations:
+          "egi.eu/flavor": "medium-environment-4-vcpu-8-gb-ram"
+        cpu_guarantee: 0.4
+        cpu_limit: 4
+        mem_guarantee: 1G
+        mem_limit: 8G
+      vo_claims:
+        - urn:geant:eosc-federation.eu:res:notebooks.open-science-cloud.ec.europa.eu:4-vcpu-8-gb-ram:act:ppa
+    - display_name: Large Environment - 8 vCPU / 16 GB RAM / GPU
+      description: >
+        The notebook environment includes Python, R, Julia and Octave kernels with GPU.
+      kubespawner_override:
+        args:
+          - "--CondaKernelSpecManager.env_filter='/opt/conda$'"
+        cpu_guarantee: 0.8
+        cpu_limit: 8
+        mem_guarantee: 2G
+        mem_limit: 16G
+        extra_annotations:
+          "egi.eu/flavor": "large-environment-8-vcpu-16-gb-ram-gpu"
+        extra_resource_guarantees:
+          nvidia.com/gpu: 1
+        extra_resource_limits:
+          nvidia.com/gpu: 1
+      vo_claims:
+        - urn:geant:eosc-federation.eu:res:notebooks.open-science-cloud.ec.europa.eu:8-vcpu-16-gb-ram-gpu:act:ppa
+  cmd: jupyterhub-singleuser-webdav-wrapper
+  extraFiles:
+    wait-remote-home.sh:
+      mode: 0755
+      mountPath: /usr/local/bin/jupyterhub-wait-remote-home
+      stringData: |-
+        #! /bin/sh
+        i=0
+        while ! grep '^webdav-fs: /home/jovyan ' /proc/mounts && test $i -lt 30; do
+          echo 'Waiting for ownClound mount...'
+          sleep 0.5
+          i=$((i+1))
+        done
+    singleuser-webdav-wrapper.sh:
+      mode: 0755
+      mountPath: /usr/local/bin/jupyterhub-singleuser-webdav-wrapper
+      stringData: |-
+        #! /bin/sh
+        #
+        # Dirty hack to make remote mount on home directory working properly:
+        #
+        # 1) wait for webdav sidecar image to kick in
+        # 2) change directory to the mounted version of itself
+        # 3) launch notebook server
+        #
+        /usr/local/bin/jupyterhub-wait-remote-home
+
+        cd .
+
+        exec jupyterhub-singleuser \
+          --FileCheckpoints.checkpoint_dir='/home/jovyan/.notebookCheckpoints' \
+          --NotebookNotary.db_file=':memory:' \
+          "$@"
+    disable-labnews:
+      mode: 0644
+      mountPath: /opt/conda/share/jupyter/lab/settings/overrides.json
+      stringData: |-
+        {
+          "@jupyterlab/apputils-extension:notification": {
+            "checkForUpdates": "false",
+            "fetchNews": "false"
+          },
+          "@jupyterlab/application-extension:context-menu": {
+            "contextMenu": [
+              {
+                "command": "filebrowser:share-main",
+                "selector": ".jp-DirListing-item[data-isdir]",
+                "disabled": true
+              }
+            ]
+          }
+        }
+
+hub:
+  services:
+    status:
+      url: "http://status-web/"
+      admin: true
+    jwt:
+      url: "http://jwt/"
+      display: false
+  # recommended to keep in sync with common/playbooks/files/jupyterhub-jwt.yaml
+  image:
+    name: eginotebooks/hub
+    tag: "sha-ae769eb"
+  config:
+    Authenticator:
+      enable_auth_state: true
+      admin_users:
+        # valtri@civ.zcu.cz
+        - 94d3cde7-3121-4b33-b4c2-526c67e8cb38@eosc-federation.eu
+      allowed_groups:
+        - urn:geant:eosc-federation.eu:res:notebooks.open-science-cloud.ec.europa.eu:2-vcpu-4-gb-ram:act:ppa
+        - urn:geant:eosc-federation.eu:res:notebooks.open-science-cloud.ec.europa.eu:4-vcpu-8-gb-ram:act:ppa
+        - urn:geant:eosc-federation.eu:res:notebooks.open-science-cloud.ec.europa.eu:8-vcpu-16-gb-ram-gpu:act:ppa
+      admin_groups:
+        - urn:geant:eosc-federation.eu:group:asg:notebooks.open-science-cloud.ec.europa.eu:role=admin
+      claim_groups_key: "entitlements"
+    EGICheckinAuthenticator:
+      checkin_host: "{{ secret['checkin_host'] }}"
+      authorize_url: "https://{{ secret['checkin_host'] }}/OIDC/authorization"
+      token_url: "https://{{ secret['checkin_host'] }}/OIDC/token"
+      userdata_url: "https://{{ secret['checkin_host'] }}/OIDC/userinfo"
+      client_id: "{{ secret['client_id'] }}"
+      client_secret: "{{ secret['client_secret'] }}"
+      oauth_callback_url: "https://{{ notebooks_hostname }}/hub/oauth_callback"
+      openid_configuration_url: "https://proxy.testing.eosc-federation.eu/.well-known/openid-configuration"
+      scope: ["openid", "profile", "email", "offline_access", "entitlements"]
+      username_claim: "sub"
+      extra_authorize_params:
+        prompt: consent
+    JupyterHub:
+      admin_access: true
+      authenticate_prometheus: false
+      authenticator_class: egi_notebooks_hub.egiauthenticator.EOSCNodeAuthenticator
+      # spawner_class: (in egi-notebooks-b2drop)
+    LabApp:
+      check_for_updates_class: jupyterlab.NeverCheckForUpdate
+  extraConfig:
+    egi-notebooks-welcome: |-
+      from egi_notebooks_hub.welcome import WelcomeHandler
+      c.JupyterHub.default_url = "/welcome"
+      c.JupyterHub.extra_handlers = [(r'/welcome', WelcomeHandler)]
+    egi-notebooks-b2drop: |-
+{%- raw %}
+      import json
+      from egi_notebooks_hub.onedata import OnedataSpawner
+      from tornado.httpclient import AsyncHTTPClient, HTTPClientError, HTTPRequest
+
+      class WebDavOIDCSpawner(OnedataSpawner):
+          # ownCloud Infinite Scale parameters
+          # (https://owncloud.dev/apis/http/graph/spaces/#list-my-spaces-get-medrives)
+          OCIS_URL = "https://drive.open-science-cloud.ec.europa.eu"
+          # personal space
+          OCIS_PERSONAL_SPACE = "/graph/v1.0/me/drives?%24filter=driveType+eq+personal"
+          # shared space
+          OCIS_SHARED_WITH_ME = "/graph/v1.0/me/drives?%24filter=driveType+eq+virtual"
+          # otter spaces
+          OCIS_SPACES = "/graph/v1.0/me/drives?%24filter=driveType+eq+project"
+
+          async def append_owncloud_sidecar(self, spawner, type, query, fallback_url=None, headers={}):
+              owncloud_url = fallback_url
+              http_client = AsyncHTTPClient()
+              req = HTTPRequest(
+                  self.OCIS_URL + query,
+                  headers=headers,
+                  method="GET",
+              )
+              try:
+                  resp = await http_client.fetch(req)
+                  body = json.loads(resp.body.decode("utf8", "replace"))
+                  self.log.debug("OCIS response: %s", body)
+                  if "value" in body:
+                      ocis_infos = body["value"]
+                      if len(ocis_infos) >= 1 and "root" in ocis_infos[0]:
+                          owncloud_url = ocis_infos[0]["root"].get("webDavUrl", None)
+              except HTTPClientError as e:
+                  self.log.error("can't query ownCloud: %s", e)
+              self.log.info("ownCloud %s URL: %s", type, owncloud_url)
+
+              if owncloud_url is None:
+                  return
+
+              if type == "home":
+                  subpath = ""
+              else:
+                 subpath = "/" + type.capitalize()
+              env = [
+                  {"name": "WEBDAV_URL", "value": owncloud_url},
+                  {"name": "WEBDAV_VENDOR", "value": "owncloud"},
+                  # XXX: strict permissions needed for .local/share/jupyter/runtime/jupyter_cookie_secret
+                  # quicker directory cache and polling
+                  {"name": "MOUNT_OPTS", "value": "--file-perms=0600 --dir-perms=0770 --dir-cache-time=1m0s --poll-interval=0m20s"},
+                  {"name": "MOUNT_PATH", "value": "/owncloud" + subpath},
+                  # default mode is "full"
+                  {"name": "VFS_CACHE_MODE", "value": "full"},
+              ]
+              if type != "home":
+                  env.append({"name": "MOUNT_WAIT_POINT", "value": "webdav-fs: /owncloud fuse.rclone"})
+              volume_mounts = [
+                  {"mountPath": "/owncloud:shared", "name": "owncloud-home"},
+                  {"mountPath": self.token_mount_path, "name": self.token_secret_volume_name, "readOnly": True},
+              ]
+              spawner.extra_containers.append(
+                  {
+                      "name": "owncloud-" + type,
+                      "image": "eginotebooks/webdav-rclone-sidecar:sha-0a62679",
+                      "args": ["bearer_token_command=cat " + self.token_path],
+                      "env": env,
+                      "resources": self.sidecar_resources,
+                      "securityContext": {
+                          "runAsUser": 1000,
+                          "fsUser": 1000,
+                          "fsGroup": 100,
+                          "privileged": True,
+                          "capabilities": {"add": ["SYS_ADMIN"]},
+                      },
+                      "volumeMounts": volume_mounts,
+                  }
+              )
+
+          async def pre_spawn_hook(self, spawner):
+              await super(WebDavOIDCSpawner, self).pre_spawn_hook(spawner)
+              auth_state = await self.user.get_auth_state()
+              # volume name as in EGI spawner
+              self.token_secret_volume_name = self._expand_user_properties(
+                self.token_secret_volume_name_template
+              )
+              self.token_path = os.path.join(self.token_mount_path, "access_token")
+
+              if auth_state:
+                  access_token = auth_state.get("access_token", None)
+                  headers = {
+                      "Accept": "application/json",
+                      "User-Agent": "JupyterHub",
+                      "Authorization": "Bearer %s" % access_token,
+                  }
+
+                  await self.append_owncloud_sidecar(spawner, "home", self.OCIS_PERSONAL_SPACE, headers=headers)
+                  await self.append_owncloud_sidecar(spawner, "shares", self.OCIS_SHARED_WITH_ME, headers=headers)
+                  await self.append_owncloud_sidecar(spawner, "spaces", self.OCIS_SPACES, headers=headers)
+              else:
+                self.log.info("No auth state, skipping ownCloud")
+
+      c.JupyterHub.spawner_class = WebDavOIDCSpawner
+      c.WebDavOIDCSpawner.token_mount_path = "/var/run/secrets/oidc/"
+      c.WebDavOIDCSpawner.http_timeout = 90
+{% endraw %}
+  extraFiles:
+    welcome.html:
+      mountPath: /usr/local/share/jupyterhub/templates/welcome.html
+      stringData: |-
+{%- raw %}
+        {% extends "login.html" %}
+{% endraw %}
+    403.html:
+      mountPath: /usr/local/share/jupyterhub/templates/403.html
+      stringData: |-
+{%- raw %}
+        {% extends "error.html" %}
+        {% block main %}
+        <div class="error">
+          <h1>Unauthorized</h1>
+          <p>You don't have the correct entitlements to access this service.</p>
+          <p>If you think you should be granted access, please open an issue!</p>
+        </div>
+        {% endblock %}
+{% endraw %}

production2/ansible.cfg

+[defaults]
+inventory=inventory
+
+[diff]
+always=true

production2/deploy.sh

+#! /bin/bash -xe
+
+#
+# Deploy EOSC production instance
+#
+
+cd terraform && terraform init && terraform apply
+cd -
+cp -pv terraform/inventory.yaml inventory/1-safespring.yaml
+
+# dynamic DNS
+ip="$(head -n 1 < terraform/fip.txt)"
+# shellstate=$(shopt -po xtrace)
+# set +o xtrace
+# # https://nsupdate.fedcloud.eu
+# vault_prefix=secrets/users/e1662e20-e34b-468c-b0ce-d899bc878364@egi.eu/eosc-production
+# FEDCLOUD_DYNAMIC_DNS=$(vault read -field data $vault_prefix/FEDCLOUD_DYNAMIC_DNS | grep ^map | head -n 1 | sed 's/map\[\(.*\)\]/\1/')
+# for auth in $FEDCLOUD_DYNAMIC_DNS; do
+# 	echo "curl -i -X GET -u $(echo "$auth" | cut -d: -f1):XXX https://nsupdate.fedcloud.eu/nic/update?myip=$ip"
+# 	curl -i -X GET -u "$auth" https://nsupdate.fedcloud.eu/nic/update?myip="$ip"
+# done
+# eval "$shellstate"
+echo "Terraform finished. Check terraform/docker-volume.sh. Continue? (CTRL-C to quit)"
+read -r _
+
+# wait for ping and ssh
+for ip in $(cat terraform/hosts.txt); do
+	while ! ping -c 1 "$ip"; do sleep 5; done
+	ssh-keygen -R "$ip"
+	while ! ssh egi@"$ip" -o ConnectTimeout=10 -o PreferredAuthentications=publickey -o StrictHostKeyChecking=no :; do sleep 10; done
+done
+
+# check ssh access
+ansible -m command -a 'uname -a' allnodes
+
+# wait cloud-init
+ansible -m shell -a 'while ! test -f /var/lib/cloud/instance/boot-finished; do sleep 2; done' allnodes
+
+# setup volumes
+ansible -m copy -a 'src=terraform/nfs-volume.sh dest=/root/ mode=preserve' nfs
+ansible -m command -a '/root/nfs-volume.sh' nfs
+ansible -m copy -a 'src=terraform/squid-volume.sh dest=/root/ mode=preserve' 'ingress[0]'
+ansible -m command -a '/root/squid-volume.sh' 'ingress[0]'
+
+# kubernetes
+ansible-playbook playbooks/k8s.yaml
+while ansible -i ./inventory -m command -a 'kubectl get pods --all-namespaces' master | tail -n +3 | grep -v ' Running '; do sleep 5; done
+# docker runtime directory after Kubernetes deployment (problem with unmounts)
+ansible -m copy -a 'src=terraform/docker-volume.sh dest=/root/ mode=preserve' 'ingress nfs worker gpu'
+ansible -m command -a '/root/docker-volume.sh' 'ingress nfs worker gpu'
+ansible-playbook playbooks/squid.yaml
+ansible-playbook playbooks/cvmfs.yaml
+
+# wait for finish
+while ansible -m command -a 'kubectl get pods --all-namespaces' master | tail -n +3 | grep -v ' Running '; do sleep 5; done

production2/deployments/hub.yaml

+../../common/deployments/hub-production.yaml
\ No newline at end of file

production2/extra

+../common/extra
\ No newline at end of file

production2/inventory/1-safespring.yaml

+---
+fip:
+  hosts:
+    89.47.190.171
+
+master:
+  hosts:
+    2001:6b0:7d:40::5e:
+      # must be IPv4 address or hostname
+      kube_server: 89.47.191.144
+
+ingress:
+  hosts:
+    2001:6b0:7d:40::1bd:
+
+nfs:
+  hosts:
+    2001:6b0:7d:40::274:
+
+worker:
+  hosts:
+    2001:6b0:7d:40::bb:
+    2001:6b0:7d:40::37e:
+    2001:6b0:7d:40::276:
+    2001:6b0:7d:40::273:
+    2001:6b0:7d:40::246:
+    2001:6b0:7d:40::13:
+    2001:6b0:7d:40::1a0:
+    2001:6b0:7d:40::3dc:
+
+gpu:
+  hosts:
+
+# using public IP of kube_server for ansible delegate_to
+kube_server:
+  hosts:
+    89.47.191.144:
+      ansible_host: 2001:6b0:7d:40::5e

production2/inventory/99-all.yaml

+---
+allnodes:
+  children:
+    master:
+    ingress:
+    nfs:
+    worker:
+
+all:
+  vars:
+    ansible_become: yes
+    ansible_user: egi
+
+    mail_local: true
+    site_name: safespring-production2
+    vault_mount_point: secrets/users/e1662e20-e34b-468c-b0ce-d899bc878364@egi.eu/eosc-production
+
+    notebooks_hostname: eu-2.notebooks.open-science-cloud.ec.europa.eu
+    grafana_hostname: grafana.eu-2.notebooks.open-science-cloud.ec.europa.eu

production2/playbooks/cvmfs.yaml

+../../common/playbooks/cvmfs.yaml
\ No newline at end of file

production2/playbooks/files/calico.yaml

+../../../common/playbooks/files/calico.yaml
\ No newline at end of file

production2/playbooks/files/etc

+../../../common/playbooks/files/etc
\ No newline at end of file

production2/playbooks/files/jupyterhub-jwt.yaml

+../../../common/playbooks/files/jupyterhub-jwt.yaml
\ No newline at end of file

production2/playbooks/files/usr

+../../../common/playbooks/files/usr
\ No newline at end of file

production2/playbooks/k8s.yaml

+../../common/playbooks/k8s.yaml
\ No newline at end of file

production2/playbooks/notebooks.yaml

+../../common/playbooks/notebooks.yaml
\ No newline at end of file

production2/playbooks/public_keys

+../../common/playbooks/public_keys
\ No newline at end of file

production2/playbooks/squid.yaml

+../../common/playbooks/squid.yaml
\ No newline at end of file

production2/playbooks/templates/etc/exports

+../../../../common/playbooks/templates/etc/exports.ipv46
\ No newline at end of file

production2/playbooks/templates/etc/mailutils.conf

+../../../../common/playbooks/templates/etc/mailutils.conf
\ No newline at end of file

production2/playbooks/templates/etc/squid

+../../../../common/playbooks/templates/etc/squid
\ No newline at end of file

production2/playbooks/upgrade.yaml

+../../common/playbooks/upgrade.yaml
\ No newline at end of file