From ff806835e712aef6f69f587275f1735a67962892 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franti=C5=A1ek=20Dvo=C5=99=C3=A1k?= <valtri@civ.zcu.cz> Date: Fri, 19 Jul 2024 16:18:48 +0000 Subject: [PATCH] New staging1 deployment @ PSNC + cosmetic cleanups --- cesnet-central/deployments/fullhub.yaml | 19 +- staging1/deployments/hub.yaml | 444 ++++++++++++++++++++++++ staging1/inventory/1-psnc.yaml | 16 +- staging1/inventory/99-all.yaml | 2 +- staging1/playbooks/notebooks.yaml | 1 + testing/deployments/hub.yaml | 21 +- 6 files changed, 476 insertions(+), 27 deletions(-) create mode 100644 staging1/deployments/hub.yaml create mode 120000 staging1/playbooks/notebooks.yaml diff --git a/cesnet-central/deployments/fullhub.yaml b/cesnet-central/deployments/fullhub.yaml index bae6e9b..d41b77e 100644 --- a/cesnet-central/deployments/fullhub.yaml +++ b/cesnet-central/deployments/fullhub.yaml @@ -9,10 +9,10 @@ ingress: kubernetes.io/ingress.class: "nginx" kubernetes.io/tls-acme: "true" hosts: - - fullhub.eosc.zcu.cz + - "{{ notebooks_hostname }}" tls: - hosts: - - fullhub.eosc.zcu.cz + - "{{ notebooks_hostname }}" secretName: acme-tls-fullhub singleuser: @@ -80,14 +80,6 @@ singleuser: mountPath: /usr/local/bin/jupyterhub-wait-remote-home stringData: |- #! /bin/sh - - # - # Dirty hack to make remote mount on home directory working properly: - # - # 1) wait for webdav sidecar image to kick in - # 2) change directory to the mounted version of itself - # 3) launch notebook server - # i=0 while ! grep '^webdav-fs: /home/jovyan ' /proc/mounts && test $i -lt 30; do echo 'Waiting for ownClound mount...' @@ -99,6 +91,13 @@ singleuser: mountPath: /usr/local/bin/jupyterhub-singleuser-webdav-wrapper stringData: |- #! /bin/sh + # + # Dirty hack to make remote mount on home directory working properly: + # + # 1) wait for webdav sidecar image to kick in + # 2) change directory to the mounted version of itself + # 3) launch notebook server + # /usr/local/bin/jupyterhub-wait-remote-home cd . diff --git a/staging1/deployments/hub.yaml b/staging1/deployments/hub.yaml new file mode 100644 index 0000000..fbf2ef5 --- /dev/null +++ b/staging1/deployments/hub.yaml @@ -0,0 +1,444 @@ +--- +proxy: + service: + type: NodePort + +ingress: + enabled: true + annotations: + kubernetes.io/ingress.class: "nginx" + kubernetes.io/tls-acme: "true" + hosts: + - "{{ notebooks_hostname }}" + tls: + - hosts: + - "{{ notebooks_hostname }}" + secretName: acme-tls-hub + +singleuser: + # keep resource limits in sync with: + # - profileList + storage: + type: none + extraVolumes: + - name: cvmfs-host + hostPath: + path: /cvmfs + type: Directory + - name: owncloud-home + empty_dir: + # - name: scratch + # ephemeral: + # volumeClaimTemplate: + # spec: + # accessModes: [ "ReadWriteOnce" ] + # storageClassName: local-path + # resources: + # requests: + # storage: "10Gi" + extraVolumeMounts: + - name: cvmfs-host + mountPath: "/cvmfs:shared" + - name: owncloud-home + mountPath: '/home/jovyan:shared' + # - name: scratch + # mountPath: '/scratch' + memory: + limit: 4G + guarantee: 512M + cpu: + limit: 2 + guarantee: .2 + defaultUrl: "/lab" + image: + name: eginotebooks/single-user + tag: "sha-b94a3ef" + profileList: + - display_name: Small Environment - 2 vCPU / 4 GB RAM + description: > + The notebook environment includes Python, R, Julia and Octave kernels. + default: true + kubespawner_override: + args: + - "--CondaKernelSpecManager.env_filter='/opt/conda$'" + extra_annotations: + "egi.eu/flavor": "small-environment-2-vcpu-4-gb-ram" + - display_name: Medium Environment - 4 vCPU / 8 GB RAM + description: > + The notebook environment includes Python, R, Julia and Octave kernels. + kubespawner_override: + args: + - "--CondaKernelSpecManager.env_filter='/opt/conda$'" + extra_annotations: + "egi.eu/flavor": "medium-environment-4-vcpu-8-gb-ram" + cpu_guarantee: 0.4 + cpu_limit: 4 + mem_guarantee: 1G + mem_limit: 8G + - display_name: Large Environment - 8 vCPU / 16 GB RAM / GPU + description: > + The notebook environment includes Python, R, Julia and Octave kernels with GPU. + kubespawner_override: + args: + - "--CondaKernelSpecManager.env_filter='/opt/conda$'" + cpu_guarantee: 0.8 + cpu_limit: 8 + mem_guarantee: 2G + mem_limit: 16G + extra_annotations: + "egi.eu/flavor": "large-environment-8-vcpu-16-gb-ram-gpu" + extra_resource_guarantees: + nvidia.com/gpu: 1 + extra_resource_limits: + nvidia.com/gpu: 1 + cmd: jupyterhub-singleuser-webdav-wrapper + extraFiles: + wait-remote-home.sh: + mode: 0755 + mountPath: /usr/local/bin/jupyterhub-wait-remote-home + stringData: |- + #! /bin/sh + i=0 + while ! grep '^webdav-fs: /home/jovyan ' /proc/mounts && test $i -lt 30; do + echo 'Waiting for ownClound mount...' + sleep 0.5 + i=$((i+1)) + done + singleuser-webdav-wrapper.sh: + mode: 0755 + mountPath: /usr/local/bin/jupyterhub-singleuser-webdav-wrapper + stringData: |- + #! /bin/sh + # + # Dirty hack to make remote mount on home directory working properly: + # + # 1) wait for webdav sidecar image to kick in + # 2) change directory to the mounted version of itself + # 3) launch notebook server + # + /usr/local/bin/jupyterhub-wait-remote-home + + cd . + + exec jupyterhub-singleuser "$@" + +hub: + services: + status: + url: "http://status-web/" + admin: true + jwt: + url: "http://jwt/" + display: false + image: + name: eginotebooks/hub + tag: "sha-323c75e" + config: + Authenticator: + enable_auth_state: true + admin_users: + # valtri@civ.zcu.cz + - c36b18fe-e03a-4a22-ab14-5965e0171410@eosc-federation.eu + allowed_groups: + - urn:geant:eosc-federation.eu:testing:group:eosc#testing.eosc-federation.eu + auto_login: true + claim_groups_key: "entitlements" + EGICheckinAuthenticator: + checkin_host: "{{ secret['checkin_host'] }}" + authorize_url: "https://{{ secret['checkin_host'] }}/OIDC/authorization" + token_url: "https://{{ secret['checkin_host'] }}/OIDC/token" + userdata_url: "https://{{ secret['checkin_host'] }}/OIDC/userinfo" + client_id: "{{ secret['client_id'] }}" + client_secret: "{{ secret['client_secret'] }}" + oauth_callback_url: "https://{{ notebooks_hostname }}/hub/oauth_callback" + openid_configuration_url: "https://proxy.testing.eosc-federation.eu/.well-known/openid-configuration" + scope: ["openid", "profile", "email", "offline_access", "entitlements"] + username_claim: "sub" + extra_authorize_params: + prompt: consent + JupyterHub: + admin_access: true + authenticate_prometheus: false + authenticator_class: egi_notebooks_hub.egiauthenticator.EOSCNodeAuthenticator + # spawner_class: (in egi-notebooks-b2drop) + extraConfig: + egi-notebooks-welcome: |- + from egi_notebooks_hub.welcome import WelcomeHandler + c.JupyterHub.default_url = "/welcome" + c.JupyterHub.extra_handlers = [(r'/welcome', WelcomeHandler)] + egi-notebooks-b2drop: |- +{%- raw %} + import base64 + import json + from jinja2 import BaseLoader + from jinja2 import Environment + from egi_notebooks_hub.onedata import OnedataSpawner + from kubernetes_asyncio.client.rest import ApiException + from tornado.httpclient import AsyncHTTPClient, HTTPClientError, HTTPRequest + + + class B2DropSpawner(OnedataSpawner): + async def auth_state_hook(self, spawner, auth_state): + await super().auth_state_hook(spawner, auth_state) + self.b2drop_ready = False + self.b2drop_user = "" + self.b2drop_pwd = "" + try: + secret = await self.api.read_namespaced_secret(self.token_secret_name, self.namespace) + except ApiException: + return + if secret and secret.data: + self.b2drop_user = base64.b64decode(secret.data.get("b2drop-user", "")).decode() + self.b2drop_pwd = base64.b64decode(secret.data.get("b2drop-pwd", "")).decode() + self.b2drop_ready = (self.b2drop_user and self.b2drop_pwd) + + def _render_options_form(self, profile_list): + # old:self._profile_list = self._init_profile_list(profile_list) + self._profile_list = self._get_initialized_profile_list(profile_list) + + profile_form_template = Environment(loader=BaseLoader).from_string( + self.profile_form_template + ) + return profile_form_template.render(profile_list=self._profile_list, b2drop_ready=self.b2drop_ready, b2drop_user=self.b2drop_user, b2drop_pwd=self.b2drop_pwd) + + async def pre_spawn_hook(self, spawner): + await super(B2DropSpawner, self).pre_spawn_hook(spawner) + b2drop_user = self.user_options.get("b2drop-user", "") + b2drop_pwd = self.user_options.get("b2drop-pwd", "") + if not (b2drop_user and b2drop_pwd): + secret = await self.api.read_namespaced_secret(self.token_secret_name, self.namespace) + if secret and secret.data: + b2drop_user = base64.b64decode(secret.data.get("b2drop-user", "")).decode() + b2drop_pwd = base64.b64decode(secret.data.get("b2drop-pwd", "")).decode() + if b2drop_user and b2drop_pwd: + volume_mounts = [ + {"mountPath": "/owncloud:shared", "name": "owncloud-home"}, + ] + spawner.extra_containers.append( + { + "name": "b2drop", + "image": "eginotebooks/webdav-rclone-sidecar:sha-0a62679", + "env": [ + {"name": "WEBDAV_URL", "value": "https://b2drop.eudat.eu/remote.php/webdav"}, + {"name": "WEBDAV_PWD", "value": b2drop_pwd}, + {"name": "WEBDAV_USER", "value": b2drop_user}, + {"name": "WEBDAV_VENDOR", "value": "other"}, + {"name": "MOUNT_PATH", "value": "/owncloud/b2drop"}, + {"name": "MOUNT_WAIT_POINT", "value": "webdav-fs: /owncloud fuse.rclone"}, + ], + "resources": self.sidecar_resources, + "securityContext": { + "runAsUser": 1000, + "fsUser": 1000, + "fsGroup": 100, + "privileged": True, + "capabilities": {"add": ["SYS_ADMIN"]}, + }, + "volumeMounts": volume_mounts, + } + ) + + def options_from_form(self, formdata): + data = super(B2DropSpawner, self)._options_from_form(formdata) + data.update({'b2drop-user': formdata.get('b2drop-user', [None])[0], + 'b2drop-pwd': formdata.get('b2drop-pwd', [None])[0]}) + return data + + + class WebDavOIDCSpawner(B2DropSpawner): + # ownCloud Infinite Scale parameters + # (https://owncloud.dev/apis/http/graph/spaces/#list-my-spaces-get-medrives) + OCIS_URL = "https://ocis.aaitest.owncloud.works" + # personal space + OCIS_PERSONAL_SPACE = "/graph/v1.0/me/drives?%24filter=driveType+eq+personal" + # shared space + OCIS_SHARED_WITH_ME = "/graph/v1.0/me/drives?%24filter=driveType+eq+virtual" + # otter spaces + OCIS_SPACES = "/graph/v1.0/me/drives?%24filter=driveType+eq+project" + + async def append_owncloud_sidecar(self, spawner, type, query, fallback_url=None, headers={}): + owncloud_url = fallback_url + http_client = AsyncHTTPClient() + req = HTTPRequest( + self.OCIS_URL + query, + headers=headers, + method="GET", + ) + try: + resp = await http_client.fetch(req) + body = json.loads(resp.body.decode("utf8", "replace")) + self.log.debug("OCIS response: %s", body) + if "value" in body: + ocis_infos = body["value"] + if len(ocis_infos) >= 1 and "root" in ocis_infos[0]: + owncloud_url = ocis_infos[0]["root"].get("webDavUrl", None) + except HTTPClientError as e: + self.log.error("can't query ownCloud: %s", e) + self.log.info("ownCloud %s URL: %s", type, owncloud_url) + + if owncloud_url is None: + return + + if type == "home": + subpath = "" + else: + subpath = "/" + type.capitalize() + env = [ + {"name": "WEBDAV_URL", "value": owncloud_url}, + {"name": "WEBDAV_VENDOR", "value": "owncloud"}, + # XXX: strict permissions needed for .local/share/jupyter/runtime/jupyter_cookie_secret + # quicker directory cache and polling + {"name": "MOUNT_OPTS", "value": "--file-perms=0600 --dir-perms=0770 --dir-cache-time=1m0s --poll-interval=0m20s"}, + {"name": "MOUNT_PATH", "value": "/owncloud" + subpath}, + # default mode is "full" + {"name": "VFS_CACHE_MODE", "value": "full"}, + ] + if type != "home": + env.append({"name": "MOUNT_WAIT_POINT", "value": "webdav-fs: /owncloud fuse.rclone"}) + volume_mounts = [ + {"mountPath": "/owncloud:shared", "name": "owncloud-home"}, + {"mountPath": self.token_mount_path, "name": self.token_secret_volume_name, "readOnly": True}, + ] + spawner.extra_containers.append( + { + "name": "owncloud-" + type, + "image": "eginotebooks/webdav-rclone-sidecar:sha-0a62679", + "args": ["bearer_token_command=cat " + self.token_path], + "env": env, + "resources": self.sidecar_resources, + "securityContext": { + "runAsUser": 1000, + "fsUser": 1000, + "fsGroup": 100, + "privileged": True, + "capabilities": {"add": ["SYS_ADMIN"]}, + }, + "volumeMounts": volume_mounts, + } + ) + + async def pre_spawn_hook(self, spawner): + await super(WebDavOIDCSpawner, self).pre_spawn_hook(spawner) + auth_state = await self.user.get_auth_state() + # volume name as in EGI spawner + self.token_secret_volume_name = self._expand_user_properties( + self.token_secret_volume_name_template + ) + self.token_path = os.path.join(self.token_mount_path, "access_token") + + if auth_state: + access_token = auth_state.get("access_token", None) + headers = { + "Accept": "application/json", + "User-Agent": "JupyterHub", + "Authorization": "Bearer %s" % access_token, + } + + await self.append_owncloud_sidecar(spawner, "home", self.OCIS_PERSONAL_SPACE, headers=headers) + await self.append_owncloud_sidecar(spawner, "shares", self.OCIS_SHARED_WITH_ME, headers=headers) + await self.append_owncloud_sidecar(spawner, "spaces", self.OCIS_SPACES, headers=headers) + else: + self.log.info("No auth state, skipping ownCloud") + + + c.JupyterHub.spawner_class = WebDavOIDCSpawner + c.B2DropSpawner.http_timeout = 90 + c.B2DropSpawner.args = ["--FileCheckpoints.checkpoint_dir='/home/jovyan/.notebookCheckpoints'"] + c.B2DropSpawner.profile_form_template = """ + <style> + /* + .profile divs holds two div tags: one for a radio button, and one + for the profile's content. + */ + #kubespawner-profiles-list .profile { + display: flex; + flex-direction: row; + font-weight: normal; + border-bottom: 1px solid #ccc; + padding-bottom: 12px; + } + + #kubespawner-profiles-list .profile .radio { + padding: 12px; + } + + /* .option divs holds a label and a select tag */ + #kubespawner-profiles-list .profile .option { + display: flex; + flex-direction: row; + align-items: center; + padding-bottom: 12px; + } + + #kubespawner-profiles-list .profile .option label { + font-weight: normal; + margin-right: 8px; + min-width: 96px; + } + </style> + + <div class='form-group' id='kubespawner-profiles-list'> + {%- for profile in profile_list %} + {#- Wrap everything in a <label> so clicking anywhere selects the option #} + <label for='profile-item-{{ profile.slug }}' class='profile'> + <div class='radio'> + <input type='radio' name='profile' id='profile-item-{{ profile.slug }}' value='{{ profile.slug }}' {% if profile.default %}checked{% endif %} /> + </div> + <div> + <h3>{{ profile.display_name }}</h3> + + {%- if profile.description %} + <p>{{ profile.description }}</p> + {%- endif %} + + {%- if profile.profile_options %} + <div> + {%- for k, option in profile.profile_options.items() %} + <div class='option'> + <label for='profile-option-{{profile.slug}}-{{k}}'>{{option.display_name}}</label> + <select name="profile-option-{{profile.slug}}-{{k}}" class="form-control"> + {%- for k, choice in option['choices'].items() %} + <option value="{{ k }}" {% if choice.default %}selected{%endif %}>{{ choice.display_name }}</option> + {%- endfor %} + </select> + </div> + {%- endfor %} + </div> + {%- endif %} + </div> + </label> + {%- endfor %} + <div class="panel-group" id="accordion" role="tablist" aria-multiselectable="true"> + <div class="panel panel-default"> + <div class="panel-heading" role="tab" id="headingOne"> + <h4 class="panel-title"> + <a class="collabpsed" role="button" data-toggle="collapse" data-parent="#accordion" href="#collapseOne" aria-expanded="false" aria-controls="collapseOne"> + B2DROP connection + </a> + {%if b2drop_ready %}<span class="label label-success">Already configured!</span>{% endif %} + </h4> + </div> + <div id="collapseOne" class="panel-collapse collapse" role="tabpanel" aria-labelledby="headingOne"> + <div class="panel-body"> + <div class='form-group'> + <label for="b2drop-user" class="form-label">B2DROP app Username</label> + <input type="text" class="form-control" name="b2drop-user" id="b2drop-user" aria-describedby="b2drop-user-help" value="{{ b2drop_user }}"> + <div id="b2drop-user-help" class="form-text">Create new app password at <a href="https://b2drop.eudat.eu/settings/user/security">B2DROP security configuration</a></div> + </div> + <div class='form-group'> + <label for="b2drop-pwd" class="form-label">B2DROP app Password</label> + <input type="password" class="form-control" name="b2drop-pwd" id="b2drop-pwd" value="{{ b2drop_pwd }}"> + </div> + </div> + </div> + </div> + </div> + """ +{% endraw %} + extraFiles: + welcome.html: + mountPath: /usr/local/share/jupyterhub/templates/welcome.html + stringData: |- +{%- raw %} + {% extends "login.html" %} +{% endraw %} diff --git a/staging1/inventory/1-psnc.yaml b/staging1/inventory/1-psnc.yaml index 250d554..22b8c29 100644 --- a/staging1/inventory/1-psnc.yaml +++ b/staging1/inventory/1-psnc.yaml @@ -1,25 +1,25 @@ --- fip: hosts: - 62.3.174.45: + 62.3.174.41: master: hosts: - 192.168.0.75: + 192.168.0.81: # must be IPv4 address or hostname - kube_server: 192.168.0.75 + kube_server: 192.168.0.81 ingress: hosts: - 192.168.0.34: + 192.168.0.210: nfs: hosts: - 192.168.0.143: + 192.168.0.139: worker: hosts: - 192.168.0.151: + 192.168.0.128: gpu: hosts: @@ -27,5 +27,5 @@ gpu: # using public IP of kube_server for ansible delegate_to kube_server: hosts: - 192.168.0.75: - ansible_host: 192.168.0.75 + 192.168.0.81: + ansible_host: 192.168.0.81 diff --git a/staging1/inventory/99-all.yaml b/staging1/inventory/99-all.yaml index 68641ba..0398c06 100644 --- a/staging1/inventory/99-all.yaml +++ b/staging1/inventory/99-all.yaml @@ -13,7 +13,7 @@ all: ansible_ssh_common_args: '-o ProxyCommand="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -W %h:%p -q egi@{{ groups["fip"][0] }}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' mail_local: true - site_name: psnc-staging + site_name: psnc-staging1 vault_mount_point: secrets/users/e1662e20-e34b-468c-b0ce-d899bc878364@egi.eu/eosc-staging notebooks_hostname: notebooks-stg1.cloud.cesnet.cz diff --git a/staging1/playbooks/notebooks.yaml b/staging1/playbooks/notebooks.yaml new file mode 120000 index 0000000..3f1a33f --- /dev/null +++ b/staging1/playbooks/notebooks.yaml @@ -0,0 +1 @@ +../../common/playbooks/notebooks.yaml \ No newline at end of file diff --git a/testing/deployments/hub.yaml b/testing/deployments/hub.yaml index a28f0ba..5d11306 100644 --- a/testing/deployments/hub.yaml +++ b/testing/deployments/hub.yaml @@ -62,12 +62,16 @@ singleuser: kubespawner_override: args: - "--CondaKernelSpecManager.env_filter='/opt/conda$'" + extra_annotations: + "egi.eu/flavor": "small-environment-2-vcpu-4-gb-ram" - display_name: Medium Environment - 4 vCPU / 8 GB RAM description: > The notebook environment includes Python, R, Julia and Octave kernels. kubespawner_override: args: - "--CondaKernelSpecManager.env_filter='/opt/conda$'" + extra_annotations: + "egi.eu/flavor": "medium-environment-4-vcpu-8-gb-ram" cpu_guarantee: 0.4 cpu_limit: 4 mem_guarantee: 1G @@ -82,6 +86,8 @@ singleuser: cpu_limit: 8 mem_guarantee: 2G mem_limit: 16G + extra_annotations: + "egi.eu/flavor": "large-environment-8-vcpu-16-gb-ram-gpu" extra_resource_guarantees: nvidia.com/gpu: 1 extra_resource_limits: @@ -93,14 +99,6 @@ singleuser: mountPath: /usr/local/bin/jupyterhub-wait-remote-home stringData: |- #! /bin/sh - - # - # Dirty hack to make remote mount on home directory working properly: - # - # 1) wait for webdav sidecar image to kick in - # 2) change directory to the mounted version of itself - # 3) launch notebook server - # i=0 while ! grep '^webdav-fs: /home/jovyan ' /proc/mounts && test $i -lt 30; do echo 'Waiting for ownClound mount...' @@ -112,6 +110,13 @@ singleuser: mountPath: /usr/local/bin/jupyterhub-singleuser-webdav-wrapper stringData: |- #! /bin/sh + # + # Dirty hack to make remote mount on home directory working properly: + # + # 1) wait for webdav sidecar image to kick in + # 2) change directory to the mounted version of itself + # 3) launch notebook server + # /usr/local/bin/jupyterhub-wait-remote-home cd . -- GitLab