From 0c0439158e0e9a748ff83bbd4181429ea199fa33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franti=C5=A1ek=20Dvo=C5=99=C3=A1k?= <valtri@civ.zcu.cz> Date: Mon, 23 Sep 2024 17:29:50 +0000 Subject: [PATCH] Deployment for Jupyter5 collaboration --- eosc-devel/deployments/collab5.yaml | 366 ++++++++++++++++++++++++++++ eosc-devel/deployments/fullhub.yaml | 4 +- eosc-devel/inventory/99-all.yaml | 1 - 3 files changed, 368 insertions(+), 3 deletions(-) create mode 100644 eosc-devel/deployments/collab5.yaml diff --git a/eosc-devel/deployments/collab5.yaml b/eosc-devel/deployments/collab5.yaml new file mode 100644 index 0000000..29d6401 --- /dev/null +++ b/eosc-devel/deployments/collab5.yaml @@ -0,0 +1,366 @@ +--- +proxy: + service: + type: NodePort + +ingress: + enabled: true + annotations: + kubernetes.io/ingress.class: "nginx" + kubernetes.io/tls-acme: "true" + hosts: + - "eosc.zcu.cz" + tls: + - hosts: + - "eosc.zcu.cz" + secretName: acme-tls-fullhub + +singleuser: + # keep resource limits in sync with: + # - profileList + storage: + type: none + extraVolumes: + - name: cvmfs-host + hostPath: + path: /cvmfs + type: Directory + - name: owncloud-home + empty_dir: + extraVolumeMounts: + - name: cvmfs-host + mountPath: "/cvmfs:shared" + - name: owncloud-home + mountPath: '/home/jovyan:shared' + memory: + limit: 4G + guarantee: 128M + cpu: + limit: 2 + guarantee: .02 + defaultUrl: "/lab" + networkPolicy: + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + serviceSelector: + matchLabels: + k8s-app: cluster-ingress + image: + name: eginotebooks/single-user-eosc + tag: "sha-dea4fa2" + profileList: + - display_name: Small Environment - 2 vCPU / 4 GB RAM (non-collaboratice) + description: > + The notebook environment includes Python, R, Julia and Octave kernels. Non-collaborative. + default: true + kubespawner_override: + args: + - "--CondaKernelSpecManager.env_filter='/opt/conda$'" + extra_annotations: + "egi.eu/flavor": "small-environment-2-vcpu-4-gb-ram" + - display_name: Small Environment - 2 vCPU / 4 GB RAM (collaboratice) + description: > + The notebook environment includes Python, R, Julia and Octave kernels. Collaborative. + kubespawner_override: + args: + - "--CondaKernelSpecManager.env_filter='/opt/conda$'" + image: "valtri/single-user:jupyter-4e-collab" + extra_annotations: + "egi.eu/flavor": "small-environment-2-vcpu-4-gb-ram" + - display_name: Small Environment - 2 vCPU / 4 GB RAM + description: > + The notebook environment includes Python, R, Julia and Octave kernels. + default: true + kubespawner_override: + args: + - "--CondaKernelSpecManager.env_filter='/opt/conda$'" + extra_annotations: + "egi.eu/flavor": "small-environment-2-vcpu-4-gb-ram" + vo_claims: + - urn:geant:eosc-federation.eu:res:notebooks.open-science-cloud.ec.europa.eu:2-vcpu-4-gb-ram:act:ppa + - display_name: Medium Environment - 4 vCPU / 8 GB RAM + description: > + The notebook environment includes Python, R, Julia and Octave kernels. + kubespawner_override: + args: + - "--CondaKernelSpecManager.env_filter='/opt/conda$'" + extra_annotations: + "egi.eu/flavor": "medium-environment-4-vcpu-8-gb-ram" + cpu_guarantee: 0.4 + cpu_limit: 4 + mem_guarantee: 1G + mem_limit: 8G + vo_claims: + - urn:geant:eosc-federation.eu:res:notebooks.open-science-cloud.ec.europa.eu:4-vcpu-8-gb-ram:act:ppa + - display_name: Large Environment - 8 vCPU / 16 GB RAM / GPU + description: > + The notebook environment includes Python, R, Julia and Octave kernels with GPU. + kubespawner_override: + args: + - "--CondaKernelSpecManager.env_filter='/opt/conda$'" + cpu_guarantee: 0.8 + cpu_limit: 8 + mem_guarantee: 2G + mem_limit: 16G + extra_annotations: + "egi.eu/flavor": "large-environment-8-vcpu-16-gb-ram-gpu" + extra_resource_guarantees: + nvidia.com/gpu: 1 + extra_resource_limits: + nvidia.com/gpu: 1 + vo_claims: + - urn:geant:eosc-federation.eu:res:notebooks.open-science-cloud.ec.europa.eu:8-vcpu-16-gb-ram-gpu:act:ppa + cmd: jupyterhub-singleuser-webdav-wrapper + extraFiles: + wait-remote-home.sh: + mode: 0755 + mountPath: /usr/local/bin/jupyterhub-wait-remote-home + stringData: |- + #! /bin/sh + i=0 + while ! grep '^webdav-fs: /home/jovyan ' /proc/mounts && test $i -lt 30; do + echo 'Waiting for ownClound mount...' + sleep 0.5 + i=$((i+1)) + done + singleuser-webdav-wrapper.sh: + mode: 0755 + mountPath: /usr/local/bin/jupyterhub-singleuser-webdav-wrapper + # NotebookNotary.db_file=':memory:' is used due to issues + # notebook notary file was causing in ~/.jupyter in ownCloud mount + # + # LabApp.custom_css=True allows to use custom CSS for EOSC style + # + # ResourceUseDisplay.mem_warning_threshold=0.25 sets for resource-usage + # extension to warn about used memory when only 25% of memory is available + # which is also used by EGI notebooks-resource-warning extension + stringData: |- + #! /bin/sh + # + # Dirty hack to make remote mount on home directory working properly: + # + # 1) wait for webdav sidecar image to kick in + # 2) change directory to the mounted version of itself + # 3) launch notebook server + # + /usr/local/bin/jupyterhub-wait-remote-home + + # Disables RTC extension. To enable it set this env variable in kubespawner_override + # to JUPYTERHUB_ALLOW_TOKEN_IN_URL="1" + if [ -z "$JUPYTERHUB_ALLOW_TOKEN_IN_URL" ]; then + jupyter-labextension disable @jupyter/collaboration-extension + jupyter-labextension lock @jupyter/collaboration-extension + fi + + cd . + exec jupyterhub-singleuser \ + --FileCheckpoints.checkpoint_dir='/home/jovyan/.notebookCheckpoints' \ + --NotebookNotary.db_file=':memory:' \ + --LabApp.custom_css=True \ + --ResourceUseDisplay.mem_warning_threshold=0.25 \ + "$@" + +hub: + services: + status: + url: "http://status-web/" + admin: true + jwt: + url: "http://jwt/" + display: false + eosc-monitor: + admin: true + display: false + api_token: "{{ secrets['zabbix_token'] }}" + # recommended to keep in sync with common/playbooks/files/jupyterhub-jwt.yaml + # keep k8s-hub version in sync with ../playbooks/notebooks.yaml + image: + name: eginotebooks/hub + # k8s-hub 4.0.0 + tag: "sha-6edf89c" + config: + Authenticator: + enable_auth_state: true + admin_users: + # valtri@civ.zcu.cz + - c36b18fe-e03a-4a22-ab14-5965e0171410@eosc-federation.eu + # Monitor user for Zabbix + - eosc-monitor + allowed_groups: + - urn:geant:eosc-federation.eu:res:notebooks.open-science-cloud.ec.europa.eu:2-vcpu-4-gb-ram:act:ppa + - urn:geant:eosc-federation.eu:res:notebooks.open-science-cloud.ec.europa.eu:4-vcpu-8-gb-ram:act:ppa + - urn:geant:eosc-federation.eu:res:notebooks.open-science-cloud.ec.europa.eu:8-vcpu-16-gb-ram-gpu:act:ppa + - urn:geant:eosc-federation.eu:testing:group:eosc + admin_groups: + - urn:geant:eosc-federation.eu:group:asg:notebooks.open-science-cloud.ec.europa.eu:role=admin + claim_groups_key: "entitlements" + auth_state_groups_key: "oauth_user.entitlements" + EGICheckinAuthenticator: + checkin_host: "{{ secrets['checkin_host'] }}" + authorize_url: "https://{{ secrets['checkin_host'] }}/OIDC/authorization" + token_url: "https://{{ secrets['checkin_host'] }}/OIDC/token" + userdata_url: "https://{{ secrets['checkin_host'] }}/OIDC/userinfo" + introspect_url: "https://{{ secrets['checkin_host'] }}/OIDC/introspect" + client_id: "{{ secrets['client_id'] }}" + client_secret: "{{ secrets['client_secret'] }}" + oauth_callback_url: "https://{{ notebooks_hostname }}/hub/oauth_callback" + openid_configuration_url: "https://{{ secrets['checkin_host'] }}/.well-known/openid-configuration" + scope: ["openid", "profile", "email", "offline_access", "entitlements"] + username_claim: "sub" + extra_authorize_params: + prompt: consent + JupyterHub: + admin_access: true + authenticate_prometheus: false + authenticator_class: egi_notebooks_hub.egiauthenticator.EOSCNodeAuthenticator + # spawner_class: (in egi-notebooks-b2drop) + LabApp: + check_for_updates_class: jupyterlab.NeverCheckForUpdate + extraConfig: + egi-notebooks-welcome: |- + from egi_notebooks_hub.welcome import WelcomeHandler + c.JupyterHub.default_url = "/welcome" + c.JupyterHub.extra_handlers = [(r'/welcome', WelcomeHandler)] + egi-notebooks-b2drop: |- +{%- raw %} + import json + from egi_notebooks_hub.onedata import OnedataSpawner + from tornado.httpclient import AsyncHTTPClient, HTTPClientError, HTTPRequest + + class WebDavOIDCSpawner(OnedataSpawner): + # ownCloud Infinite Scale parameters + # (https://owncloud.dev/apis/http/graph/spaces/#list-my-spaces-get-medrives) + OCIS_URL = "https://ocis-testing.apps.bst2-test.paas.psnc.pl" + # personal space + OCIS_PERSONAL_SPACE = "/graph/v1.0/me/drives?%24filter=driveType+eq+personal" + # shared space + OCIS_SHARED_WITH_ME = "/graph/v1.0/me/drives?%24filter=driveType+eq+virtual" + # otter spaces + OCIS_SPACES = "/graph/v1.0/me/drives?%24filter=driveType+eq+project" + + async def append_owncloud_sidecar(self, spawner, type, query, fallback_url=None, headers={}): + owncloud_url = fallback_url + http_client = AsyncHTTPClient() + req = HTTPRequest( + self.OCIS_URL + query, + headers=headers, + method="GET", + ) + try: + resp = await http_client.fetch(req) + body = json.loads(resp.body.decode("utf8", "replace")) + self.log.debug("OCIS response: %s", body) + if "value" in body: + ocis_infos = body["value"] + if len(ocis_infos) >= 1 and "root" in ocis_infos[0]: + owncloud_url = ocis_infos[0]["root"].get("webDavUrl", None) + except HTTPClientError as e: + self.log.error("can't query ownCloud: %s", e) + self.log.info("ownCloud %s URL: %s", type, owncloud_url) + + if owncloud_url is None: + return + + if type == "home": + # Jupyter side + subpath = "" + # ownCloud backend side + remote_path = "/notebooks_service" + else: + # Jupyter side + subpath = "/" + type.capitalize() + # ownCloud backend side + remote_path = "/" + env = [ + {"name": "WEBDAV_URL", "value": owncloud_url}, + {"name": "WEBDAV_VENDOR", "value": "owncloud"}, + # XXX: strict permissions needed for .local/share/jupyter/runtime/jupyter_cookie_secret + # quicker directory cache and polling + {"name": "MOUNT_OPTS", "value": "--file-perms=0600 --dir-perms=0770 --dir-cache-time=1m0s --poll-interval=0m20s"}, + {"name": "MOUNT_PATH", "value": "/owncloud" + subpath}, + # default mode is "full" + {"name": "VFS_CACHE_MODE", "value": "full"}, + # remote path to mount on ownCloud backend + {"name": "REMOTE_PATH", "value": remote_path} + ] + if type != "home": + env.append({"name": "MOUNT_WAIT_POINT", "value": "webdav-fs: /owncloud fuse.rclone"}) + volume_mounts = [ + {"mountPath": "/owncloud:shared", "name": "owncloud-home"}, + {"mountPath": self.token_mount_path, "name": self.token_secret_volume_name, "readOnly": True}, + ] + spawner.extra_containers.append( + { + "name": "owncloud-" + type, + # To be changed. This is temporary image with + # rclone fix for ownCloud not yet upstreamed + "image":"eginotebooks/webdav-rclone-sidecar-forked:1.2", + "args": ["bearer_token_command=cat " + self.token_path], + "env": env, + "resources": self.sidecar_resources, + "securityContext": { + "runAsUser": 1000, + "fsUser": 1000, + "fsGroup": 100, + "privileged": True, + "capabilities": {"add": ["SYS_ADMIN"]}, + }, + "volumeMounts": volume_mounts, + } + ) + + async def pre_spawn_hook(self, spawner): + await super(WebDavOIDCSpawner, self).pre_spawn_hook(spawner) + auth_state = await self.user.get_auth_state() + # volume name as in EGI spawner + self.token_secret_volume_name = self._expand_user_properties( + self.token_secret_volume_name_template + ) + self.token_path = os.path.join(self.token_mount_path, "access_token") + + if auth_state: + access_token = auth_state.get("access_token", None) + headers = { + "Accept": "application/json", + "User-Agent": "JupyterHub", + "Authorization": "Bearer %s" % access_token, + } + + await self.append_owncloud_sidecar(spawner, "home", self.OCIS_PERSONAL_SPACE, headers=headers) + await self.append_owncloud_sidecar(spawner, "shares", self.OCIS_SHARED_WITH_ME, headers=headers) + await self.append_owncloud_sidecar(spawner, "spaces", self.OCIS_SPACES, headers=headers) + else: + self.log.info("No auth state, skipping ownCloud") + + c.JupyterHub.spawner_class = WebDavOIDCSpawner + c.WebDavOIDCSpawner.token_mount_path = "/var/run/secrets/oidc/" + c.WebDavOIDCSpawner.http_timeout = 90 +{% endraw %} + templatePaths: + - /egi-notebooks-hub/ec-templates + extraFiles: + welcome.html: + mountPath: /usr/local/share/jupyterhub/templates/welcome.html + stringData: |- +{%- raw %} + {% extends "login.html" %} +{% endraw %} + 403.html: + mountPath: /usr/local/share/jupyterhub/templates/403.html + stringData: |- +{%- raw %} + {% extends "error.html" %} + {% block main %} + <div class="error"> + <h1>Unauthorized</h1> + <p>You don't have the correct entitlements to access this service.</p> + <p>If you think you should be granted access, please open an issue!</p> + </div> + {% endblock %} +{% endraw %} + +debug: + enabled: true \ No newline at end of file diff --git a/eosc-devel/deployments/fullhub.yaml b/eosc-devel/deployments/fullhub.yaml index 67f34a4..cc2919a 100644 --- a/eosc-devel/deployments/fullhub.yaml +++ b/eosc-devel/deployments/fullhub.yaml @@ -9,10 +9,10 @@ ingress: kubernetes.io/ingress.class: "nginx" kubernetes.io/tls-acme: "true" hosts: - - "{{ notebooks_hostname }}" + - "fullhub.eosc.zcu.cz" tls: - hosts: - - "{{ notebooks_hostname }}" + - "fullhub.eosc.zcu.cz" secretName: acme-tls-fullhub singleuser: diff --git a/eosc-devel/inventory/99-all.yaml b/eosc-devel/inventory/99-all.yaml index 5b1474d..05497cc 100644 --- a/eosc-devel/inventory/99-all.yaml +++ b/eosc-devel/inventory/99-all.yaml @@ -15,7 +15,6 @@ all: site_name: cesnet-central vault_mount_point: secrets/users/e1662e20-e34b-468c-b0ce-d899bc878364@egi.eu/eosc-dev - notebooks_hostname: fullhub.eosc.zcu.cz binder_hostname: replay.eosc.zcu.cz old_binder_hostname: binder.eosc.zcu.cz docker2_hostname: registry.eosc.zcu.cz -- GitLab