From ec1e7e4ce6a8391ce15087e9ae6e48bdfb4e63bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Franti=C5=A1ek=20Dvo=C5=99=C3=A1k?= <valtri@civ.zcu.cz>
Date: Mon, 10 Mar 2025 16:59:12 +0000
Subject: [PATCH] EGI devel - initial notebooks deployment
---
egi-devel/deployments/hub.yaml | 390 +++++++++++++++++++++++++++++++++
1 file changed, 390 insertions(+)
create mode 100644 egi-devel/deployments/hub.yaml
diff --git a/egi-devel/deployments/hub.yaml b/egi-devel/deployments/hub.yaml
new file mode 100644
index 0000000..3bfc60b
--- /dev/null
+++ b/egi-devel/deployments/hub.yaml
@@ -0,0 +1,390 @@
+---
+proxy:
+ service:
+ type: NodePort
+
+ingress:
+ enabled: true
+ annotations:
+ kubernetes.io/ingress.class: "nginx"
+ kubernetes.io/tls-acme: "true"
+ hosts:
+ - "{{ notebooks_hostname }}"
+ tls:
+ - hosts:
+ - "{{ notebooks_hostname }}"
+ secretName: acme-tls-hub
+
+singleuser:
+ # keep resource limits in sync with:
+ # - profileList
+ # - EGI web-page https://www.egi.eu/service/notebooks/
+ storage:
+ capacity: 10Gi
+ dynamic:
+ pvcNameTemplate: claim-{userid}{servername}
+ volumeNameTemplate: vol-{userid}{servername}
+ storageAccessModes: ["ReadWriteMany"]
+ extraVolumes:
+ - name: cvmfs-host
+ hostPath:
+ path: /cvmfs
+ type: Directory
+ - name: b2drop
+ empty_dir:
+ extraVolumeMounts:
+ - name: cvmfs-host
+ mountPath: "/cvmfs:shared"
+ - name: b2drop
+ mountPath: '/mnt/b2drop:shared'
+ lifecycleHooks:
+ postStart:
+ exec: { "command": ["/bin/sh", "-c", "ln -snf /mnt/oneclient $HOME/datahub; ln -snf /mnt/b2drop $HOME/b2drop; ln -snf /cvmfs $HOME/cvmfs; mkdir -p /home/jovyan/.notebookCheckpoints"] }
+ memory:
+ limit: 6G
+ guarantee: 128M
+ cpu:
+ limit: 2
+ guarantee: .02
+ defaultUrl: "/lab"
+ networkPolicy:
+ egress:
+ - to:
+ - namespaceSelector:
+ matchLabels:
+ kubernetes.io/metadata.name: kube-system
+ podSelector:
+ matchLabels:
+ app.kubernetes.io/instance: cluster-ingress
+ image:
+ name: eginotebooks/single-user
+ tag: "sha-0e47d79"
+ profileList:
+ - display_name: Default EGI environment - 6 GB RAM / 2 core
+ description: >
+ The Default notebook environment includes Python, R, Julia and Octave kernels.
+ default: true
+ kubespawner_override:
+ args:
+ - "--CondaKernelSpecManager.env_filter='/opt/conda$'"
+ - display_name: EGI environment with Elyra and AI tools - 6 GB RAM / 2 core
+ description: >
+ The Default notebook environment includes Python, R, Julia and Octave kernels extended with Elyra and AI tools.
+ kubespawner_override:
+ args:
+ - "--CondaKernelSpecManager.env_filter='/opt/conda$'"
+ image: "eginotebooks/single-user-ai:sha-0e47d79"
+ - display_name: RELIANCE project environment - 12 GB RAM / 2 core
+ description: >
+ Notebook environment for RELIANCE project includes Python, R, Julia and Octave kernels
+ kubespawner_override:
+ args:
+ - "--CondaKernelSpecManager.env_filter='/opt/conda$'"
+ mem_guarantee: 2G
+ mem_limit: 12G
+ vo_claims:
+ - urn:mace:egi.eu:group:notebooks-support#sso.egi.eu
+ - urn:mace:egi.eu:group:vo.reliance-project.eu:role=member#aai.egi.eu
+ - urn:mace:egi.eu:www.egi.eu:notebooks-support:member@egi.eu
+ - urn:mace:egi.eu:group:notebooks-support#sso.egi.eu
+ # TODO: MatLab
+ - display_name: EISCAT environment - 4 GB RAM / 2 cores
+ description: >
+ The EISCAT environment.
+ kubespawner_override:
+ image: "ingemarh/guisdap"
+ imagePullPolicy: Always
+ vo_claims:
+ - urn:mace:egi.eu:group:cc-eiscat3d#sso.egi.eu
+ - urn:mace:egi.eu:group:eiscat.se:Hub:role=member#aai.egi.eu
+ - urn:mace:egi.eu:group:notebooks-support#sso.egi.eu
+ - urn:mace:egi.eu:www.egi.eu:notebooks-support:member@egi.eu
+ - urn:mace:egi.eu:group:notebooks-support#sso.egi.eu
+
+hub:
+ services:
+ status:
+ url: "http://status-web/"
+ admin: true
+ # recommended to keep in sync with common/playbooks/files/jupyterhub-jwt.yaml
+ # keep k8s-hub version in sync with ../playbooks/notebooks.yaml
+ image:
+ name: eginotebooks/hub
+ # k8s-hub 4.0.0
+ tag: "sha-aef23d2"
+ config:
+ Authenticator:
+ enable_auth_state: true
+ admin_users:
+ - 529a87e5ce04cd5ddd7161734d02df0e2199a11452430803e714cb1309cc3907@egi.eu
+ - 025166931789a0f57793a6092726c2ad89387a4cc167e7c63c5d85fc91021d18@egi.eu
+ - 7ce47695f1e7fc91a1156e672f4a47576559938cdbe840355e2429e3a05b4ff8@egi.eu
+ # fdvorak2 @ aai.egi.eu
+ - 52cc7599bd1553c9d63e34e4c90b7e84d44967490c28bb4c53fe97b0c881d677@egi.eu
+ # fdvorak2 @ aai-dev.egi.eu
+ - c481e0a85e1ae0a5a1480a63e62295ca2f9ac652244947995bd4a0210fbcb77c@egi.eu
+ # jhradil3 @ aai-dev.egi.eu
+ - 240c0594fe34ac26cffd82fd0ad85f29d9ad9dfbb46febb05ed42db0bff594d1@egi.eu
+ # keep in sync with:
+ # - cesnet/playbooks/templates/binder.yaml
+ # - documentation/content/en/users/dev-env/notebooks/_index.md
+ allowed_groups:
+ # EISCAT Dirac testing
+ - urn:mace:egi.eu:group:cc-eiscat3d#sso.egi.eu
+ - urn:mace:egi.eu:group:auger:role=member#aai.egi.eu
+ - urn:mace:egi.eu:group:biomed:role=member#aai.egi.eu
+ - urn:mace:egi.eu:group:vo.cessda.eduteams.org:role=member#aai.egi.eu
+ - urn:mace:egi.eu:group:eiscat.se:Hub:role=member#aai.egi.eu
+ - urn:mace:egi.eu:group:eval.c-scale.eu:role=member#aai.egi.eu
+ - urn:mace:egi.eu:group:vo.environmental.egi.eu:role=member#aai.egi.eu
+ - urn:mace:egi.eu:group:vo.lethe-project.eu:lethe-notebooks:role=member#aai.egi.eu
+ - urn:mace:egi.eu:group:vo.panosc.eu:role=vm_operator#aai.egi.eu
+ - urn:mace:egi.eu:group:vo.reliance-project.eu:role=member#aai.egi.eu
+ - urn:mace:egi.eu:group:vo.access.egi.eu:role=member#aai.egi.eu
+ - urn:mace:egi.eu:group:vo.notebooks.egi.eu:role=member#aai.egi.eu
+ - urn:mace:egi.eu:group:vo.bioexcel.eu:role=member#aai.egi.eu
+ - urn:mace:egi.eu:group:vo.egu2024.egi.eu:role=member#aai.egi.eu
+ - urn:mace:egi.eu:www.egi.eu:fedcloud-users:member@egi.eu
+ - urn:mace:egi.eu:www.egi.eu:techsolutions:member@egi.eu
+ # changed 2022-10
+ - urn:mace:egi.eu:group:fedcloud-users#sso.egi.eu
+ - urn:mace:egi.eu:group:supplier-notebooks#sso.egi.eu
+ - urn:mace:egi.eu:group:techsolutions#sso.egi.eu
+ - urn:mace:egi.eu:group:notebooks-support#sso.egi.eu
+ auto_login: true
+ claim_groups_key: "entitlements"
+ OnedataAuthenticator:
+ oneprovider_host: "cesnet-oneprovider-01.datahub.egi.eu"
+ authorize_url: "https://{{ secrets['checkin_host'] }}/auth/realms/egi/protocol/openid-connect/auth"
+ token_url: "https://{{ secrets['checkin_host'] }}/auth/realms/egi/protocol/openid-connect/token"
+ userdata_url: "https://{{ secrets['checkin_host'] }}/auth/realms/egi/protocol/openid-connect/userinfo"
+ client_id: "{{ secrets['client_id'] }}"
+ client_secret: "{{ secrets['client_secret'] }}"
+ oauth_callback_url: "https://{{ notebooks_hostname }}/hub/oauth_callback"
+ scope: ["openid", "profile", "email", "offline_access", "eduperson_scoped_affiliation", "eduperson_entitlement"]
+ username_key: "sub"
+ OnedataSpawner:
+ sidecar_image: "eginotebooks/oneclient-sidecar:sha-9789b9a"
+ force_direct_io: true
+ http_timeout: 60
+ args:
+ - "--FileCheckpoints.checkpoint_dir='/home/jovyan/.notebookCheckpoints'"
+ JupyterHub:
+ admin_access: true
+ authenticate_prometheus: false
+ authenticator_class: egi_notebooks_hub.onedata.OnedataAuthenticator
+ # spawner_class: (in egi-notebooks-b2drop)
+ extraConfig:
+ egi-notebooks-welcome: |-
+ from egi_notebooks_hub.welcome import WelcomeHandler
+ c.JupyterHub.default_url = "/welcome"
+ c.JupyterHub.extra_handlers = [(r'/welcome', WelcomeHandler)]
+ egi-notebooks-b2drop: |-
+{%- raw %}
+ import base64
+ from jinja2 import BaseLoader
+ from jinja2 import Environment
+ from egi_notebooks_hub.onedata import OnedataSpawner
+ from kubernetes_asyncio.client.rest import ApiException
+
+ class B2DropSpawner(OnedataSpawner):
+ async def auth_state_hook(self, spawner, auth_state):
+ await super().auth_state_hook(spawner, auth_state)
+ self.b2drop_ready = False
+ self.b2drop_user = ""
+ self.b2drop_pwd = ""
+ try:
+ secret = await self.api.read_namespaced_secret(self.token_secret_name, self.namespace)
+ except ApiException:
+ return
+ if secret and secret.data:
+ self.b2drop_user = base64.b64decode(secret.data.get("b2drop-user", "")).decode()
+ self.b2drop_pwd = base64.b64decode(secret.data.get("b2drop-pwd", "")).decode()
+ self.b2drop_ready = (self.b2drop_user and self.b2drop_pwd)
+
+ def _render_options_form(self, profile_list):
+ # old hub: self._profile_list = self._init_profile_list(profile_list)
+ self._profile_list = self._get_initialized_profile_list(profile_list)
+ profile_form_template = Environment(loader=BaseLoader).from_string(
+ self.profile_form_template
+ )
+ return profile_form_template.render(profile_list=self._profile_list, b2drop_ready=self.b2drop_ready, b2drop_user=self.b2drop_user, b2drop_pwd=self.b2drop_pwd)
+
+ async def pre_spawn_hook(self, spawner):
+ await super(B2DropSpawner, self).pre_spawn_hook(spawner)
+ b2drop_user = self.user_options.get("b2drop-user", "")
+ b2drop_pwd = self.user_options.get("b2drop-pwd", "")
+ b2drop_remember = self.user_options.get("b2drop-remember", None)
+ if not (b2drop_user and b2drop_pwd):
+ secret = await self.api.read_namespaced_secret(self.token_secret_name, self.namespace)
+ if secret and secret.data:
+ b2drop_user = base64.b64decode(secret.data.get("b2drop-user", "")).decode()
+ b2drop_pwd = base64.b64decode(secret.data.get("b2drop-pwd", "")).decode()
+ if b2drop_user and b2drop_pwd:
+ volume_mounts = [
+ {"mountPath": "/b2drop:shared", "name": "b2drop"},
+ ]
+ spawner.extra_containers.append(
+ {
+ "name": "b2drop",
+ "image": "eginotebooks/webdav-sidecar:sha-e5e8df2",
+ "env": [
+ {"name": "WEBDAV_URL", "value": "https://b2drop.eudat.eu/remote.php/webdav"},
+ {"name": "WEBDAV_PWD", "value": b2drop_pwd},
+ {"name": "WEBDAV_USER", "value": b2drop_user},
+ {"name": "MOUNT_PATH", "value": "/b2drop"},
+ ],
+ "resources": self.sidecar_resources,
+ # "command": cmd,
+ "securityContext": {
+ "runAsUser": 0,
+ "privileged": True,
+ "capabilities": {"add": ["SYS_ADMIN"]},
+ },
+ "volumeMounts": volume_mounts,
+ "lifecycle": {
+ "preStop": {
+ "exec": {"command": ["umount", "-l", "/b2drop"]}
+ },
+ },
+ }
+ )
+ if b2drop_remember:
+ await self._update_secret({"b2drop-user": b2drop_user,
+ "b2drop-pwd": b2drop_pwd})
+ else:
+ await self._update_secret({"b2drop-user": "", "b2drop-pwd": ""})
+
+ def options_from_form(self, formdata):
+ data = super(B2DropSpawner, self)._options_from_form(formdata)
+ data.update({'b2drop-user': formdata.get('b2drop-user', [None])[0],
+ 'b2drop-remember': formdata.get('b2drop-remember', [None])[0],
+ 'b2drop-pwd': formdata.get('b2drop-pwd', [None])[0]})
+ return data
+
+ c.JupyterHub.spawner_class = B2DropSpawner
+ c.B2DropSpawner.http_timeout = 60
+ c.B2DropSpawner.args = ["--FileCheckpoints.checkpoint_dir='/home/jovyan/.notebookCheckpoints'"]
+ c.B2DropSpawner.profile_form_template = """
+ <style>
+ /*
+ .profile divs holds two div tags: one for a radio button, and one
+ for the profile's content.
+ */
+ #kubespawner-profiles-list .profile {
+ display: flex;
+ flex-direction: row;
+ font-weight: normal;
+ border-bottom: 1px solid #ccc;
+ padding-bottom: 12px;
+ }
+
+ #kubespawner-profiles-list .profile .radio {
+ padding: 12px;
+ }
+
+ /* .option divs holds a label and a select tag */
+ #kubespawner-profiles-list .profile .option {
+ display: flex;
+ flex-direction: row;
+ align-items: center;
+ padding-bottom: 12px;
+ }
+
+ #kubespawner-profiles-list .profile .option label {
+ font-weight: normal;
+ margin-right: 8px;
+ min-width: 96px;
+ }
+ </style>
+
+ <div class='form-group' id='kubespawner-profiles-list'>
+ {%- for profile in profile_list %}
+ {#- Wrap everything in a <label> so clicking anywhere selects the option #}
+ <label for='profile-item-{{ profile.slug }}' class='profile'>
+ <div class='radio'>
+ <input type='radio' name='profile' id='profile-item-{{ profile.slug }}' value='{{ profile.slug }}' {% if profile.default %}checked{% endif %} />
+ </div>
+ <div>
+ <h3>{{ profile.display_name }}</h3>
+
+ {%- if profile.description %}
+ <p>{{ profile.description }}</p>
+ {%- endif %}
+
+ {%- if profile.profile_options %}
+ <div>
+ {%- for k, option in profile.profile_options.items() %}
+ <div class='option'>
+ <label for='profile-option-{{profile.slug}}-{{k}}'>{{option.display_name}}</label>
+ <select name="profile-option-{{profile.slug}}-{{k}}" class="form-control">
+ {%- for k, choice in option['choices'].items() %}
+ <option value="{{ k }}" {% if choice.default %}selected{%endif %}>{{ choice.display_name }}</option>
+ {%- endfor %}
+ </select>
+ </div>
+ {%- endfor %}
+ </div>
+ {%- endif %}
+ </div>
+ </label>
+ {%- endfor %}
+ <div class="panel-group" id="accordion" role="tablist" aria-multiselectable="true">
+ <div class="panel panel-default">
+ <div class="panel-heading" role="tab" id="headingOne">
+ <h4 class="panel-title">
+ <a class="collabpsed" role="button" data-toggle="collapse" data-parent="#accordion" href="#collapseOne" aria-expanded="false" aria-controls="collapseOne">
+ B2DROP connection
+ </a>
+ {%if b2drop_ready %}<span class="label label-success">Already configured!</span>{% endif %}
+ </h4>
+ </div>
+ <div id="collapseOne" class="panel-collapse collapse" role="tabpanel" aria-labelledby="headingOne">
+ <div class="panel-body">
+ <div class='form-group'>
+ <label for="b2drop-user" class="form-label">B2DROP app Username</label>
+ <input type="text" class="form-control" name="b2drop-user" id="b2drop-user" aria-describedby="b2drop-user-help" value="{{ b2drop_user }}">
+ <div id="b2drop-user-help" class="form-text">Create new app password at <a href="https://b2drop.eudat.eu/settings/user/security">B2DROP security configuration</a></div>
+ </div>
+ <div class='form-group'>
+ <label for="b2drop-pwd" class="form-label">B2DROP app Password</label>
+ <input type="password" class="form-control" name="b2drop-pwd" id="b2drop-pwd" value="{{ b2drop_pwd }}">
+ </div>
+ <div class='form-group'>
+ <input type="checkbox" id="b2drop-remember" name="b2drop-remember" {%if b2drop_ready %}checked{% endif %}>
+ <label class="form-check-label" for="from-check-input">Remember B2DROP credentials</label>
+ </div>
+ </div>
+ </div>
+ </div>
+ </div>
+ """
+{% endraw %}
+ templatePaths:
+ - /egi-notebooks-hub/templates
+ extraFiles:
+ login.html:
+ mountPath: /egi-notebooks-hub/templates/login.html
+ stringData: |-
+{%- raw %}
+ {% extends "egi-login.html" %}
+ {% block main_intro %}
+ <h1><img alt="Notebooks Logo" src="{{ static_url('images/egi-icon-notebooks.svg') }}"
+ height="100">Notebooks</h1>
+ <p>
+ Notebooks is an environment based on <a href="http://jupyter.org/">Jupyter</a> and
+ the <a href="https://www.egi.eu/services/cloud-compute/">EGI cloud service</a> that
+ offers a browser-based, scalable tool for interactive data analysis. The Notebooks
+ environment provides users with notebooks where they can combine text, mathematics,
+ computations and rich media output.
+ </p>
+ <p>
+ Access requires a valid <a href="https://docs.egi.eu/users/check-in/signup">EGI account</a>
+ and <a href="https://docs.egi.eu/users/dev-env/notebooks/#notebooks-for-researchers">
+ enrolling to one of the supported VOs</a>.
+ </p>
+ <p>
+ Default environment provides 4 vCPU cores, 6 GB RAM and 10GB of personal storage space per user.
+ </p>
+ {% endblock main_intro %}
+{% endraw %}
--
GitLab