Skip to content
Snippets Groups Projects
Commit ec1e7e4c authored by František Dvořák's avatar František Dvořák
Browse files

EGI devel - initial notebooks deployment

parent 577e6982
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
egi-devel/deployments/hub.yaml 0 → 100644
+ 390
− 0
View file @ ec1e7e4c
---
proxy:
service:
type: NodePort
ingress:
enabled: true
annotations:
kubernetes.io/ingress.class: "nginx"
kubernetes.io/tls-acme: "true"
hosts:
- "{{ notebooks_hostname }}"
tls:
- hosts:
- "{{ notebooks_hostname }}"
secretName: acme-tls-hub
singleuser:
# keep resource limits in sync with:
# - profileList
# - EGI web-page https://www.egi.eu/service/notebooks/
storage:
capacity: 10Gi
dynamic:
pvcNameTemplate: claim-{userid}{servername}
volumeNameTemplate: vol-{userid}{servername}
storageAccessModes: ["ReadWriteMany"]
extraVolumes:
- name: cvmfs-host
hostPath:
path: /cvmfs
type: Directory
- name: b2drop
empty_dir:
extraVolumeMounts:
- name: cvmfs-host
mountPath: "/cvmfs:shared"
- name: b2drop
mountPath: '/mnt/b2drop:shared'
lifecycleHooks:
postStart:
exec: { "command": ["/bin/sh", "-c", "ln -snf /mnt/oneclient $HOME/datahub; ln -snf /mnt/b2drop $HOME/b2drop; ln -snf /cvmfs $HOME/cvmfs; mkdir -p /home/jovyan/.notebookCheckpoints"] }
memory:
limit: 6G
guarantee: 128M
cpu:
limit: 2
guarantee: .02
defaultUrl: "/lab"
networkPolicy:
egress:
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
podSelector:
matchLabels:
app.kubernetes.io/instance: cluster-ingress
image:
name: eginotebooks/single-user
tag: "sha-0e47d79"
profileList:
- display_name: Default EGI environment - 6 GB RAM / 2 core
description: >
The Default notebook environment includes Python, R, Julia and Octave kernels.
default: true
kubespawner_override:
args:
- "--CondaKernelSpecManager.env_filter='/opt/conda$'"
- display_name: EGI environment with Elyra and AI tools - 6 GB RAM / 2 core
description: >
The Default notebook environment includes Python, R, Julia and Octave kernels extended with Elyra and AI tools.
kubespawner_override:
args:
- "--CondaKernelSpecManager.env_filter='/opt/conda$'"
image: "eginotebooks/single-user-ai:sha-0e47d79"
- display_name: RELIANCE project environment - 12 GB RAM / 2 core
description: >
Notebook environment for RELIANCE project includes Python, R, Julia and Octave kernels
kubespawner_override:
args:
- "--CondaKernelSpecManager.env_filter='/opt/conda$'"
mem_guarantee: 2G
mem_limit: 12G
vo_claims:
- urn:mace:egi.eu:group:notebooks-support#sso.egi.eu
- urn:mace:egi.eu:group:vo.reliance-project.eu:role=member#aai.egi.eu
- urn:mace:egi.eu:www.egi.eu:notebooks-support:member@egi.eu
- urn:mace:egi.eu:group:notebooks-support#sso.egi.eu
# TODO: MatLab
- display_name: EISCAT environment - 4 GB RAM / 2 cores
description: >
The EISCAT environment.
kubespawner_override:
image: "ingemarh/guisdap"
imagePullPolicy: Always
vo_claims:
- urn:mace:egi.eu:group:cc-eiscat3d#sso.egi.eu
- urn:mace:egi.eu:group:eiscat.se:Hub:role=member#aai.egi.eu
- urn:mace:egi.eu:group:notebooks-support#sso.egi.eu
- urn:mace:egi.eu:www.egi.eu:notebooks-support:member@egi.eu
- urn:mace:egi.eu:group:notebooks-support#sso.egi.eu
hub:
services:
status:
url: "http://status-web/"
admin: true
# recommended to keep in sync with common/playbooks/files/jupyterhub-jwt.yaml
# keep k8s-hub version in sync with ../playbooks/notebooks.yaml
image:
name: eginotebooks/hub
# k8s-hub 4.0.0
tag: "sha-aef23d2"
config:
Authenticator:
enable_auth_state: true
admin_users:
- 529a87e5ce04cd5ddd7161734d02df0e2199a11452430803e714cb1309cc3907@egi.eu
- 025166931789a0f57793a6092726c2ad89387a4cc167e7c63c5d85fc91021d18@egi.eu
- 7ce47695f1e7fc91a1156e672f4a47576559938cdbe840355e2429e3a05b4ff8@egi.eu
# fdvorak2 @ aai.egi.eu
- 52cc7599bd1553c9d63e34e4c90b7e84d44967490c28bb4c53fe97b0c881d677@egi.eu
# fdvorak2 @ aai-dev.egi.eu
- c481e0a85e1ae0a5a1480a63e62295ca2f9ac652244947995bd4a0210fbcb77c@egi.eu
# jhradil3 @ aai-dev.egi.eu
- 240c0594fe34ac26cffd82fd0ad85f29d9ad9dfbb46febb05ed42db0bff594d1@egi.eu
# keep in sync with:
# - cesnet/playbooks/templates/binder.yaml
# - documentation/content/en/users/dev-env/notebooks/_index.md
allowed_groups:
# EISCAT Dirac testing
- urn:mace:egi.eu:group:cc-eiscat3d#sso.egi.eu
- urn:mace:egi.eu:group:auger:role=member#aai.egi.eu
- urn:mace:egi.eu:group:biomed:role=member#aai.egi.eu
- urn:mace:egi.eu:group:vo.cessda.eduteams.org:role=member#aai.egi.eu
- urn:mace:egi.eu:group:eiscat.se:Hub:role=member#aai.egi.eu
- urn:mace:egi.eu:group:eval.c-scale.eu:role=member#aai.egi.eu
- urn:mace:egi.eu:group:vo.environmental.egi.eu:role=member#aai.egi.eu
- urn:mace:egi.eu:group:vo.lethe-project.eu:lethe-notebooks:role=member#aai.egi.eu
- urn:mace:egi.eu:group:vo.panosc.eu:role=vm_operator#aai.egi.eu
- urn:mace:egi.eu:group:vo.reliance-project.eu:role=member#aai.egi.eu
- urn:mace:egi.eu:group:vo.access.egi.eu:role=member#aai.egi.eu
- urn:mace:egi.eu:group:vo.notebooks.egi.eu:role=member#aai.egi.eu
- urn:mace:egi.eu:group:vo.bioexcel.eu:role=member#aai.egi.eu
- urn:mace:egi.eu:group:vo.egu2024.egi.eu:role=member#aai.egi.eu
- urn:mace:egi.eu:www.egi.eu:fedcloud-users:member@egi.eu
- urn:mace:egi.eu:www.egi.eu:techsolutions:member@egi.eu
# changed 2022-10
- urn:mace:egi.eu:group:fedcloud-users#sso.egi.eu
- urn:mace:egi.eu:group:supplier-notebooks#sso.egi.eu
- urn:mace:egi.eu:group:techsolutions#sso.egi.eu
- urn:mace:egi.eu:group:notebooks-support#sso.egi.eu
auto_login: true
claim_groups_key: "entitlements"
OnedataAuthenticator:
oneprovider_host: "cesnet-oneprovider-01.datahub.egi.eu"
authorize_url: "https://{{ secrets['checkin_host'] }}/auth/realms/egi/protocol/openid-connect/auth"
token_url: "https://{{ secrets['checkin_host'] }}/auth/realms/egi/protocol/openid-connect/token"
userdata_url: "https://{{ secrets['checkin_host'] }}/auth/realms/egi/protocol/openid-connect/userinfo"
client_id: "{{ secrets['client_id'] }}"
client_secret: "{{ secrets['client_secret'] }}"
oauth_callback_url: "https://{{ notebooks_hostname }}/hub/oauth_callback"
scope: ["openid", "profile", "email", "offline_access", "eduperson_scoped_affiliation", "eduperson_entitlement"]
username_key: "sub"
OnedataSpawner:
sidecar_image: "eginotebooks/oneclient-sidecar:sha-9789b9a"
force_direct_io: true
http_timeout: 60
args:
- "--FileCheckpoints.checkpoint_dir='/home/jovyan/.notebookCheckpoints'"
JupyterHub:
admin_access: true
authenticate_prometheus: false
authenticator_class: egi_notebooks_hub.onedata.OnedataAuthenticator
# spawner_class: (in egi-notebooks-b2drop)
extraConfig:
egi-notebooks-welcome: |-
from egi_notebooks_hub.welcome import WelcomeHandler
c.JupyterHub.default_url = "/welcome"
c.JupyterHub.extra_handlers = [(r'/welcome', WelcomeHandler)]
egi-notebooks-b2drop: |-
{%- raw %}
import base64
from jinja2 import BaseLoader
from jinja2 import Environment
from egi_notebooks_hub.onedata import OnedataSpawner
from kubernetes_asyncio.client.rest import ApiException
class B2DropSpawner(OnedataSpawner):
async def auth_state_hook(self, spawner, auth_state):
await super().auth_state_hook(spawner, auth_state)
self.b2drop_ready = False
self.b2drop_user = ""
self.b2drop_pwd = ""
try:
secret = await self.api.read_namespaced_secret(self.token_secret_name, self.namespace)
except ApiException:
return
if secret and secret.data:
self.b2drop_user = base64.b64decode(secret.data.get("b2drop-user", "")).decode()
self.b2drop_pwd = base64.b64decode(secret.data.get("b2drop-pwd", "")).decode()
self.b2drop_ready = (self.b2drop_user and self.b2drop_pwd)
def _render_options_form(self, profile_list):
# old hub: self._profile_list = self._init_profile_list(profile_list)
self._profile_list = self._get_initialized_profile_list(profile_list)
profile_form_template = Environment(loader=BaseLoader).from_string(
self.profile_form_template
)
return profile_form_template.render(profile_list=self._profile_list, b2drop_ready=self.b2drop_ready, b2drop_user=self.b2drop_user, b2drop_pwd=self.b2drop_pwd)
async def pre_spawn_hook(self, spawner):
await super(B2DropSpawner, self).pre_spawn_hook(spawner)
b2drop_user = self.user_options.get("b2drop-user", "")
b2drop_pwd = self.user_options.get("b2drop-pwd", "")
b2drop_remember = self.user_options.get("b2drop-remember", None)
if not (b2drop_user and b2drop_pwd):
secret = await self.api.read_namespaced_secret(self.token_secret_name, self.namespace)
if secret and secret.data:
b2drop_user = base64.b64decode(secret.data.get("b2drop-user", "")).decode()
b2drop_pwd = base64.b64decode(secret.data.get("b2drop-pwd", "")).decode()
if b2drop_user and b2drop_pwd:
volume_mounts = [
{"mountPath": "/b2drop:shared", "name": "b2drop"},
]
spawner.extra_containers.append(
{
"name": "b2drop",
"image": "eginotebooks/webdav-sidecar:sha-e5e8df2",
"env": [
{"name": "WEBDAV_URL", "value": "https://b2drop.eudat.eu/remote.php/webdav"},
{"name": "WEBDAV_PWD", "value": b2drop_pwd},
{"name": "WEBDAV_USER", "value": b2drop_user},
{"name": "MOUNT_PATH", "value": "/b2drop"},
],
"resources": self.sidecar_resources,
# "command": cmd,
"securityContext": {
"runAsUser": 0,
"privileged": True,
"capabilities": {"add": ["SYS_ADMIN"]},
},
"volumeMounts": volume_mounts,
"lifecycle": {
"preStop": {
"exec": {"command": ["umount", "-l", "/b2drop"]}
},
},
}
)
if b2drop_remember:
await self._update_secret({"b2drop-user": b2drop_user,
"b2drop-pwd": b2drop_pwd})
else:
await self._update_secret({"b2drop-user": "", "b2drop-pwd": ""})
def options_from_form(self, formdata):
data = super(B2DropSpawner, self)._options_from_form(formdata)
data.update({'b2drop-user': formdata.get('b2drop-user', [None])[0],
'b2drop-remember': formdata.get('b2drop-remember', [None])[0],
'b2drop-pwd': formdata.get('b2drop-pwd', [None])[0]})
return data
c.JupyterHub.spawner_class = B2DropSpawner
c.B2DropSpawner.http_timeout = 60
c.B2DropSpawner.args = ["--FileCheckpoints.checkpoint_dir='/home/jovyan/.notebookCheckpoints'"]
c.B2DropSpawner.profile_form_template = """
<style>
/*
.profile divs holds two div tags: one for a radio button, and one
for the profile's content.
*/
#kubespawner-profiles-list .profile {
display: flex;
flex-direction: row;
font-weight: normal;
border-bottom: 1px solid #ccc;
padding-bottom: 12px;
}
#kubespawner-profiles-list .profile .radio {
padding: 12px;
}
/* .option divs holds a label and a select tag */
#kubespawner-profiles-list .profile .option {
display: flex;
flex-direction: row;
align-items: center;
padding-bottom: 12px;
}
#kubespawner-profiles-list .profile .option label {
font-weight: normal;
margin-right: 8px;
min-width: 96px;
}
</style>
<div class='form-group' id='kubespawner-profiles-list'>
{%- for profile in profile_list %}
{#- Wrap everything in a <label> so clicking anywhere selects the option #}
<label for='profile-item-{{ profile.slug }}' class='profile'>
<div class='radio'>
<input type='radio' name='profile' id='profile-item-{{ profile.slug }}' value='{{ profile.slug }}' {% if profile.default %}checked{% endif %} />
</div>
<div>
<h3>{{ profile.display_name }}</h3>
{%- if profile.description %}
<p>{{ profile.description }}</p>
{%- endif %}
{%- if profile.profile_options %}
<div>
{%- for k, option in profile.profile_options.items() %}
<div class='option'>
<label for='profile-option-{{profile.slug}}-{{k}}'>{{option.display_name}}</label>
<select name="profile-option-{{profile.slug}}-{{k}}" class="form-control">
{%- for k, choice in option['choices'].items() %}
<option value="{{ k }}" {% if choice.default %}selected{%endif %}>{{ choice.display_name }}</option>
{%- endfor %}
</select>
</div>
{%- endfor %}
</div>
{%- endif %}
</div>
</label>
{%- endfor %}
<div class="panel-group" id="accordion" role="tablist" aria-multiselectable="true">
<div class="panel panel-default">
<div class="panel-heading" role="tab" id="headingOne">
<h4 class="panel-title">
<a class="collabpsed" role="button" data-toggle="collapse" data-parent="#accordion" href="#collapseOne" aria-expanded="false" aria-controls="collapseOne">
B2DROP connection
</a>
{%if b2drop_ready %}<span class="label label-success">Already configured!</span>{% endif %}
</h4>
</div>
<div id="collapseOne" class="panel-collapse collapse" role="tabpanel" aria-labelledby="headingOne">
<div class="panel-body">
<div class='form-group'>
<label for="b2drop-user" class="form-label">B2DROP app Username</label>
<input type="text" class="form-control" name="b2drop-user" id="b2drop-user" aria-describedby="b2drop-user-help" value="{{ b2drop_user }}">
<div id="b2drop-user-help" class="form-text">Create new app password at <a href="https://b2drop.eudat.eu/settings/user/security">B2DROP security configuration</a></div>
</div>
<div class='form-group'>
<label for="b2drop-pwd" class="form-label">B2DROP app Password</label>
<input type="password" class="form-control" name="b2drop-pwd" id="b2drop-pwd" value="{{ b2drop_pwd }}">
</div>
<div class='form-group'>
<input type="checkbox" id="b2drop-remember" name="b2drop-remember" {%if b2drop_ready %}checked{% endif %}>
<label class="form-check-label" for="from-check-input">Remember B2DROP credentials</label>
</div>
</div>
</div>
</div>
</div>
"""
{% endraw %}
templatePaths:
- /egi-notebooks-hub/templates
extraFiles:
login.html:
mountPath: /egi-notebooks-hub/templates/login.html
stringData: |-
{%- raw %}
{% extends "egi-login.html" %}
{% block main_intro %}
<h1><img alt="Notebooks Logo" src="{{ static_url('images/egi-icon-notebooks.svg') }}"
height="100">Notebooks</h1>
<p>
Notebooks is an environment based on <a href="http://jupyter.org/">Jupyter</a> and
the <a href="https://www.egi.eu/services/cloud-compute/">EGI cloud service</a> that
offers a browser-based, scalable tool for interactive data analysis. The Notebooks
environment provides users with notebooks where they can combine text, mathematics,
computations and rich media output.
</p>
<p>
Access requires a valid <a href="https://docs.egi.eu/users/check-in/signup">EGI account</a>
and <a href="https://docs.egi.eu/users/dev-env/notebooks/#notebooks-for-researchers">
enrolling to one of the supported VOs</a>.
</p>
<p>
Default environment provides 4 vCPU cores, 6 GB RAM and 10GB of personal storage space per user.
</p>
{% endblock main_intro %}
{% endraw %}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment