From 9f277f319da1283126435a236a0ff1e6e8b05df1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Franti=C5=A1ek=20Dvo=C5=99=C3=A1k?= <valtri@civ.zcu.cz>
Date: Wed, 31 Jul 2024 12:17:37 +0000
Subject: [PATCH] New staging2 deployment @ SafeSpring + cleanups
---
common/deployments/hub-staging.yaml | 449 +++++++++++++++++
staging1/deployments/hub.yaml | 450 +-----------------
staging2/deployments/hub.yaml | 1 +
staging2/inventory/1-safespring.yaml | 31 ++
staging2/inventory/99-all.yaml | 3 +-
staging2/playbooks/cvmfs.yaml | 1 +
staging2/playbooks/files/calico.yaml | 1 +
staging2/playbooks/files/etc | 1 +
staging2/playbooks/files/jupyterhub-jwt.yaml | 1 +
staging2/playbooks/files/usr | 1 +
staging2/playbooks/k8s.yaml | 1 +
staging2/playbooks/notebooks.yaml | 1 +
staging2/playbooks/public_keys | 1 +
staging2/playbooks/squid.yaml | 1 +
staging2/playbooks/templates/etc/exports | 1 +
.../playbooks/templates/etc/mailutils.conf | 1 +
staging2/playbooks/templates/etc/squid | 1 +
staging2/playbooks/upgrade.yaml | 1 +
staging2/terraform/terraform.tfvars | 6 +-
staging2/terraform/vms.tf | 2 +-
20 files changed, 501 insertions(+), 454 deletions(-)
create mode 100644 common/deployments/hub-staging.yaml
mode change 100644 => 120000 staging1/deployments/hub.yaml
create mode 120000 staging2/deployments/hub.yaml
create mode 100644 staging2/inventory/1-safespring.yaml
create mode 120000 staging2/playbooks/cvmfs.yaml
create mode 120000 staging2/playbooks/files/calico.yaml
create mode 120000 staging2/playbooks/files/etc
create mode 120000 staging2/playbooks/files/jupyterhub-jwt.yaml
create mode 120000 staging2/playbooks/files/usr
create mode 120000 staging2/playbooks/k8s.yaml
create mode 120000 staging2/playbooks/notebooks.yaml
create mode 120000 staging2/playbooks/public_keys
create mode 120000 staging2/playbooks/squid.yaml
create mode 120000 staging2/playbooks/templates/etc/exports
create mode 120000 staging2/playbooks/templates/etc/mailutils.conf
create mode 120000 staging2/playbooks/templates/etc/squid
create mode 120000 staging2/playbooks/upgrade.yaml
diff --git a/common/deployments/hub-staging.yaml b/common/deployments/hub-staging.yaml
new file mode 100644
index 0000000..4c75a56
--- /dev/null
+++ b/common/deployments/hub-staging.yaml
@@ -0,0 +1,449 @@
+---
+proxy:
+ service:
+ type: NodePort
+
+ingress:
+ enabled: true
+ annotations:
+ kubernetes.io/ingress.class: "nginx"
+ kubernetes.io/tls-acme: "true"
+ hosts:
+ - "{{ notebooks_hostname }}"
+ tls:
+ - hosts:
+ - "{{ notebooks_hostname }}"
+ secretName: acme-tls-hub
+
+singleuser:
+ # keep resource limits in sync with:
+ # - profileList
+ storage:
+ type: none
+ extraVolumes:
+ - name: cvmfs-host
+ hostPath:
+ path: /cvmfs
+ type: Directory
+ - name: owncloud-home
+ empty_dir:
+ # - name: scratch
+ # ephemeral:
+ # volumeClaimTemplate:
+ # spec:
+ # accessModes: [ "ReadWriteOnce" ]
+ # storageClassName: local-path
+ # resources:
+ # requests:
+ # storage: "10Gi"
+ extraVolumeMounts:
+ - name: cvmfs-host
+ mountPath: "/cvmfs:shared"
+ - name: owncloud-home
+ mountPath: '/home/jovyan:shared'
+ # - name: scratch
+ # mountPath: '/scratch'
+ memory:
+ limit: 4G
+ guarantee: 512M
+ cpu:
+ limit: 2
+ guarantee: .2
+ defaultUrl: "/lab"
+ image:
+ name: eginotebooks/single-user
+ tag: "sha-b94a3ef"
+ profileList:
+ - display_name: Small Environment - 2 vCPU / 4 GB RAM
+ description: >
+ The notebook environment includes Python, R, Julia and Octave kernels.
+ default: true
+ kubespawner_override:
+ args:
+ - "--CondaKernelSpecManager.env_filter='/opt/conda$'"
+ extra_annotations:
+ "egi.eu/flavor": "small-environment-2-vcpu-4-gb-ram"
+ - display_name: Medium Environment - 4 vCPU / 8 GB RAM
+ description: >
+ The notebook environment includes Python, R, Julia and Octave kernels.
+ kubespawner_override:
+ args:
+ - "--CondaKernelSpecManager.env_filter='/opt/conda$'"
+ extra_annotations:
+ "egi.eu/flavor": "medium-environment-4-vcpu-8-gb-ram"
+ cpu_guarantee: 0.4
+ cpu_limit: 4
+ mem_guarantee: 1G
+ mem_limit: 8G
+ - display_name: Large Environment - 8 vCPU / 16 GB RAM / GPU
+ description: >
+ The notebook environment includes Python, R, Julia and Octave kernels with GPU.
+ kubespawner_override:
+ args:
+ - "--CondaKernelSpecManager.env_filter='/opt/conda$'"
+ cpu_guarantee: 0.8
+ cpu_limit: 8
+ mem_guarantee: 2G
+ mem_limit: 16G
+ extra_annotations:
+ "egi.eu/flavor": "large-environment-8-vcpu-16-gb-ram-gpu"
+ extra_resource_guarantees:
+ nvidia.com/gpu: 1
+ extra_resource_limits:
+ nvidia.com/gpu: 1
+ cmd: jupyterhub-singleuser-webdav-wrapper
+ extraFiles:
+ wait-remote-home.sh:
+ mode: 0755
+ mountPath: /usr/local/bin/jupyterhub-wait-remote-home
+ stringData: |-
+ #! /bin/sh
+ i=0
+ while ! grep '^webdav-fs: /home/jovyan ' /proc/mounts && test $i -lt 30; do
+ echo 'Waiting for ownClound mount...'
+ sleep 0.5
+ i=$((i+1))
+ done
+ singleuser-webdav-wrapper.sh:
+ mode: 0755
+ mountPath: /usr/local/bin/jupyterhub-singleuser-webdav-wrapper
+ stringData: |-
+ #! /bin/sh
+ #
+ # Dirty hack to make remote mount on home directory working properly:
+ #
+ # 1) wait for webdav sidecar image to kick in
+ # 2) change directory to the mounted version of itself
+ # 3) launch notebook server
+ #
+ /usr/local/bin/jupyterhub-wait-remote-home
+
+ cd .
+
+ exec jupyterhub-singleuser \
+ --FileCheckpoints.checkpoint_dir='/home/jovyan/.notebookCheckpoints' \
+ --LabApp.news_url=None \
+ "$@"
+
+hub:
+ services:
+ status:
+ url: "http://status-web/"
+ admin: true
+ jwt:
+ url: "http://jwt/"
+ display: false
+ # recommended to keep in sync with common/playbooks/files/jupyterhub-jwt.yaml
+ image:
+ name: eginotebooks/hub
+ tag: "sha-323c75e"
+ config:
+ Authenticator:
+ enable_auth_state: true
+ admin_users:
+ # valtri@civ.zcu.cz
+ - 94d3cde7-3121-4b33-b4c2-526c67e8cb38@eosc-federation.eu
+ allowed_groups:
+ - urn:geant:eosc-federation.eu:staging:group:eosc#staging.eosc-federation.eu
+ auto_login: true
+ claim_groups_key: "entitlements"
+ EGICheckinAuthenticator:
+ checkin_host: "{{ secret['checkin_host'] }}"
+ authorize_url: "https://{{ secret['checkin_host'] }}/OIDC/authorization"
+ token_url: "https://{{ secret['checkin_host'] }}/OIDC/token"
+ userdata_url: "https://{{ secret['checkin_host'] }}/OIDC/userinfo"
+ client_id: "{{ secret['client_id'] }}"
+ client_secret: "{{ secret['client_secret'] }}"
+ oauth_callback_url: "https://{{ notebooks_hostname }}/hub/oauth_callback"
+ openid_configuration_url: "https://proxy.testing.eosc-federation.eu/.well-known/openid-configuration"
+ scope: ["openid", "profile", "email", "offline_access", "entitlements"]
+ username_claim: "sub"
+ extra_authorize_params:
+ prompt: consent
+ JupyterHub:
+ admin_access: true
+ authenticate_prometheus: false
+ authenticator_class: egi_notebooks_hub.egiauthenticator.EOSCNodeAuthenticator
+ # spawner_class: (in egi-notebooks-b2drop)
+ LabApp:
+ check_for_updates_class: jupyterlab.NeverCheckForUpdate
+ extraConfig:
+ egi-notebooks-welcome: |-
+ from egi_notebooks_hub.welcome import WelcomeHandler
+ c.JupyterHub.default_url = "/welcome"
+ c.JupyterHub.extra_handlers = [(r'/welcome', WelcomeHandler)]
+ egi-notebooks-b2drop: |-
+{%- raw %}
+ import base64
+ import json
+ from jinja2 import BaseLoader
+ from jinja2 import Environment
+ from egi_notebooks_hub.onedata import OnedataSpawner
+ from kubernetes_asyncio.client.rest import ApiException
+ from tornado.httpclient import AsyncHTTPClient, HTTPClientError, HTTPRequest
+
+
+ class B2DropSpawner(OnedataSpawner):
+ async def auth_state_hook(self, spawner, auth_state):
+ await super().auth_state_hook(spawner, auth_state)
+ self.b2drop_ready = False
+ self.b2drop_user = ""
+ self.b2drop_pwd = ""
+ try:
+ secret = await self.api.read_namespaced_secret(self.token_secret_name, self.namespace)
+ except ApiException:
+ return
+ if secret and secret.data:
+ self.b2drop_user = base64.b64decode(secret.data.get("b2drop-user", "")).decode()
+ self.b2drop_pwd = base64.b64decode(secret.data.get("b2drop-pwd", "")).decode()
+ self.b2drop_ready = (self.b2drop_user and self.b2drop_pwd)
+
+ def _render_options_form(self, profile_list):
+ # old:self._profile_list = self._init_profile_list(profile_list)
+ self._profile_list = self._get_initialized_profile_list(profile_list)
+
+ profile_form_template = Environment(loader=BaseLoader).from_string(
+ self.profile_form_template
+ )
+ return profile_form_template.render(profile_list=self._profile_list, b2drop_ready=self.b2drop_ready, b2drop_user=self.b2drop_user, b2drop_pwd=self.b2drop_pwd)
+
+ async def pre_spawn_hook(self, spawner):
+ await super(B2DropSpawner, self).pre_spawn_hook(spawner)
+ b2drop_user = self.user_options.get("b2drop-user", "")
+ b2drop_pwd = self.user_options.get("b2drop-pwd", "")
+ if not (b2drop_user and b2drop_pwd):
+ secret = await self.api.read_namespaced_secret(self.token_secret_name, self.namespace)
+ if secret and secret.data:
+ b2drop_user = base64.b64decode(secret.data.get("b2drop-user", "")).decode()
+ b2drop_pwd = base64.b64decode(secret.data.get("b2drop-pwd", "")).decode()
+ if b2drop_user and b2drop_pwd:
+ volume_mounts = [
+ {"mountPath": "/owncloud:shared", "name": "owncloud-home"},
+ ]
+ spawner.extra_containers.append(
+ {
+ "name": "b2drop",
+ "image": "eginotebooks/webdav-rclone-sidecar:sha-0a62679",
+ "env": [
+ {"name": "WEBDAV_URL", "value": "https://b2drop.eudat.eu/remote.php/webdav"},
+ {"name": "WEBDAV_PWD", "value": b2drop_pwd},
+ {"name": "WEBDAV_USER", "value": b2drop_user},
+ {"name": "WEBDAV_VENDOR", "value": "other"},
+ {"name": "MOUNT_PATH", "value": "/owncloud/b2drop"},
+ {"name": "MOUNT_WAIT_POINT", "value": "webdav-fs: /owncloud fuse.rclone"},
+ ],
+ "resources": self.sidecar_resources,
+ "securityContext": {
+ "runAsUser": 1000,
+ "fsUser": 1000,
+ "fsGroup": 100,
+ "privileged": True,
+ "capabilities": {"add": ["SYS_ADMIN"]},
+ },
+ "volumeMounts": volume_mounts,
+ }
+ )
+
+ def options_from_form(self, formdata):
+ data = super(B2DropSpawner, self)._options_from_form(formdata)
+ data.update({'b2drop-user': formdata.get('b2drop-user', [None])[0],
+ 'b2drop-pwd': formdata.get('b2drop-pwd', [None])[0]})
+ return data
+
+
+ class WebDavOIDCSpawner(B2DropSpawner):
+ # ownCloud Infinite Scale parameters
+ # (https://owncloud.dev/apis/http/graph/spaces/#list-my-spaces-get-medrives)
+ OCIS_URL = "https://ocis-staging.apps.bst2-test.paas.psnc.pl"
+ # personal space
+ OCIS_PERSONAL_SPACE = "/graph/v1.0/me/drives?%24filter=driveType+eq+personal"
+ # shared space
+ OCIS_SHARED_WITH_ME = "/graph/v1.0/me/drives?%24filter=driveType+eq+virtual"
+ # otter spaces
+ OCIS_SPACES = "/graph/v1.0/me/drives?%24filter=driveType+eq+project"
+
+ async def append_owncloud_sidecar(self, spawner, type, query, fallback_url=None, headers={}):
+ owncloud_url = fallback_url
+ http_client = AsyncHTTPClient()
+ req = HTTPRequest(
+ self.OCIS_URL + query,
+ headers=headers,
+ method="GET",
+ )
+ try:
+ resp = await http_client.fetch(req)
+ body = json.loads(resp.body.decode("utf8", "replace"))
+ self.log.debug("OCIS response: %s", body)
+ if "value" in body:
+ ocis_infos = body["value"]
+ if len(ocis_infos) >= 1 and "root" in ocis_infos[0]:
+ owncloud_url = ocis_infos[0]["root"].get("webDavUrl", None)
+ except HTTPClientError as e:
+ self.log.error("can't query ownCloud: %s", e)
+ self.log.info("ownCloud %s URL: %s", type, owncloud_url)
+
+ if owncloud_url is None:
+ return
+
+ if type == "home":
+ subpath = ""
+ else:
+ subpath = "/" + type.capitalize()
+ env = [
+ {"name": "WEBDAV_URL", "value": owncloud_url},
+ {"name": "WEBDAV_VENDOR", "value": "owncloud"},
+ # XXX: strict permissions needed for .local/share/jupyter/runtime/jupyter_cookie_secret
+ # quicker directory cache and polling
+ {"name": "MOUNT_OPTS", "value": "--file-perms=0600 --dir-perms=0770 --dir-cache-time=1m0s --poll-interval=0m20s"},
+ {"name": "MOUNT_PATH", "value": "/owncloud" + subpath},
+ # default mode is "full"
+ {"name": "VFS_CACHE_MODE", "value": "full"},
+ ]
+ if type != "home":
+ env.append({"name": "MOUNT_WAIT_POINT", "value": "webdav-fs: /owncloud fuse.rclone"})
+ volume_mounts = [
+ {"mountPath": "/owncloud:shared", "name": "owncloud-home"},
+ {"mountPath": self.token_mount_path, "name": self.token_secret_volume_name, "readOnly": True},
+ ]
+ spawner.extra_containers.append(
+ {
+ "name": "owncloud-" + type,
+ "image": "eginotebooks/webdav-rclone-sidecar:sha-0a62679",
+ "args": ["bearer_token_command=cat " + self.token_path],
+ "env": env,
+ "resources": self.sidecar_resources,
+ "securityContext": {
+ "runAsUser": 1000,
+ "fsUser": 1000,
+ "fsGroup": 100,
+ "privileged": True,
+ "capabilities": {"add": ["SYS_ADMIN"]},
+ },
+ "volumeMounts": volume_mounts,
+ }
+ )
+
+ async def pre_spawn_hook(self, spawner):
+ await super(WebDavOIDCSpawner, self).pre_spawn_hook(spawner)
+ auth_state = await self.user.get_auth_state()
+ # volume name as in EGI spawner
+ self.token_secret_volume_name = self._expand_user_properties(
+ self.token_secret_volume_name_template
+ )
+ self.token_path = os.path.join(self.token_mount_path, "access_token")
+
+ if auth_state:
+ access_token = auth_state.get("access_token", None)
+ headers = {
+ "Accept": "application/json",
+ "User-Agent": "JupyterHub",
+ "Authorization": "Bearer %s" % access_token,
+ }
+
+ await self.append_owncloud_sidecar(spawner, "home", self.OCIS_PERSONAL_SPACE, headers=headers)
+ await self.append_owncloud_sidecar(spawner, "shares", self.OCIS_SHARED_WITH_ME, headers=headers)
+ await self.append_owncloud_sidecar(spawner, "spaces", self.OCIS_SPACES, headers=headers)
+ else:
+ self.log.info("No auth state, skipping ownCloud")
+
+
+ c.JupyterHub.spawner_class = WebDavOIDCSpawner
+ c.B2DropSpawner.http_timeout = 90
+ c.B2DropSpawner.profile_form_template = """
+ <style>
+ /*
+ .profile divs holds two div tags: one for a radio button, and one
+ for the profile's content.
+ */
+ #kubespawner-profiles-list .profile {
+ display: flex;
+ flex-direction: row;
+ font-weight: normal;
+ border-bottom: 1px solid #ccc;
+ padding-bottom: 12px;
+ }
+
+ #kubespawner-profiles-list .profile .radio {
+ padding: 12px;
+ }
+
+ /* .option divs holds a label and a select tag */
+ #kubespawner-profiles-list .profile .option {
+ display: flex;
+ flex-direction: row;
+ align-items: center;
+ padding-bottom: 12px;
+ }
+
+ #kubespawner-profiles-list .profile .option label {
+ font-weight: normal;
+ margin-right: 8px;
+ min-width: 96px;
+ }
+ </style>
+
+ <div class='form-group' id='kubespawner-profiles-list'>
+ {%- for profile in profile_list %}
+ {#- Wrap everything in a <label> so clicking anywhere selects the option #}
+ <label for='profile-item-{{ profile.slug }}' class='profile'>
+ <div class='radio'>
+ <input type='radio' name='profile' id='profile-item-{{ profile.slug }}' value='{{ profile.slug }}' {% if profile.default %}checked{% endif %} />
+ </div>
+ <div>
+ <h3>{{ profile.display_name }}</h3>
+
+ {%- if profile.description %}
+ <p>{{ profile.description }}</p>
+ {%- endif %}
+
+ {%- if profile.profile_options %}
+ <div>
+ {%- for k, option in profile.profile_options.items() %}
+ <div class='option'>
+ <label for='profile-option-{{profile.slug}}-{{k}}'>{{option.display_name}}</label>
+ <select name="profile-option-{{profile.slug}}-{{k}}" class="form-control">
+ {%- for k, choice in option['choices'].items() %}
+ <option value="{{ k }}" {% if choice.default %}selected{%endif %}>{{ choice.display_name }}</option>
+ {%- endfor %}
+ </select>
+ </div>
+ {%- endfor %}
+ </div>
+ {%- endif %}
+ </div>
+ </label>
+ {%- endfor %}
+ <div class="panel-group" id="accordion" role="tablist" aria-multiselectable="true">
+ <div class="panel panel-default">
+ <div class="panel-heading" role="tab" id="headingOne">
+ <h4 class="panel-title">
+ <a class="collabpsed" role="button" data-toggle="collapse" data-parent="#accordion" href="#collapseOne" aria-expanded="false" aria-controls="collapseOne">
+ B2DROP connection
+ </a>
+ {%if b2drop_ready %}<span class="label label-success">Already configured!</span>{% endif %}
+ </h4>
+ </div>
+ <div id="collapseOne" class="panel-collapse collapse" role="tabpanel" aria-labelledby="headingOne">
+ <div class="panel-body">
+ <div class='form-group'>
+ <label for="b2drop-user" class="form-label">B2DROP app Username</label>
+ <input type="text" class="form-control" name="b2drop-user" id="b2drop-user" aria-describedby="b2drop-user-help" value="{{ b2drop_user }}">
+ <div id="b2drop-user-help" class="form-text">Create new app password at <a href="https://b2drop.eudat.eu/settings/user/security">B2DROP security configuration</a></div>
+ </div>
+ <div class='form-group'>
+ <label for="b2drop-pwd" class="form-label">B2DROP app Password</label>
+ <input type="password" class="form-control" name="b2drop-pwd" id="b2drop-pwd" value="{{ b2drop_pwd }}">
+ </div>
+ </div>
+ </div>
+ </div>
+ </div>
+ """
+{% endraw %}
+ extraFiles:
+ welcome.html:
+ mountPath: /usr/local/share/jupyterhub/templates/welcome.html
+ stringData: |-
+{%- raw %}
+ {% extends "login.html" %}
+{% endraw %}
diff --git a/staging1/deployments/hub.yaml b/staging1/deployments/hub.yaml
deleted file mode 100644
index 4c75a56..0000000
--- a/staging1/deployments/hub.yaml
+++ /dev/null
@@ -1,449 +0,0 @@
----
-proxy:
- service:
- type: NodePort
-
-ingress:
- enabled: true
- annotations:
- kubernetes.io/ingress.class: "nginx"
- kubernetes.io/tls-acme: "true"
- hosts:
- - "{{ notebooks_hostname }}"
- tls:
- - hosts:
- - "{{ notebooks_hostname }}"
- secretName: acme-tls-hub
-
-singleuser:
- # keep resource limits in sync with:
- # - profileList
- storage:
- type: none
- extraVolumes:
- - name: cvmfs-host
- hostPath:
- path: /cvmfs
- type: Directory
- - name: owncloud-home
- empty_dir:
- # - name: scratch
- # ephemeral:
- # volumeClaimTemplate:
- # spec:
- # accessModes: [ "ReadWriteOnce" ]
- # storageClassName: local-path
- # resources:
- # requests:
- # storage: "10Gi"
- extraVolumeMounts:
- - name: cvmfs-host
- mountPath: "/cvmfs:shared"
- - name: owncloud-home
- mountPath: '/home/jovyan:shared'
- # - name: scratch
- # mountPath: '/scratch'
- memory:
- limit: 4G
- guarantee: 512M
- cpu:
- limit: 2
- guarantee: .2
- defaultUrl: "/lab"
- image:
- name: eginotebooks/single-user
- tag: "sha-b94a3ef"
- profileList:
- - display_name: Small Environment - 2 vCPU / 4 GB RAM
- description: >
- The notebook environment includes Python, R, Julia and Octave kernels.
- default: true
- kubespawner_override:
- args:
- - "--CondaKernelSpecManager.env_filter='/opt/conda$'"
- extra_annotations:
- "egi.eu/flavor": "small-environment-2-vcpu-4-gb-ram"
- - display_name: Medium Environment - 4 vCPU / 8 GB RAM
- description: >
- The notebook environment includes Python, R, Julia and Octave kernels.
- kubespawner_override:
- args:
- - "--CondaKernelSpecManager.env_filter='/opt/conda$'"
- extra_annotations:
- "egi.eu/flavor": "medium-environment-4-vcpu-8-gb-ram"
- cpu_guarantee: 0.4
- cpu_limit: 4
- mem_guarantee: 1G
- mem_limit: 8G
- - display_name: Large Environment - 8 vCPU / 16 GB RAM / GPU
- description: >
- The notebook environment includes Python, R, Julia and Octave kernels with GPU.
- kubespawner_override:
- args:
- - "--CondaKernelSpecManager.env_filter='/opt/conda$'"
- cpu_guarantee: 0.8
- cpu_limit: 8
- mem_guarantee: 2G
- mem_limit: 16G
- extra_annotations:
- "egi.eu/flavor": "large-environment-8-vcpu-16-gb-ram-gpu"
- extra_resource_guarantees:
- nvidia.com/gpu: 1
- extra_resource_limits:
- nvidia.com/gpu: 1
- cmd: jupyterhub-singleuser-webdav-wrapper
- extraFiles:
- wait-remote-home.sh:
- mode: 0755
- mountPath: /usr/local/bin/jupyterhub-wait-remote-home
- stringData: |-
- #! /bin/sh
- i=0
- while ! grep '^webdav-fs: /home/jovyan ' /proc/mounts && test $i -lt 30; do
- echo 'Waiting for ownClound mount...'
- sleep 0.5
- i=$((i+1))
- done
- singleuser-webdav-wrapper.sh:
- mode: 0755
- mountPath: /usr/local/bin/jupyterhub-singleuser-webdav-wrapper
- stringData: |-
- #! /bin/sh
- #
- # Dirty hack to make remote mount on home directory working properly:
- #
- # 1) wait for webdav sidecar image to kick in
- # 2) change directory to the mounted version of itself
- # 3) launch notebook server
- #
- /usr/local/bin/jupyterhub-wait-remote-home
-
- cd .
-
- exec jupyterhub-singleuser \
- --FileCheckpoints.checkpoint_dir='/home/jovyan/.notebookCheckpoints' \
- --LabApp.news_url=None \
- "$@"
-
-hub:
- services:
- status:
- url: "http://status-web/"
- admin: true
- jwt:
- url: "http://jwt/"
- display: false
- # recommended to keep in sync with common/playbooks/files/jupyterhub-jwt.yaml
- image:
- name: eginotebooks/hub
- tag: "sha-323c75e"
- config:
- Authenticator:
- enable_auth_state: true
- admin_users:
- # valtri@civ.zcu.cz
- - 94d3cde7-3121-4b33-b4c2-526c67e8cb38@eosc-federation.eu
- allowed_groups:
- - urn:geant:eosc-federation.eu:staging:group:eosc#staging.eosc-federation.eu
- auto_login: true
- claim_groups_key: "entitlements"
- EGICheckinAuthenticator:
- checkin_host: "{{ secret['checkin_host'] }}"
- authorize_url: "https://{{ secret['checkin_host'] }}/OIDC/authorization"
- token_url: "https://{{ secret['checkin_host'] }}/OIDC/token"
- userdata_url: "https://{{ secret['checkin_host'] }}/OIDC/userinfo"
- client_id: "{{ secret['client_id'] }}"
- client_secret: "{{ secret['client_secret'] }}"
- oauth_callback_url: "https://{{ notebooks_hostname }}/hub/oauth_callback"
- openid_configuration_url: "https://proxy.testing.eosc-federation.eu/.well-known/openid-configuration"
- scope: ["openid", "profile", "email", "offline_access", "entitlements"]
- username_claim: "sub"
- extra_authorize_params:
- prompt: consent
- JupyterHub:
- admin_access: true
- authenticate_prometheus: false
- authenticator_class: egi_notebooks_hub.egiauthenticator.EOSCNodeAuthenticator
- # spawner_class: (in egi-notebooks-b2drop)
- LabApp:
- check_for_updates_class: jupyterlab.NeverCheckForUpdate
- extraConfig:
- egi-notebooks-welcome: |-
- from egi_notebooks_hub.welcome import WelcomeHandler
- c.JupyterHub.default_url = "/welcome"
- c.JupyterHub.extra_handlers = [(r'/welcome', WelcomeHandler)]
- egi-notebooks-b2drop: |-
-{%- raw %}
- import base64
- import json
- from jinja2 import BaseLoader
- from jinja2 import Environment
- from egi_notebooks_hub.onedata import OnedataSpawner
- from kubernetes_asyncio.client.rest import ApiException
- from tornado.httpclient import AsyncHTTPClient, HTTPClientError, HTTPRequest
-
-
- class B2DropSpawner(OnedataSpawner):
- async def auth_state_hook(self, spawner, auth_state):
- await super().auth_state_hook(spawner, auth_state)
- self.b2drop_ready = False
- self.b2drop_user = ""
- self.b2drop_pwd = ""
- try:
- secret = await self.api.read_namespaced_secret(self.token_secret_name, self.namespace)
- except ApiException:
- return
- if secret and secret.data:
- self.b2drop_user = base64.b64decode(secret.data.get("b2drop-user", "")).decode()
- self.b2drop_pwd = base64.b64decode(secret.data.get("b2drop-pwd", "")).decode()
- self.b2drop_ready = (self.b2drop_user and self.b2drop_pwd)
-
- def _render_options_form(self, profile_list):
- # old:self._profile_list = self._init_profile_list(profile_list)
- self._profile_list = self._get_initialized_profile_list(profile_list)
-
- profile_form_template = Environment(loader=BaseLoader).from_string(
- self.profile_form_template
- )
- return profile_form_template.render(profile_list=self._profile_list, b2drop_ready=self.b2drop_ready, b2drop_user=self.b2drop_user, b2drop_pwd=self.b2drop_pwd)
-
- async def pre_spawn_hook(self, spawner):
- await super(B2DropSpawner, self).pre_spawn_hook(spawner)
- b2drop_user = self.user_options.get("b2drop-user", "")
- b2drop_pwd = self.user_options.get("b2drop-pwd", "")
- if not (b2drop_user and b2drop_pwd):
- secret = await self.api.read_namespaced_secret(self.token_secret_name, self.namespace)
- if secret and secret.data:
- b2drop_user = base64.b64decode(secret.data.get("b2drop-user", "")).decode()
- b2drop_pwd = base64.b64decode(secret.data.get("b2drop-pwd", "")).decode()
- if b2drop_user and b2drop_pwd:
- volume_mounts = [
- {"mountPath": "/owncloud:shared", "name": "owncloud-home"},
- ]
- spawner.extra_containers.append(
- {
- "name": "b2drop",
- "image": "eginotebooks/webdav-rclone-sidecar:sha-0a62679",
- "env": [
- {"name": "WEBDAV_URL", "value": "https://b2drop.eudat.eu/remote.php/webdav"},
- {"name": "WEBDAV_PWD", "value": b2drop_pwd},
- {"name": "WEBDAV_USER", "value": b2drop_user},
- {"name": "WEBDAV_VENDOR", "value": "other"},
- {"name": "MOUNT_PATH", "value": "/owncloud/b2drop"},
- {"name": "MOUNT_WAIT_POINT", "value": "webdav-fs: /owncloud fuse.rclone"},
- ],
- "resources": self.sidecar_resources,
- "securityContext": {
- "runAsUser": 1000,
- "fsUser": 1000,
- "fsGroup": 100,
- "privileged": True,
- "capabilities": {"add": ["SYS_ADMIN"]},
- },
- "volumeMounts": volume_mounts,
- }
- )
-
- def options_from_form(self, formdata):
- data = super(B2DropSpawner, self)._options_from_form(formdata)
- data.update({'b2drop-user': formdata.get('b2drop-user', [None])[0],
- 'b2drop-pwd': formdata.get('b2drop-pwd', [None])[0]})
- return data
-
-
- class WebDavOIDCSpawner(B2DropSpawner):
- # ownCloud Infinite Scale parameters
- # (https://owncloud.dev/apis/http/graph/spaces/#list-my-spaces-get-medrives)
- OCIS_URL = "https://ocis-staging.apps.bst2-test.paas.psnc.pl"
- # personal space
- OCIS_PERSONAL_SPACE = "/graph/v1.0/me/drives?%24filter=driveType+eq+personal"
- # shared space
- OCIS_SHARED_WITH_ME = "/graph/v1.0/me/drives?%24filter=driveType+eq+virtual"
- # otter spaces
- OCIS_SPACES = "/graph/v1.0/me/drives?%24filter=driveType+eq+project"
-
- async def append_owncloud_sidecar(self, spawner, type, query, fallback_url=None, headers={}):
- owncloud_url = fallback_url
- http_client = AsyncHTTPClient()
- req = HTTPRequest(
- self.OCIS_URL + query,
- headers=headers,
- method="GET",
- )
- try:
- resp = await http_client.fetch(req)
- body = json.loads(resp.body.decode("utf8", "replace"))
- self.log.debug("OCIS response: %s", body)
- if "value" in body:
- ocis_infos = body["value"]
- if len(ocis_infos) >= 1 and "root" in ocis_infos[0]:
- owncloud_url = ocis_infos[0]["root"].get("webDavUrl", None)
- except HTTPClientError as e:
- self.log.error("can't query ownCloud: %s", e)
- self.log.info("ownCloud %s URL: %s", type, owncloud_url)
-
- if owncloud_url is None:
- return
-
- if type == "home":
- subpath = ""
- else:
- subpath = "/" + type.capitalize()
- env = [
- {"name": "WEBDAV_URL", "value": owncloud_url},
- {"name": "WEBDAV_VENDOR", "value": "owncloud"},
- # XXX: strict permissions needed for .local/share/jupyter/runtime/jupyter_cookie_secret
- # quicker directory cache and polling
- {"name": "MOUNT_OPTS", "value": "--file-perms=0600 --dir-perms=0770 --dir-cache-time=1m0s --poll-interval=0m20s"},
- {"name": "MOUNT_PATH", "value": "/owncloud" + subpath},
- # default mode is "full"
- {"name": "VFS_CACHE_MODE", "value": "full"},
- ]
- if type != "home":
- env.append({"name": "MOUNT_WAIT_POINT", "value": "webdav-fs: /owncloud fuse.rclone"})
- volume_mounts = [
- {"mountPath": "/owncloud:shared", "name": "owncloud-home"},
- {"mountPath": self.token_mount_path, "name": self.token_secret_volume_name, "readOnly": True},
- ]
- spawner.extra_containers.append(
- {
- "name": "owncloud-" + type,
- "image": "eginotebooks/webdav-rclone-sidecar:sha-0a62679",
- "args": ["bearer_token_command=cat " + self.token_path],
- "env": env,
- "resources": self.sidecar_resources,
- "securityContext": {
- "runAsUser": 1000,
- "fsUser": 1000,
- "fsGroup": 100,
- "privileged": True,
- "capabilities": {"add": ["SYS_ADMIN"]},
- },
- "volumeMounts": volume_mounts,
- }
- )
-
- async def pre_spawn_hook(self, spawner):
- await super(WebDavOIDCSpawner, self).pre_spawn_hook(spawner)
- auth_state = await self.user.get_auth_state()
- # volume name as in EGI spawner
- self.token_secret_volume_name = self._expand_user_properties(
- self.token_secret_volume_name_template
- )
- self.token_path = os.path.join(self.token_mount_path, "access_token")
-
- if auth_state:
- access_token = auth_state.get("access_token", None)
- headers = {
- "Accept": "application/json",
- "User-Agent": "JupyterHub",
- "Authorization": "Bearer %s" % access_token,
- }
-
- await self.append_owncloud_sidecar(spawner, "home", self.OCIS_PERSONAL_SPACE, headers=headers)
- await self.append_owncloud_sidecar(spawner, "shares", self.OCIS_SHARED_WITH_ME, headers=headers)
- await self.append_owncloud_sidecar(spawner, "spaces", self.OCIS_SPACES, headers=headers)
- else:
- self.log.info("No auth state, skipping ownCloud")
-
-
- c.JupyterHub.spawner_class = WebDavOIDCSpawner
- c.B2DropSpawner.http_timeout = 90
- c.B2DropSpawner.profile_form_template = """
- <style>
- /*
- .profile divs holds two div tags: one for a radio button, and one
- for the profile's content.
- */
- #kubespawner-profiles-list .profile {
- display: flex;
- flex-direction: row;
- font-weight: normal;
- border-bottom: 1px solid #ccc;
- padding-bottom: 12px;
- }
-
- #kubespawner-profiles-list .profile .radio {
- padding: 12px;
- }
-
- /* .option divs holds a label and a select tag */
- #kubespawner-profiles-list .profile .option {
- display: flex;
- flex-direction: row;
- align-items: center;
- padding-bottom: 12px;
- }
-
- #kubespawner-profiles-list .profile .option label {
- font-weight: normal;
- margin-right: 8px;
- min-width: 96px;
- }
- </style>
-
- <div class='form-group' id='kubespawner-profiles-list'>
- {%- for profile in profile_list %}
- {#- Wrap everything in a <label> so clicking anywhere selects the option #}
- <label for='profile-item-{{ profile.slug }}' class='profile'>
- <div class='radio'>
- <input type='radio' name='profile' id='profile-item-{{ profile.slug }}' value='{{ profile.slug }}' {% if profile.default %}checked{% endif %} />
- </div>
- <div>
- <h3>{{ profile.display_name }}</h3>
-
- {%- if profile.description %}
- <p>{{ profile.description }}</p>
- {%- endif %}
-
- {%- if profile.profile_options %}
- <div>
- {%- for k, option in profile.profile_options.items() %}
- <div class='option'>
- <label for='profile-option-{{profile.slug}}-{{k}}'>{{option.display_name}}</label>
- <select name="profile-option-{{profile.slug}}-{{k}}" class="form-control">
- {%- for k, choice in option['choices'].items() %}
- <option value="{{ k }}" {% if choice.default %}selected{%endif %}>{{ choice.display_name }}</option>
- {%- endfor %}
- </select>
- </div>
- {%- endfor %}
- </div>
- {%- endif %}
- </div>
- </label>
- {%- endfor %}
- <div class="panel-group" id="accordion" role="tablist" aria-multiselectable="true">
- <div class="panel panel-default">
- <div class="panel-heading" role="tab" id="headingOne">
- <h4 class="panel-title">
- <a class="collabpsed" role="button" data-toggle="collapse" data-parent="#accordion" href="#collapseOne" aria-expanded="false" aria-controls="collapseOne">
- B2DROP connection
- </a>
- {%if b2drop_ready %}<span class="label label-success">Already configured!</span>{% endif %}
- </h4>
- </div>
- <div id="collapseOne" class="panel-collapse collapse" role="tabpanel" aria-labelledby="headingOne">
- <div class="panel-body">
- <div class='form-group'>
- <label for="b2drop-user" class="form-label">B2DROP app Username</label>
- <input type="text" class="form-control" name="b2drop-user" id="b2drop-user" aria-describedby="b2drop-user-help" value="{{ b2drop_user }}">
- <div id="b2drop-user-help" class="form-text">Create new app password at <a href="https://b2drop.eudat.eu/settings/user/security">B2DROP security configuration</a></div>
- </div>
- <div class='form-group'>
- <label for="b2drop-pwd" class="form-label">B2DROP app Password</label>
- <input type="password" class="form-control" name="b2drop-pwd" id="b2drop-pwd" value="{{ b2drop_pwd }}">
- </div>
- </div>
- </div>
- </div>
- </div>
- """
-{% endraw %}
- extraFiles:
- welcome.html:
- mountPath: /usr/local/share/jupyterhub/templates/welcome.html
- stringData: |-
-{%- raw %}
- {% extends "login.html" %}
-{% endraw %}
diff --git a/staging1/deployments/hub.yaml b/staging1/deployments/hub.yaml
new file mode 120000
index 0000000..3fda462
--- /dev/null
+++ b/staging1/deployments/hub.yaml
@@ -0,0 +1 @@
+/home/valtri/notebooks-operations.eosc/common/deployments/hub-staging.yaml
\ No newline at end of file
diff --git a/staging2/deployments/hub.yaml b/staging2/deployments/hub.yaml
new file mode 120000
index 0000000..3fda462
--- /dev/null
+++ b/staging2/deployments/hub.yaml
@@ -0,0 +1 @@
+/home/valtri/notebooks-operations.eosc/common/deployments/hub-staging.yaml
\ No newline at end of file
diff --git a/staging2/inventory/1-safespring.yaml b/staging2/inventory/1-safespring.yaml
new file mode 100644
index 0000000..6d6bc57
--- /dev/null
+++ b/staging2/inventory/1-safespring.yaml
@@ -0,0 +1,31 @@
+---
+fip:
+ hosts:
+ 89.47.191.176
+
+master:
+ hosts:
+ 2001:6b0:7d:40::80:
+ # must be IPv4 address or hostname
+ kube_server: 89.47.191.61
+
+ingress:
+ hosts:
+ 2001:6b0:7d:40::12:
+
+nfs:
+ hosts:
+ 2001:6b0:7d:40::17c:
+
+worker:
+ hosts:
+ 2001:6b0:7d:40::81:
+
+gpu:
+ hosts:
+
+# using public IP of kube_server for ansible delegate_to
+kube_server:
+ hosts:
+ 89.47.191.61:
+ ansible_host: 2001:6b0:7d:40::80
diff --git a/staging2/inventory/99-all.yaml b/staging2/inventory/99-all.yaml
index 75f4d04..3c59b9c 100644
--- a/staging2/inventory/99-all.yaml
+++ b/staging2/inventory/99-all.yaml
@@ -11,7 +11,8 @@ all:
ansible_become: yes
ansible_user: egi
- site_name: safespring-staging
+ mail_local: true
+ site_name: safespring-staging2
vault_mount_point: secrets/users/e1662e20-e34b-468c-b0ce-d899bc878364@egi.eu/eosc-staging
notebooks_hostname: notebooks-stg2.cloud.cesnet.cz
diff --git a/staging2/playbooks/cvmfs.yaml b/staging2/playbooks/cvmfs.yaml
new file mode 120000
index 0000000..2e82cca
--- /dev/null
+++ b/staging2/playbooks/cvmfs.yaml
@@ -0,0 +1 @@
+../../common/playbooks/cvmfs.yaml
\ No newline at end of file
diff --git a/staging2/playbooks/files/calico.yaml b/staging2/playbooks/files/calico.yaml
new file mode 120000
index 0000000..732c864
--- /dev/null
+++ b/staging2/playbooks/files/calico.yaml
@@ -0,0 +1 @@
+../../../common/playbooks/files/calico.yaml
\ No newline at end of file
diff --git a/staging2/playbooks/files/etc b/staging2/playbooks/files/etc
new file mode 120000
index 0000000..ed53b87
--- /dev/null
+++ b/staging2/playbooks/files/etc
@@ -0,0 +1 @@
+../../../common/playbooks/files/etc
\ No newline at end of file
diff --git a/staging2/playbooks/files/jupyterhub-jwt.yaml b/staging2/playbooks/files/jupyterhub-jwt.yaml
new file mode 120000
index 0000000..59f9ac2
--- /dev/null
+++ b/staging2/playbooks/files/jupyterhub-jwt.yaml
@@ -0,0 +1 @@
+../../../common/playbooks/files/jupyterhub-jwt.yaml
\ No newline at end of file
diff --git a/staging2/playbooks/files/usr b/staging2/playbooks/files/usr
new file mode 120000
index 0000000..b034223
--- /dev/null
+++ b/staging2/playbooks/files/usr
@@ -0,0 +1 @@
+../../../common/playbooks/files/usr
\ No newline at end of file
diff --git a/staging2/playbooks/k8s.yaml b/staging2/playbooks/k8s.yaml
new file mode 120000
index 0000000..117aed6
--- /dev/null
+++ b/staging2/playbooks/k8s.yaml
@@ -0,0 +1 @@
+../../common/playbooks/k8s.yaml
\ No newline at end of file
diff --git a/staging2/playbooks/notebooks.yaml b/staging2/playbooks/notebooks.yaml
new file mode 120000
index 0000000..3f1a33f
--- /dev/null
+++ b/staging2/playbooks/notebooks.yaml
@@ -0,0 +1 @@
+../../common/playbooks/notebooks.yaml
\ No newline at end of file
diff --git a/staging2/playbooks/public_keys b/staging2/playbooks/public_keys
new file mode 120000
index 0000000..6ef4918
--- /dev/null
+++ b/staging2/playbooks/public_keys
@@ -0,0 +1 @@
+../../common/playbooks/public_keys
\ No newline at end of file
diff --git a/staging2/playbooks/squid.yaml b/staging2/playbooks/squid.yaml
new file mode 120000
index 0000000..114c327
--- /dev/null
+++ b/staging2/playbooks/squid.yaml
@@ -0,0 +1 @@
+../../common/playbooks/squid.yaml
\ No newline at end of file
diff --git a/staging2/playbooks/templates/etc/exports b/staging2/playbooks/templates/etc/exports
new file mode 120000
index 0000000..3ef288e
--- /dev/null
+++ b/staging2/playbooks/templates/etc/exports
@@ -0,0 +1 @@
+../../../../common/playbooks/templates/etc/exports.ipv46
\ No newline at end of file
diff --git a/staging2/playbooks/templates/etc/mailutils.conf b/staging2/playbooks/templates/etc/mailutils.conf
new file mode 120000
index 0000000..dbd8a1f
--- /dev/null
+++ b/staging2/playbooks/templates/etc/mailutils.conf
@@ -0,0 +1 @@
+../../../../common/playbooks/templates/etc/mailutils.conf
\ No newline at end of file
diff --git a/staging2/playbooks/templates/etc/squid b/staging2/playbooks/templates/etc/squid
new file mode 120000
index 0000000..352b598
--- /dev/null
+++ b/staging2/playbooks/templates/etc/squid
@@ -0,0 +1 @@
+../../../../common/playbooks/templates/etc/squid
\ No newline at end of file
diff --git a/staging2/playbooks/upgrade.yaml b/staging2/playbooks/upgrade.yaml
new file mode 120000
index 0000000..0f9e3f4
--- /dev/null
+++ b/staging2/playbooks/upgrade.yaml
@@ -0,0 +1 @@
+../../common/playbooks/upgrade.yaml
\ No newline at end of file
diff --git a/staging2/terraform/terraform.tfvars b/staging2/terraform/terraform.tfvars
index 41038d0..fbd4191 100644
--- a/staging2/terraform/terraform.tfvars
+++ b/staging2/terraform/terraform.tfvars
@@ -6,7 +6,7 @@ site_name = "staging2"
# These may need some adjustment for your provider
master_flavor_name = "l2.c4r8.100"
-worker_flavor_name = "l2.c8r16.100"
+worker_flavor_name = "l2.c16r32.100"
# XXX: replace this for GPU flavor, once available
gpu_flavor_name = "l2.c2r4.100"
@@ -20,10 +20,10 @@ gpu_workers = 0
docker_volumes_size = 384
# NFS volume
-nfs_volume_size = 256
+nfs_volume_size = 200
# scratch volume
-scratch_volumes_size = 128
+scratch_volumes_size = 0
# squid volume
squid_volume_size = 128
diff --git a/staging2/terraform/vms.tf b/staging2/terraform/vms.tf
index 94976b0..d46b46e 100644
--- a/staging2/terraform/vms.tf
+++ b/staging2/terraform/vms.tf
@@ -42,7 +42,7 @@ resource "openstack_networking_secgroup_rule_v2" "ping6" {
ethertype = "IPv6"
port_range_min = 128
port_range_max = 0
- protocol = "icmp"
+ protocol = "icmp" # icmp / ipv6-icmp
remote_ip_prefix = "::/0"
security_group_id = openstack_networking_secgroup_v2.ping.id
}
--
GitLab