ENVRI-Hub initial import

envri-hub/ansible.cfg

+[defaults]
+inventory=inventory
+
+[diff]
+always=true

envri-hub/deploy.sh

+#! /bin/bash -xe
+
+#
+# Deploy ENVRI-HUB VRE
+#
+
+ip=admin.envri-vre.cloud.cesnet.cz
+# wait for ping and ssh
+{
+	while ! ping -c 1 "$ip"; do sleep 5; done
+	ssh-keygen -R "$ip"
+	while ! ssh ubuntu@"$ip" -o ConnectTimeout=10 -o PreferredAuthentications=publickey -o StrictHostKeyChecking=no :; do sleep 10; done
+}
+
+# check ssh access
+ansible -m command -a 'uname -a' allnodes
+
+# wait cloud-init
+ansible -m shell -a 'while ! test -f /var/lib/cloud/instance/boot-finished; do sleep 2; done' allnodes
+
+# kubernetes
+# ansible-playbook playbooks/squid.yaml
+# ansible-playbook playbooks/cvmfs.yaml
+
+# wait for finish
+# while ansible -m command -a 'kubectl get pods --all-namespaces' master | tail -n +3 | grep -Ev ' (Running|Completed) '; do sleep 5; done

envri-hub/deployments/envri-hub.yaml

+---
+proxy:
+  service:
+    type: NodePort
+
+ingress:
+  enabled: true
+  annotations:
+    kubernetes.io/ingress.class: "nginx"
+    kubernetes.io/tls-acme: "true"
+  hosts:
+    - "{{ notebooks_hostname }}"
+  tls:
+    - hosts:
+        - "{{ notebooks_hostname }}"
+      secretName: acme-tls-envri-hub
+
+singleuser:
+  # keep resource limits in sync with:
+  # - profileList
+  storage:
+    capacity: 10Gi
+    dynamic:
+      pvcNameTemplate: claim-{userid}{servername}
+      volumeNameTemplate: vol-{userid}{servername}
+      storageClass: csi-sc-cinderplugin
+    extraVolumes:
+      # - name: cvmfs-host
+      #   hostPath:
+      #     path: /cvmfs
+      #     type: Directory
+    extraVolumeMounts:
+      # - name: cvmfs-host
+      #   mountPath: "/cvmfs:shared"
+  memory:
+    limit: 4G
+    guarantee: 128M
+  cpu:
+    limit: 2
+    guarantee: .2
+  defaultUrl: "/lab"
+  image:
+    name: eginotebooks/single-user
+    tag: "sha-0f4a63c"
+  profileList:
+    - display_name: Default environment - 4 GB RAM / 2 core
+      description: >
+        The default notebook environment includes Python, R, Julia and Octave kernels.
+      default: true
+      kubespawner_override:
+        args:
+          - "--CondaKernelSpecManager.env_filter='/opt/conda$'"
+
+hub:
+  db:
+    pvc:
+      storageClassName: csi-sc-cinderplugin
+  services:
+    status:
+      url: "http://status-web/"
+      admin: true
+  image:
+    name: eginotebooks/hub
+    tag: "sha-2fa0db6"
+  config:
+    Authenticator:
+      enable_auth_state: true
+      admin_users:
+        # valtri@civ.zcu.cz
+        - 52cc7599bd1553c9d63e34e4c90b7e84d44967490c28bb4c53fe97b0c881d677@egi.eu
+      allowed_groups:
+        - urn:mace:egi.eu:group:envri-hub-next-all#sso.egi.eu
+      claim_groups_key: "eduperson_entitlement"
+    EGICheckinAuthenticator:
+      checkin_host: "{{ secrets['checkin_host']}}"
+      authorize_url: "https://{{ secrets['checkin_host']}}/auth/realms/egi/protocol/openid-connect/auth"
+      token_url: "https://{{ secrets['checkin_host']}}/auth/realms/egi/protocol/openid-connect/token"
+      userdata_url: "https://{{ secrets['checkin_host']}}/auth/realms/egi/protocol/openid-connect/userinfo"
+      client_id: "{{ secrets['client_id'] }}"
+      client_secret: "{{ secrets['client_secret'] }}"
+      oauth_callback_url: "https://{{ notebooks_hostname }}/hub/oauth_callback"
+      # (unused, for JWT wrapper)
+      openid_configuration_url: "https://{{ secrets['checkin_host'] }}/.well-known/openid-configuration"
+      scope: ["openid", "profile", "email", "offline_access", "eduperson_entitlement"]
+      username_key: "sub"
+    JupyterHub:
+      admin_access: true
+      authenticate_prometheus: false
+      authenticator_class: egi_notebooks_hub.egiauthenticator.EGICheckinAuthenticator
+      spawner_class: egi_notebooks_hub.onedata.OnedataSpawner
+    OnedataSpawner:
+      http_timeout: 60
+      token_mount_path: "/var/run/secrets/oidc/"
+      args:
+        - "--FileCheckpoints.checkpoint_dir='/home/jovyan/.notebookCheckpoints'"
+    LabApp:
+      check_for_updates_class: jupyterlab.NeverCheckForUpdate
+  extraConfig:
+    egi-notebooks-welcome: |-
+      from egi_notebooks_hub.welcome import WelcomeHandler
+      c.JupyterHub.default_url = "/welcome"
+      c.JupyterHub.extra_handlers = [(r'/welcome', WelcomeHandler)]
+  templatePaths:
+    - /egi-notebooks-hub/egi-templates
+  extraFiles:
+    welcome.html:
+      mountPath: /usr/local/share/jupyterhub/templates/welcome.html
+      stringData: |-
+{%- raw %}
+        {% extends "login.html" %}
+{% endraw %}
+    403.html:
+      mountPath: /usr/local/share/jupyterhub/templates/403.html
+      stringData: |-
+{%- raw %}
+        {% extends "error.html" %}
+        {% block main %}
+        <div class="error">
+          <h1>Unauthorized</h1>
+          <p>You don't have the correct entitlements to access this service.</p>
+          <p>If you think you should be granted access, please open an issue!</p>
+        </div>
+        {% endblock %}
+{% endraw %}

envri-hub/inventory/1-envri-hub.yaml

+---
+# fip:
+#   hosts:
+#     147.251.245.108:
+
+ingress_0:
+  hosts:
+
+ingress:
+  hosts:
+
+master:
+  hosts:
+    10.0.0.11:
+    10.0.0.12:
+    10.0.0.13:
+
+nfs:
+  hosts:
+
+worker:
+  hosts:
+    10.0.0.70:
+    10.0.0.89:
+    10.0.0.251:
+    10.0.0.211:
+
+gpu:
+  hosts:

envri-hub/inventory/99-all.yaml

+---
+allnodes:
+  children:
+    master:
+    ingress:
+    nfs:
+    worker:
+    gpu:
+
+all:
+  vars:
+    ansible_become: yes
+    ansible_user: ubuntu
+    ansible_ssh_common_args: '-o ProxyCommand="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -W %h:%p -q ubuntu@admin.envri-vre.cloud.cesnet.cz" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'
+
+    site_name: envri-hub
+    vault_mount_point: secrets/users/e1662e20-e34b-468c-b0ce-d899bc878364@egi.eu/envri-hub
+
+    notebooks_hostname: envri-vre.cloud.cesnet.cz
+    grafana_hostname: grafana.envri-vre.cloud.cesnet.cz

envri-hub/playbooks/files/etc

+../../../common/playbooks/files/etc
\ No newline at end of file

envri-hub/playbooks/k8s.yaml

+---
+- name: Basic setup
+  hosts: allnodes
+  become: true
+  tasks:
+    - name: Add SSH keys
+      authorized_key:
+        user: ubuntu
+        state: present
+        key: '{{ item }}'
+      with_file:
+        - public_keys/jhradil
+        - public_keys/pailozian
+        - public_keys/sustr
+        - public_keys/valtri
+    - name: Site install packages
+      package:
+        name:
+          - atop
+          - git
+          - mc
+          - vim
+    - name: Site touch
+      file:
+        path: "/{{ site_name | upper }}"
+        state: touch
+        mode: 0644
+
+- name: K8s customization
+  hosts: master[0]
+  become: true
+  tasks:
+    - name: Site k8s cheat sheets
+      copy:
+        dest: /etc/profile.d/k8s-cheats.sh
+        src: files//etc/profile.d/k8s-cheats.sh
+        mode: preserve
+    - name: Wait for helm
+      command: helm version
+      register: result
+      until: result.rc == 0
+      retries: 20
+      delay: 10
+      environment:
+        KUBECONFIG: /etc/kubernetes/admin.conf
+      when: true
+    - name: Create custom fact directory
+      file:
+        path: "/etc/ansible/facts.d"
+        mode: 0755
+        recurse: true
+        state: "directory"
+    - name: Create helm repos custom fact
+      copy:
+        src: files/etc/ansible/facts.d/helm_repos.fact
+        dest: /etc/ansible/facts.d/helm_repos.fact
+        mode: 0755
+    - name: Reload custom facts
+      setup:
+        filter: ansible_local
+    - name: Cert-manager
+      vars:
+        version: 1.16.1
+        config: >-
+          --version={{ version }}
+          --set ingressShim.defaultIssuerName=letsencrypt-prod
+          --set ingressShim.defaultIssuerKind=ClusterIssuer
+          --set ingressShim.defaultIssuerGroup=cert-manager.io
+      shell: |-
+        helm status --namespace cert-manager certs-man
+        if [ $? -ne 0 ]; then
+            kubectl create namespace cert-manager
+            kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v{{ version }}/cert-manager.crds.yaml
+            helm repo add jetstack https://charts.jetstack.io
+            helm repo update
+            helm install --namespace cert-manager {{ config }} certs-man jetstack/cert-manager
+        else
+            helm upgrade --namespace cert-manager {{ config }} certs-man jetstack/cert-manager
+        fi
+      environment:
+        KUBECONFIG: /etc/kubernetes/admin.conf
+        PATH: /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin
+      when: true
+    - name: Cluster issuer file
+      copy:
+        dest: /tmp/clusterissuer.yaml
+        mode: 0644
+        content: |
+          apiVersion: cert-manager.io/v1
+          kind: ClusterIssuer
+          metadata:
+            name: letsencrypt-prod
+          spec:
+            acme:
+              email: valtri@civ.zcu.cz
+              server: https://acme-v02.api.letsencrypt.org/directory
+              privateKeySecretRef:
+                name: cluster-issuer-account-key
+              # Add a single challenge solver, HTTP01 using nginx
+              solvers:
+              - http01:
+                  ingress:
+                    class: nginx
+    - name: Cluster issuer
+      command:
+        kubectl apply -f /tmp/clusterissuer.yaml
+      environment:
+        KUBECONFIG: /etc/kubernetes/admin.conf
+        PATH: /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin
+      when: true

envri-hub/playbooks/notebooks.yaml

+---
+- name: Notebooks deployments
+  hosts: master[0]
+  become: true
+  tasks:
+    - name: Configure helm repo
+      shell: |-
+        helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/
+        helm repo add eginotebooks https://egi-federation.github.io/egi-notebooks-chart/
+        helm repo update
+      when: "'jupyterhub' not in ansible_local.helm_repos | map(attribute='name') | list or
+             'eginotebooks' not in ansible_local.helm_repos | map(attribute='name') | list"
+    - name: Get Secrets from Vault for notebooks
+      vars:
+        name: "{{ item | basename | splitext | first }}"
+      set_fact:
+        deployment_secrets: "{{ deployment_secrets|default({}) | combine({name: lookup('community.hashi_vault.hashi_vault',
+          (vault_mount_point, 'deployment-' + name)  | join('/'),  token_validate=false)}) }}"
+      with_fileglob:
+        - "../deployments/*.yaml"
+    - name: Debug Deployments Secrets
+      debug:
+        msg: "{{ item.key }} = {{ item.value }}"
+      loop: "{{ deployment_secrets | dict2items }}"
+    - name: Copy config file to master
+      vars:
+        name: "{{ item | basename | splitext | first }}"
+        secrets: "{{ deployment_secrets[name] }}"
+      template:
+        src: "{{ item }}"
+        dest: "/tmp/{{ item | basename }}"
+        mode: 0600
+      with_fileglob:
+        - "../deployments/*.yaml"
+    - name: Deploy/upgrade notebook instance
+      vars:
+        name: "{{ item | basename | splitext | first }}"
+        version: "3.2.1" # app 4.0.2 (2023-11-27)
+      shell: |-
+        helm status --namespace {{ name }} {{ name }}
+        if [ $? -ne 0 ]; then
+            helm install --create-namespace --namespace {{ name }} \
+                -f /tmp/{{ item | basename }} --version {{ version }} --timeout 2h \
+                 {{ name }} jupyterhub/jupyterhub
+        else
+            helm upgrade --version {{ version }} -f /tmp/{{ item | basename }} --timeout 2h \
+                --namespace {{ name }} {{ name }} jupyterhub/jupyterhub
+        fi
+      environment:
+        KUBECONFIG: /etc/kubernetes/admin.conf
+        PATH: /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin
+      when: true
+      with_fileglob:
+        - "../deployments/*.yaml"
+    - name: Deploy/upgrade notebook monitoring instance
+      vars:
+        name: "{{ item | basename | splitext | first }}"
+        monitor_version: "0.3.1"
+      shell: |-
+        helm status --namespace {{ name }} {{ name }}-monitor
+        if [ $? -ne 0 ]; then
+            helm install --namespace {{ name }} \
+                -f /tmp/{{ item | basename }} --version {{ monitor_version }} \
+                {{ name }}-monitor eginotebooks/notebooks-monitor
+        else
+            helm upgrade --version {{ monitor_version }} \
+                -f /tmp/{{ item | basename }} --namespace {{ name }} \
+                {{ name }}-monitor eginotebooks/notebooks-monitor
+        fi
+      environment:
+        KUBECONFIG: /etc/kubernetes/admin.conf
+        PATH: /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin
+      when: true
+      with_fileglob:
+        - "../deployments/*.yaml"
+    - name: Configure secrets management for the hub
+      vars:
+        name: "{{ item | basename | splitext | first }}"
+      shell: |-
+        kubectl apply -f - << EOF
+        ---
+        kind: Role
+        apiVersion: rbac.authorization.k8s.io/v1
+        metadata:
+          name: hub-secrets
+          namespace: {{ name }}
+        rules:
+          - apiGroups: [""]       # "" indicates the core API group
+            resources: ["secrets"]
+            verbs: ["get", "watch", "list", "create", "delete", "patch", "update"]
+        ---
+        kind: RoleBinding
+        apiVersion: rbac.authorization.k8s.io/v1
+        metadata:
+          name: hub-secrets
+          namespace: {{ name }}
+        subjects:
+          - kind: ServiceAccount
+            name: hub
+            namespace: {{ name }}
+        roleRef:
+          kind: Role
+          name: hub-secrets
+          apiGroup: rbac.authorization.k8s.io
+        EOF
+      environment:
+        KUBECONFIG: /etc/kubernetes/admin.conf
+        PATH: /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin
+      when: true
+      with_fileglob:
+        - "../deployments/*.yaml"

envri-hub/playbooks/public_keys

+../../common/playbooks/public_keys
\ No newline at end of file