Prometheus and Grafana for ENVRI-Hub

envri-hub/extra

+../common/extra
\ No newline at end of file

envri-hub/playbooks/k8s.yaml

@@ -108,3 +108,92 @@
         KUBECONFIG: /etc/kubernetes/admin.conf
         PATH: /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin
       when: true
+    # Accounting / monitoring needs
+    - name: Helm repo add prometheus-community
+      shell: |-
+        helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
+        helm repo update
+      when: "'prometheus-community' not in ansible_local.helm_repos | map(attribute='name') | list"
+    - name: Prometheus configuration
+      copy:
+        dest: /tmp/prometheus.yaml
+        mode: 0600
+        content: |
+          alertmanager:
+            persistence:
+              storageClass: csi-sc-cinderplugin
+          kube-state-metrics:
+            metricAnnotationsAllowList:
+              - pods=[hub.jupyter.org/username,egi.eu/primary_group,egi.eu/flavor]
+          server:
+            persistentVolume:
+              storageClass: csi-sc-cinderplugin
+    - name: Prometheus
+      vars:
+        version: 25.28.0 # app v2.55.0
+        config: >-
+          --version={{ version }}
+          -f /tmp/prometheus.yaml
+      shell: |-
+        helm status --namespace prometheus prometheus
+        if [ $? -ne 0 ]; then
+            kubectl create ns prometheus >/dev/null 2>&1 || true
+            helm install --namespace prometheus {{ config }} prometheus prometheus-community/prometheus
+        else
+            helm upgrade --namespace prometheus {{ config }} prometheus prometheus-community/prometheus
+        fi
+      environment:
+        KUBECONFIG: /etc/kubernetes/admin.conf
+        PATH: /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin
+      when: true
+    - name: Grafana configuration
+      copy:
+        dest: /tmp/grafana.yaml
+        mode: 0640
+        content: |
+          ingress:
+            enabled: true
+            annotations:
+              kubernetes.io/ingress.class: "nginx"
+              kubernetes.io/tls-acme: "true"
+            hosts:
+            - "{{ grafana_hostname }}"
+            tls:
+            - hosts:
+              - "{{ grafana_hostname }}"
+              secretName: acme-tls-grafana
+          datasources:
+           datasources.yaml:
+             apiVersion: 1
+             datasources:
+              - name: Prometheus
+                type: prometheus
+                access: Server
+                orgId: 1
+                url: http://prometheus-server.prometheus.svc.cluster.local
+                isDefault: true
+                version: 1
+                editable: false
+          sidecar:
+            dashboards:
+              enabled: true
+    - name: Grafana
+      vars:
+        version: 8.5.11 # app 11.3.0
+        config: >-
+          --version={{ version }}
+          -f /tmp/grafana.yaml
+      shell: |-
+        helm status --namespace grafana grafana
+        if [ $? -ne 0 ]; then
+            kubectl create ns grafana
+            helm repo add grafana https://grafana.github.io/helm-charts
+            helm repo update
+            helm install --namespace grafana {{ config }} grafana grafana/grafana
+        else
+            helm upgrade --namespace grafana {{ config }} grafana grafana/grafana
+        fi
+      environment:
+        KUBECONFIG: /etc/kubernetes/admin.conf
+        PATH: /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin
+      when: true

envri-hub/playbooks/notebooks.yaml

@@ -109,3 +109,22 @@
       when: true
       with_fileglob:
         - "../deployments/*.yaml"
+
+    # do the extra bits of configuration
+    # here we should have all the namespaces, pre-requirements in place
+    # XXX: this won't remove things that are delete from the directory
+    - name: Copy extra configuration files
+      copy:
+        src: "{{ item }}"
+        dest: "/tmp/{{ item | basename }}"
+        mode: 0600
+      with_fileglob:
+        - "../extra/*.yaml"
+    - name: Extra configuration
+      command: |-
+          kubectl apply -f /tmp/{{ item | basename }}
+      environment:
+        KUBECONFIG: /etc/kubernetes/admin.conf
+      with_fileglob:
+        - "../extra/*.yaml"
+      when: true