Logging using Fluent Bit

* outputs: GELF, Elasticsearch
* generic options from Vault
* tuned index for Elasticsearch
* TLS support including client certificate
* cluser name and tag in logs

cesnet-central/deploy.sh

@@ -27,3 +27,5 @@ ansible-playbook playbooks/cvmfs.yaml
 
 # wait for finish
 while ansible -m command -a 'kubectl get pods --all-namespaces' master | tail -n +3 | grep -v ' Running '; do sleep 5; done
+
+ansible-playbook playbooks/security-logs.yaml

cesnet-central/playbooks/security-logs.yaml

+../../common/playbooks/security-logs.yaml
\ No newline at end of file

cesnet-central/playbooks/templates/fluent-bit-secrets.yaml.j2

+../../../common/playbooks/templates/fluent-bit-secrets.yaml.j2
\ No newline at end of file

cesnet-central/playbooks/templates/fluent-bit.yaml.j2

+../../../common/playbooks/templates/fluent-bit.yaml.j2
\ No newline at end of file

common/playbooks/security-logs.yaml

+---
+#
+# Secrets in "/{{ site_name }}":
+#
+# * fluent_es_host (optional): enable elasticsearch output
+# * fluent_es_index: Index option (when used, 'node-' or 'kube-' prefix is added)
+# * fluent_es_*: elasticsearch output additional options (tls, http_user, ...)
+#
+# * fluent_gelf_host (optional): enable graylog output
+# * fluent_gelf_mode (optional, "tls", "tcp", or "udp")
+# * fluent_gelf_*: graylog output additional options
+#
+# Secrets in "/{{ site_name }}" related to TLS:
+#
+# * fluent_secrets_ca (optional): propagated to /secrets/fluent.ca
+# * fluent_secrets_crt (optional): propagated to /secrets/fluent.crt
+# * fluent_secrets_key (optional): propagated to /secrets/fluent.key
+# * fluent_*_tls (optional): "On"
+# * fluent_*_tls.ca_file (optional): "/secrets/fluent.ca"
+# * fluent_*_tls.crt_file (optional): "/secrets/fluent.crt"
+# * fluent_*_tls.key_file (optional): "/secrets/fluent.key"
+# * fluent_*_tls.key_password (optional)
+# * fluent_*_tls.verify (optional)
+# * fluent_*_tls.verify_hostname (optional): "On"
+# * fluent_*_tls.vhost (optional)
+#
+# Self-sign certificate HOWTO (for TLS clients): [1]
+#
+# openssl req -x509 -newkey rsa:4096 -sha256 -nodes -keyout self_signed.key -out self_signed.crt -subj "/CN=test.host.net"
+#
+# [1] https://docs.fluentbit.io/manual/administration/transport-security#tips-and-tricks
+#
+# For GELF: add self_signed.crt to authorized client certificates directory.
+#
+- name: Fluent Bit Configuration
+  hosts: master
+  become: true
+  vars:
+    namespace: fluent-bit
+    version: "0.47.7" # app 3.1.6
+  tasks:
+    - name: Configure helm repo
+      shell: |-
+        helm repo add fluent https://fluent.github.io/helm-charts
+        helm repo update
+      when: "'fluent' not in ansible_local.helm_repos | map(attribute='name') | list"
+    - name: Get Secrets from Vault
+      set_fact:
+        secrets: "{{ lookup('community.hashi_vault.hashi_vault', vault_mount_point + '/site-' + site_name,
+          token_validate=false) }}"
+    - name: Debug Secrets
+      debug:
+        msg: "{{ item.key }} = {{ item.value }}"
+      loop: "{{ secrets | dict2items }}"
+    - name: Set Fluent TLS Fact From Secrets
+      set_fact:
+        fluent_has_tls: "{{ 'fluent_secrets_ca' in secrets or 'fluent_secrets_crt' in secrets or 'fluent_secrets_key' in  secrets }}"
+    - name: Create Fluent TLS Secrets File
+      template:
+        src: templates/fluent-bit-secrets.yaml.j2
+        dest: /tmp/fluent-bit-secrets.yaml
+        mode: 0600
+      when: fluent_has_tls
+    - name: Create Fluent TLS Secrets Object
+      command:
+        cmd: kubectl apply -f /tmp/fluent-bit-secrets.yaml
+      environment:
+        KUBECONFIG: /etc/kubernetes/admin.conf
+        PATH: /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin
+      when: fluent_has_tls
+    - name: Fluent Bit Configuration
+      template:
+        src: templates/fluent-bit.yaml.j2
+        dest: /tmp/fluent-bit.yaml
+        mode: 0600
+    - name: Deploy/upgrade Fluent Bit
+      shell: |-
+        helm status --namespace {{ namespace }} fluent-bit
+        if [ $? -ne 0 ]; then
+            helm install --create-namespace --namespace {{ namespace }} \
+                -f /tmp/fluent-bit.yaml \
+                fluent-bit fluent/fluent-bit
+        else
+            helm upgrade --namespace {{ namespace }} \
+                -f /tmp/fluent-bit.yaml \
+                fluent-bit fluent/fluent-bit
+        fi
+      environment:
+        KUBECONFIG: /etc/kubernetes/admin.conf
+        PATH: /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin
+      when: true

common/playbooks/templates/fluent-bit-secrets.yaml.j2

+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: fluent-output-tls
+  namespace: {{ namespace }}
+stringData:
+{% if 'fluent_secrets_ca' in secrets %}
+    ca.crt: |
+      {{ secrets['fluent_secrets_ca'] | indent(6) }}
+{%- endif %}
+{% if 'fluent_secrets_crt' in secrets %}
+    fluent.crt: |
+      {{ secrets['fluent_secrets_crt'] | indent(6) }}
+{%- endif %}
+{% if 'fluent_secrets_key' in secrets %}
+    fluent.key: |
+      {{ secrets['fluent_secrets_key'] | indent(6) }}
+{%- endif %}

common/playbooks/templates/fluent-bit.yaml.j2

+---
+config:
+  inputs: |
+    [INPUT]
+        Name                     tail
+        Path                     /var/log/containers/*.log
+        DB                       /var/log/fluent-bit.db
+        multiline.parser         docker, cri
+        Tag                      kube.*
+        Mem_Buf_Limit            5MB
+        # Skip_Long_Lines On
+
+    [INPUT]
+        Name systemd
+        Tag host.*
+        Systemd_Filter _SYSTEMD_UNIT=kubelet.service
+        Read_From_Tail On
+
+  filters: |
+    [FILTER]
+        Name                    kubernetes
+        Match                   kube.*
+        Merge_Log_Key           log
+        Merge_Log               On
+        Keep_Log                Off
+        Annotations             Off
+        Labels                  Off
+        K8S-Logging.Parser      On
+        K8S-Logging.Exclude     On
+
+    [FILTER]
+        Name                    modify
+        Match                   *
+        Add                     cluster {{ site_name }}
+        Add                     tag eosc
+
+  outputs: |
+    # [OUTPUT]
+    #     Name                    stdout
+    #     Match                   host.*
+
+    # [OUTPUT]
+    #     Name                    stdout
+    #     Match                   kube.*
+
+{% if 'fluent_es_host' in secrets %}
+    [OUTPUT]
+        Name                    es
+        Match                   kube.*
+{% for key, value in secrets.items() %}
+{% if key | regex_search('^fluent_es_') %}
+{% if key == 'fluent_es_index' %}
+{% set value = ['kube', value | default(omit)] | join('-') %}
+{% endif %}
+        {{ '%-23s' | format(key | regex_replace('^fluent_es_', '') | title) }} {{ value }}
+{% endif %}
+{% endfor %}
+
+    [OUTPUT]
+        Name                    es
+        Match                   node.*
+{% for key, value in secrets.items() %}
+{% if key | regex_search('^fluent_es_') %}
+{% if key == 'fluent_es_index' %}
+{% set value = ['node', value | default(omit)] | join('-') %}
+{% endif %}
+        {{ '%-23s' | format(key | regex_replace('^fluent_es_', '') | title) }} {{ value }}
+{% endif %}
+{% endfor %}
+
+{% endif %}
+{% if 'fluent_gelf_host' in secrets %}
+    [OUTPUT]
+        Name                    gelf
+        Match                   kube.*
+        Gelf_Host_Key           host
+        Gelf_Short_Message_Key  log
+{% for key, value in secrets.items() %}
+{% if key | regex_search('^fluent_gelf_') %}
+        {{ '%-23s' | format(key | regex_replace('^fluent_gelf_', '')) | title }} {{ value }}
+{% endif %}
+{% endfor %}
+
+    [OUTPUT]
+        Name                    gelf
+        Match                   host.*
+        Gelf_Host_Key           _HOSTNAME
+        Gelf_Short_Message_Key  MESSAGE
+{% for key, value in secrets.items() %}
+{% if key | regex_search('^fluent_gelf_') %}
+        {{ '%-23s' | format(key | regex_replace('^fluent_gelf_', '')) | title }} {{ value }}
+{% endif %}
+{% endfor %}
+
+{% endif %}
+{% if fluent_has_tls %}
+extraVolumes:
+  - name: fluent-output-tls
+    secret:
+      secretName: fluent-output-tls
+
+extraVolumeMounts:
+  - name: fluent-output-tls
+    readOnly: true
+    mountPath: "/secrets"
+
+{% endif %}
+# daemonset runnable on control plane nodes
+tolerations:
+  - key: node-role.kubernetes.io/control-plane
+    operator: Exists
+    effect: NoSchedule
+  - key: node-role.kubernetes.io/master
+    operator: Exists
+    effect: NoSchedule

staging1/deploy.sh

@@ -55,3 +55,4 @@ ansible-playbook playbooks/cvmfs.yaml
 while ansible -m command -a 'kubectl get pods --all-namespaces' master | tail -n +3 | grep -v ' Running '; do sleep 5; done
 
 ansible-playbook playbooks/security-assets.yaml
+ansible-playbook playbooks/security-logs.yaml

staging1/playbooks/security-logs.yaml

+../../common/playbooks/security-logs.yaml
\ No newline at end of file

staging1/playbooks/templates/fluent-bit-secrets.yaml.j2

+/home/valtri/notebooks-operations.eosc/common/playbooks/templates/fluent-bit-secrets.yaml.j2
\ No newline at end of file

staging1/playbooks/templates/fluent-bit.yaml.j2

+../../../common/playbooks/templates/fluent-bit.yaml.j2
\ No newline at end of file