From b367c8f72c8ff5e5aeb5f02ef0fbf76ac4080d5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Franti=C5=A1ek=20Dvo=C5=99=C3=A1k?= <valtri@civ.zcu.cz>
Date: Tue, 27 Aug 2024 15:09:06 +0000
Subject: [PATCH] Logging using Fluent Bit
* outputs: GELF, Elasticsearch
* generic options from Vault
* tuned index for Elasticsearch
* TLS support including client certificate
* cluser name and tag in logs
---
cesnet-central/deploy.sh | 2 +
cesnet-central/playbooks/security-logs.yaml | 1 +
.../templates/fluent-bit-secrets.yaml.j2 | 1 +
.../playbooks/templates/fluent-bit.yaml.j2 | 1 +
common/playbooks/security-logs.yaml | 91 ++++++++++++++
.../templates/fluent-bit-secrets.yaml.j2 | 19 +++
common/playbooks/templates/fluent-bit.yaml.j2 | 115 ++++++++++++++++++
staging1/deploy.sh | 1 +
staging1/playbooks/security-logs.yaml | 1 +
.../templates/fluent-bit-secrets.yaml.j2 | 1 +
.../playbooks/templates/fluent-bit.yaml.j2 | 1 +
11 files changed, 234 insertions(+)
create mode 120000 cesnet-central/playbooks/security-logs.yaml
create mode 120000 cesnet-central/playbooks/templates/fluent-bit-secrets.yaml.j2
create mode 120000 cesnet-central/playbooks/templates/fluent-bit.yaml.j2
create mode 100644 common/playbooks/security-logs.yaml
create mode 100644 common/playbooks/templates/fluent-bit-secrets.yaml.j2
create mode 100644 common/playbooks/templates/fluent-bit.yaml.j2
create mode 120000 staging1/playbooks/security-logs.yaml
create mode 120000 staging1/playbooks/templates/fluent-bit-secrets.yaml.j2
create mode 120000 staging1/playbooks/templates/fluent-bit.yaml.j2
diff --git a/cesnet-central/deploy.sh b/cesnet-central/deploy.sh
index d5606cf..764f9c6 100755
--- a/cesnet-central/deploy.sh
+++ b/cesnet-central/deploy.sh
@@ -27,3 +27,5 @@ ansible-playbook playbooks/cvmfs.yaml
# wait for finish
while ansible -m command -a 'kubectl get pods --all-namespaces' master | tail -n +3 | grep -v ' Running '; do sleep 5; done
+
+ansible-playbook playbooks/security-logs.yaml
diff --git a/cesnet-central/playbooks/security-logs.yaml b/cesnet-central/playbooks/security-logs.yaml
new file mode 120000
index 0000000..0149b19
--- /dev/null
+++ b/cesnet-central/playbooks/security-logs.yaml
@@ -0,0 +1 @@
+../../common/playbooks/security-logs.yaml
\ No newline at end of file
diff --git a/cesnet-central/playbooks/templates/fluent-bit-secrets.yaml.j2 b/cesnet-central/playbooks/templates/fluent-bit-secrets.yaml.j2
new file mode 120000
index 0000000..c64dcee
--- /dev/null
+++ b/cesnet-central/playbooks/templates/fluent-bit-secrets.yaml.j2
@@ -0,0 +1 @@
+../../../common/playbooks/templates/fluent-bit-secrets.yaml.j2
\ No newline at end of file
diff --git a/cesnet-central/playbooks/templates/fluent-bit.yaml.j2 b/cesnet-central/playbooks/templates/fluent-bit.yaml.j2
new file mode 120000
index 0000000..4ccab19
--- /dev/null
+++ b/cesnet-central/playbooks/templates/fluent-bit.yaml.j2
@@ -0,0 +1 @@
+../../../common/playbooks/templates/fluent-bit.yaml.j2
\ No newline at end of file
diff --git a/common/playbooks/security-logs.yaml b/common/playbooks/security-logs.yaml
new file mode 100644
index 0000000..ee20f2e
--- /dev/null
+++ b/common/playbooks/security-logs.yaml
@@ -0,0 +1,91 @@
+---
+#
+# Secrets in "/{{ site_name }}":
+#
+# * fluent_es_host (optional): enable elasticsearch output
+# * fluent_es_index: Index option (when used, 'node-' or 'kube-' prefix is added)
+# * fluent_es_*: elasticsearch output additional options (tls, http_user, ...)
+#
+# * fluent_gelf_host (optional): enable graylog output
+# * fluent_gelf_mode (optional, "tls", "tcp", or "udp")
+# * fluent_gelf_*: graylog output additional options
+#
+# Secrets in "/{{ site_name }}" related to TLS:
+#
+# * fluent_secrets_ca (optional): propagated to /secrets/fluent.ca
+# * fluent_secrets_crt (optional): propagated to /secrets/fluent.crt
+# * fluent_secrets_key (optional): propagated to /secrets/fluent.key
+# * fluent_*_tls (optional): "On"
+# * fluent_*_tls.ca_file (optional): "/secrets/fluent.ca"
+# * fluent_*_tls.crt_file (optional): "/secrets/fluent.crt"
+# * fluent_*_tls.key_file (optional): "/secrets/fluent.key"
+# * fluent_*_tls.key_password (optional)
+# * fluent_*_tls.verify (optional)
+# * fluent_*_tls.verify_hostname (optional): "On"
+# * fluent_*_tls.vhost (optional)
+#
+# Self-sign certificate HOWTO (for TLS clients): [1]
+#
+# openssl req -x509 -newkey rsa:4096 -sha256 -nodes -keyout self_signed.key -out self_signed.crt -subj "/CN=test.host.net"
+#
+# [1] https://docs.fluentbit.io/manual/administration/transport-security#tips-and-tricks
+#
+# For GELF: add self_signed.crt to authorized client certificates directory.
+#
+- name: Fluent Bit Configuration
+ hosts: master
+ become: true
+ vars:
+ namespace: fluent-bit
+ version: "0.47.7" # app 3.1.6
+ tasks:
+ - name: Configure helm repo
+ shell: |-
+ helm repo add fluent https://fluent.github.io/helm-charts
+ helm repo update
+ when: "'fluent' not in ansible_local.helm_repos | map(attribute='name') | list"
+ - name: Get Secrets from Vault
+ set_fact:
+ secrets: "{{ lookup('community.hashi_vault.hashi_vault', vault_mount_point + '/site-' + site_name,
+ token_validate=false) }}"
+ - name: Debug Secrets
+ debug:
+ msg: "{{ item.key }} = {{ item.value }}"
+ loop: "{{ secrets | dict2items }}"
+ - name: Set Fluent TLS Fact From Secrets
+ set_fact:
+ fluent_has_tls: "{{ 'fluent_secrets_ca' in secrets or 'fluent_secrets_crt' in secrets or 'fluent_secrets_key' in secrets }}"
+ - name: Create Fluent TLS Secrets File
+ template:
+ src: templates/fluent-bit-secrets.yaml.j2
+ dest: /tmp/fluent-bit-secrets.yaml
+ mode: 0600
+ when: fluent_has_tls
+ - name: Create Fluent TLS Secrets Object
+ command:
+ cmd: kubectl apply -f /tmp/fluent-bit-secrets.yaml
+ environment:
+ KUBECONFIG: /etc/kubernetes/admin.conf
+ PATH: /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin
+ when: fluent_has_tls
+ - name: Fluent Bit Configuration
+ template:
+ src: templates/fluent-bit.yaml.j2
+ dest: /tmp/fluent-bit.yaml
+ mode: 0600
+ - name: Deploy/upgrade Fluent Bit
+ shell: |-
+ helm status --namespace {{ namespace }} fluent-bit
+ if [ $? -ne 0 ]; then
+ helm install --create-namespace --namespace {{ namespace }} \
+ -f /tmp/fluent-bit.yaml \
+ fluent-bit fluent/fluent-bit
+ else
+ helm upgrade --namespace {{ namespace }} \
+ -f /tmp/fluent-bit.yaml \
+ fluent-bit fluent/fluent-bit
+ fi
+ environment:
+ KUBECONFIG: /etc/kubernetes/admin.conf
+ PATH: /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin
+ when: true
diff --git a/common/playbooks/templates/fluent-bit-secrets.yaml.j2 b/common/playbooks/templates/fluent-bit-secrets.yaml.j2
new file mode 100644
index 0000000..6c2dcba
--- /dev/null
+++ b/common/playbooks/templates/fluent-bit-secrets.yaml.j2
@@ -0,0 +1,19 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+ name: fluent-output-tls
+ namespace: {{ namespace }}
+stringData:
+{% if 'fluent_secrets_ca' in secrets %}
+ ca.crt: |
+ {{ secrets['fluent_secrets_ca'] | indent(6) }}
+{%- endif %}
+{% if 'fluent_secrets_crt' in secrets %}
+ fluent.crt: |
+ {{ secrets['fluent_secrets_crt'] | indent(6) }}
+{%- endif %}
+{% if 'fluent_secrets_key' in secrets %}
+ fluent.key: |
+ {{ secrets['fluent_secrets_key'] | indent(6) }}
+{%- endif %}
diff --git a/common/playbooks/templates/fluent-bit.yaml.j2 b/common/playbooks/templates/fluent-bit.yaml.j2
new file mode 100644
index 0000000..e16921d
--- /dev/null
+++ b/common/playbooks/templates/fluent-bit.yaml.j2
@@ -0,0 +1,115 @@
+---
+config:
+ inputs: |
+ [INPUT]
+ Name tail
+ Path /var/log/containers/*.log
+ DB /var/log/fluent-bit.db
+ multiline.parser docker, cri
+ Tag kube.*
+ Mem_Buf_Limit 5MB
+ # Skip_Long_Lines On
+
+ [INPUT]
+ Name systemd
+ Tag host.*
+ Systemd_Filter _SYSTEMD_UNIT=kubelet.service
+ Read_From_Tail On
+
+ filters: |
+ [FILTER]
+ Name kubernetes
+ Match kube.*
+ Merge_Log_Key log
+ Merge_Log On
+ Keep_Log Off
+ Annotations Off
+ Labels Off
+ K8S-Logging.Parser On
+ K8S-Logging.Exclude On
+
+ [FILTER]
+ Name modify
+ Match *
+ Add cluster {{ site_name }}
+ Add tag eosc
+
+ outputs: |
+ # [OUTPUT]
+ # Name stdout
+ # Match host.*
+
+ # [OUTPUT]
+ # Name stdout
+ # Match kube.*
+
+{% if 'fluent_es_host' in secrets %}
+ [OUTPUT]
+ Name es
+ Match kube.*
+{% for key, value in secrets.items() %}
+{% if key | regex_search('^fluent_es_') %}
+{% if key == 'fluent_es_index' %}
+{% set value = ['kube', value | default(omit)] | join('-') %}
+{% endif %}
+ {{ '%-23s' | format(key | regex_replace('^fluent_es_', '') | title) }} {{ value }}
+{% endif %}
+{% endfor %}
+
+ [OUTPUT]
+ Name es
+ Match node.*
+{% for key, value in secrets.items() %}
+{% if key | regex_search('^fluent_es_') %}
+{% if key == 'fluent_es_index' %}
+{% set value = ['node', value | default(omit)] | join('-') %}
+{% endif %}
+ {{ '%-23s' | format(key | regex_replace('^fluent_es_', '') | title) }} {{ value }}
+{% endif %}
+{% endfor %}
+
+{% endif %}
+{% if 'fluent_gelf_host' in secrets %}
+ [OUTPUT]
+ Name gelf
+ Match kube.*
+ Gelf_Host_Key host
+ Gelf_Short_Message_Key log
+{% for key, value in secrets.items() %}
+{% if key | regex_search('^fluent_gelf_') %}
+ {{ '%-23s' | format(key | regex_replace('^fluent_gelf_', '')) | title }} {{ value }}
+{% endif %}
+{% endfor %}
+
+ [OUTPUT]
+ Name gelf
+ Match host.*
+ Gelf_Host_Key _HOSTNAME
+ Gelf_Short_Message_Key MESSAGE
+{% for key, value in secrets.items() %}
+{% if key | regex_search('^fluent_gelf_') %}
+ {{ '%-23s' | format(key | regex_replace('^fluent_gelf_', '')) | title }} {{ value }}
+{% endif %}
+{% endfor %}
+
+{% endif %}
+{% if fluent_has_tls %}
+extraVolumes:
+ - name: fluent-output-tls
+ secret:
+ secretName: fluent-output-tls
+
+extraVolumeMounts:
+ - name: fluent-output-tls
+ readOnly: true
+ mountPath: "/secrets"
+
+{% endif %}
+# daemonset runnable on control plane nodes
+tolerations:
+ - key: node-role.kubernetes.io/control-plane
+ operator: Exists
+ effect: NoSchedule
+ - key: node-role.kubernetes.io/master
+ operator: Exists
+ effect: NoSchedule
diff --git a/staging1/deploy.sh b/staging1/deploy.sh
index 68cffd8..d509ed3 100755
--- a/staging1/deploy.sh
+++ b/staging1/deploy.sh
@@ -55,3 +55,4 @@ ansible-playbook playbooks/cvmfs.yaml
while ansible -m command -a 'kubectl get pods --all-namespaces' master | tail -n +3 | grep -v ' Running '; do sleep 5; done
ansible-playbook playbooks/security-assets.yaml
+ansible-playbook playbooks/security-logs.yaml
diff --git a/staging1/playbooks/security-logs.yaml b/staging1/playbooks/security-logs.yaml
new file mode 120000
index 0000000..0149b19
--- /dev/null
+++ b/staging1/playbooks/security-logs.yaml
@@ -0,0 +1 @@
+../../common/playbooks/security-logs.yaml
\ No newline at end of file
diff --git a/staging1/playbooks/templates/fluent-bit-secrets.yaml.j2 b/staging1/playbooks/templates/fluent-bit-secrets.yaml.j2
new file mode 120000
index 0000000..86f2455
--- /dev/null
+++ b/staging1/playbooks/templates/fluent-bit-secrets.yaml.j2
@@ -0,0 +1 @@
+/home/valtri/notebooks-operations.eosc/common/playbooks/templates/fluent-bit-secrets.yaml.j2
\ No newline at end of file
diff --git a/staging1/playbooks/templates/fluent-bit.yaml.j2 b/staging1/playbooks/templates/fluent-bit.yaml.j2
new file mode 120000
index 0000000..4ccab19
--- /dev/null
+++ b/staging1/playbooks/templates/fluent-bit.yaml.j2
@@ -0,0 +1 @@
+../../../common/playbooks/templates/fluent-bit.yaml.j2
\ No newline at end of file
--
GitLab