Overview
Image Added
Pre Requisits
Install Brew (Mac)
...
Install Fluentbit
Code Block |
---|
$ mkdir tmp
$ helmcd repotmp
$ addgit fluentclone https://fluent.github.iocom/fluent/helm-charts.git
"fluent" has been added to your repositories
$ helm install fluent-bit fluent$ cd helm-charts/charts/fluent-bit
NAME: fluent-bit
LAST DEPLOYED: Wed Jul 14 09:04:58 2021
NAMESPACE: default
STATUS: deployed
REVISION: 1
NOTES:
Get Fluent Bit build information by running these commands:
export POD_NAME=$(kubectl get pods --namespace default -l "app.kubernetes.io/name=fluent-bit,app.kubernetes.io/instance=fluent-bit" -o jsonpath="{.items[0].metadata.name}")
echo "curl http://127.0.0.1:2020 for Fluent Bit build information"
kubectl --namespace default port-forward $POD_NAME 2020:2020
|
$ helm show values fluent/fluent-bit
Add the output config for azure. We can comment out the other OUTPUTS since we won't be using them.
Code Block |
---|
...
[OUTPUT]
Name azure
Match *
Customer_ID XXX
Shared_Key XXXX
|
Install
Code Block |
---|
$ helm install fluent-bit . |
Verify that it has been installed
Code Block |
---|
$ kubectl get pods |
Output:
Code Block |
---|
NAME READY STATUS RESTARTS AGE
fluent-bit-d7hr2 1/1 Running 0 38s |
Check fluent-bit logs for errors
Code Block |
---|
kubectl logs -f fluent-bit-d7hr2 |
Generate Some Logs
Using our test pod, we will generate some failed login attempts
Code Block |
---|
$ ssh root@localhost -p 30022
root@localhost's password:
Permission denied, please try again.
root@localhost's password:
Permission denied, please try again.
root@localhost's password:
root@localhost: Permission denied (publickey,password). |
View Logs in Azure Sentinel
Image Added
Example Queries
Code Block |
---|
fluentbit_CL
fluentbit_CL
| where kubernetes_labels_app_s contains "ubuntu"
fluentbit_CL
| where kubernetes_labels_app_s contains "ubuntu"
| where log_s contains "Failed password"
|
Code Block |
---|
# Default values for fluent-bit.
# kind -- DaemonSet or Deployment
kind: DaemonSet
# replicaCount -- Only applicable if kind=Deployment
replicaCount: 1
image:
repository: fluent/fluent-bit
pullPolicy: Always
# tag:
testFramework:
image:
repository: busybox
pullPolicy: Always
tag: latest
imagePullSecrets: []
nameOverride: ""
fullnameOverride: ""
serviceAccount:
create: true
annotations: {}
name:
rbac:
create: true
podSecurityPolicy:
create: false
annotations: {}
podSecurityContext:
{}
# fsGroup: 2000
dnsConfig: {}
# nameservers:
# - 1.2.3.4
# searches:
# - ns1.svc.cluster-domain.example
# - my.dns.search.suffix
# options:
# - name: ndots
# value: "2"
# - name: edns0
hostAliases: []
# - ip: "1.2.3.4"
# hostnames:
# - "foo.local"
# - "bar.local"
securityContext:
{}
# capabilities:
# drop:
# - ALL
# readOnlyRootFilesystem: true
# runAsNonRoot: true
# runAsUser: 1000
service:
type: ClusterIP
port: 2020
labels:
{}
annotations:
{}
# prometheus.io/path: "/api/v1/metrics/prometheus"
# prometheus.io/port: "2020"
# prometheus.io/scrape: "true"
serviceMonitor:
enabled: false
# namespace: monitoring
# interval: 10s
# scrapeTimeout: 10s
# selector:
# prometheus: my-prometheus
prometheusRule:
enabled: false
# namespace: ""
# additionnalLabels: {}
# rules:
# - alert: NoOutputBytesProcessed
# expr: rate(fluentbit_output_proc_bytes_total[5m]) == 0
# annotations:
# message: |
# Fluent Bit instance {{ $labels.instance }}'s output plugin {{ $labels.name }} has not processed any
# bytes for at least 15 minutes.
# summary: No Output Bytes Processed
# for: 15m
# labels:
# severity: critical
dashboards:
enabled: false
labelKey: grafana_dashboard
annotations: {}
livenessProbe: {}
# httpGet:
# path: /
# port: http
readinessProbe:
# httpGet:
# path: /
# port: http
resources:
{}
# limits:
# cpu: 100m
# memory: 128Mi
# requests:
# cpu: 100m
# memory: 128Mi
nodeSelector: {}
tolerations: []
affinity: {}
podAnnotations: {}
podLabels: {}
priorityClassName: ""
env: []
envFrom: []
extraContainers: []
# - name: do-something
# image: busybox
# command: ['do', 'something']
extraPorts: []
# - port: 5170
# containerPort: 5170
# protocol: TCP
# name: tcp
extraVolumes: []
extraVolumeMounts: []
updateStrategy: {}
# type: RollingUpdate
# rollingUpdate:
# maxUnavailable: 1
# Make use of a pre-defined configmap instead of the one templated here
existingConfigMap: ""
networkPolicy:
enabled: false
# ingress:
# from: []
luaScripts: {}
## https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/configuration-file
config:
service: |
[SERVICE]
Flush 1
Daemon Off
Log_Level info
Parsers_File parsers.conf
Parsers_File custom_parsers.conf
HTTP_Server On
HTTP_Listen 0.0.0.0
HTTP_Port {{ .Values.service.port }}
## https://docs.fluentbit.io/manual/pipeline/inputs
inputs: |
[INPUT]
Name tail
Path /var/log/containers/*.log
Parser docker
Tag kube.*
Mem_Buf_Limit 5MB
Skip_Long_Lines On
[INPUT]
Name systemd
Tag host.*
Systemd_Filter _SYSTEMD_UNIT=kubelet.service
Read_From_Tail On
## https://docs.fluentbit.io/manual/pipeline/filters
filters: |
[FILTER]
Name kubernetes
Match kube.*
Merge_Log On
Keep_Log Off
K8S-Logging.Parser On
K8S-Logging.Exclude On
## https://docs.fluentbit.io/manual/pipeline/outputs
outputs: |
[OUTPUT]
Name es
Match kube.*
Host elasticsearch-master
Logstash_Format On
Retry_Limit False
[OUTPUT]
Name es
Match host.*
Host elasticsearch-master
Logstash_Format On
Logstash_Prefix node
Retry_Limit False
## https://docs.fluentbit.io/manual/pipeline/parsers
customParsers: |
[PARSER]
Name docker_no_time
Format json
Time_Keep Off
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L
# The config volume is mounted by default, either to the existingConfigMap value, or the default of "fluent-bit.fullname"
volumeMounts:
- name: config
mountPath: /fluent-bit/etc/fluent-bit.conf
subPath: fluent-bit.conf
- name: config
mountPath: /fluent-bit/etc/custom_parsers.conf
subPath: custom_parsers.conf
daemonSetVolumes:
- name: varlog
hostPath:
path: /var/log
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
- name: etcmachineid
hostPath:
path: /etc/machine-id
type: File
daemonSetVolumeMounts:
- name: varlog
mountPath: /var/log
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
- name: etcmachineid
mountPath: /etc/machine-id
readOnly: true
args: []
command: []
initContainers: []
# - name: do-something
# image: busybox
# command: ['do', 'something'] |
References
...