Versions Compared

Key

  • This line was added.
  • This line was removed.
  • Formatting was changed.

Table of Contents


Overview

Image Added

Pre Requisits

Install Brew (Mac)

...

Install Fluentbit


Code Block
$ mkdir tmp 
$ helmcd repotmp
$ addgit fluentclone https://fluent.github.iocom/fluent/helm-charts.git
"fluent" has been added to your repositories

$ helm install fluent-bit fluent$ cd helm-charts/charts/fluent-bit
NAME: fluent-bit
LAST DEPLOYED: Wed Jul 14 09:04:58 2021
NAMESPACE: default
STATUS: deployed
REVISION: 1
NOTES:
Get Fluent Bit build information by running these commands:

export POD_NAME=$(kubectl get pods --namespace default -l "app.kubernetes.io/name=fluent-bit,app.kubernetes.io/instance=fluent-bit" -o jsonpath="{.items[0].metadata.name}")
echo "curl http://127.0.0.1:2020 for Fluent Bit build information"
kubectl --namespace default port-forward $POD_NAME 2020:2020

$ helm show values fluent/fluent-bit

/
$ vi values.yaml 


Add the output config for azure. We can comment out the other OUTPUTS since we won't be using them.

Code Block
...
    [OUTPUT]
        Name azure
		Match *
        Customer_ID XXX
        Shared_Key XXXX

Install

Code Block
$ helm install fluent-bit .


Verify that it has been installed

Code Block
$ kubectl get pods

Output:

Code Block
NAME                     READY   STATUS    RESTARTS   AGE
fluent-bit-d7hr2         1/1     Running   0          38s


Check fluent-bit logs for errors

Code Block
kubectl logs -f fluent-bit-d7hr2


Generate Some Logs

Using our test pod, we will generate some failed login attempts

Code Block
$ ssh root@localhost -p 30022
root@localhost's password: 
Permission denied, please try again.
root@localhost's password: 
Permission denied, please try again.
root@localhost's password: 
root@localhost: Permission denied (publickey,password).


View Logs in Azure Sentinel


Image Added


Example Queries

Code Block
fluentbit_CL


fluentbit_CL
| where kubernetes_labels_app_s contains "ubuntu"

fluentbit_CL
| where kubernetes_labels_app_s contains "ubuntu"
| where log_s contains "Failed password"
Code Block
# Default values for fluent-bit.

# kind -- DaemonSet or Deployment
kind: DaemonSet

# replicaCount -- Only applicable if kind=Deployment
replicaCount: 1

image:
  repository: fluent/fluent-bit
  pullPolicy: Always
  # tag:

testFramework:
  image:
    repository: busybox
    pullPolicy: Always
    tag: latest

imagePullSecrets: []
nameOverride: ""
fullnameOverride: ""

serviceAccount:
  create: true
  annotations: {}
  name:

rbac:
  create: true

podSecurityPolicy:
  create: false
  annotations: {}

podSecurityContext:
  {}
  # fsGroup: 2000
dnsConfig: {}
  # nameservers:
  #   - 1.2.3.4
  # searches:
  #   - ns1.svc.cluster-domain.example
  #   - my.dns.search.suffix
  # options:
  #   - name: ndots
#     value: "2"
#   - name: edns0

hostAliases: []
  # - ip: "1.2.3.4"
  #   hostnames:
  #   - "foo.local"
  #   - "bar.local"

securityContext:
  {}
  # capabilities:
  #   drop:
  #   - ALL
  # readOnlyRootFilesystem: true
  # runAsNonRoot: true
  # runAsUser: 1000

service:
  type: ClusterIP
  port: 2020
  labels:
    {}
  annotations:
    {}
    # prometheus.io/path: "/api/v1/metrics/prometheus"
    # prometheus.io/port: "2020"
    # prometheus.io/scrape: "true"

serviceMonitor:
  enabled: false
  # namespace: monitoring
  # interval: 10s
  # scrapeTimeout: 10s
  # selector:
  #  prometheus: my-prometheus

prometheusRule:
  enabled: false
  # namespace: ""
  # additionnalLabels: {}
  # rules:
  # - alert: NoOutputBytesProcessed
  #   expr: rate(fluentbit_output_proc_bytes_total[5m]) == 0
  #   annotations:
  #     message: |
  #       Fluent Bit instance {{ $labels.instance }}'s output plugin {{ $labels.name }} has not processed any
  #       bytes for at least 15 minutes.
  #     summary: No Output Bytes Processed
  #   for: 15m
  #   labels:
  #     severity: critical

dashboards:
  enabled: false
  labelKey: grafana_dashboard
  annotations: {}


livenessProbe: {}
  # httpGet:
  #   path: /
  #   port: http

readinessProbe:
  # httpGet:
  #   path: /
  #   port: http

resources:
  {}
  # limits:
  #   cpu: 100m
  #   memory: 128Mi
  # requests:
  #   cpu: 100m
  #   memory: 128Mi

nodeSelector: {}

tolerations: []

affinity: {}

podAnnotations: {}

podLabels: {}

priorityClassName: ""

env: []

envFrom: []

extraContainers: []
  # - name: do-something
  #   image: busybox
  #   command: ['do', 'something']

extraPorts: []
#   - port: 5170
#     containerPort: 5170
#     protocol: TCP
#     name: tcp

extraVolumes: []

extraVolumeMounts: []

updateStrategy: {}
  # type: RollingUpdate
  # rollingUpdate:
  #   maxUnavailable: 1

# Make use of a pre-defined configmap instead of the one templated here
existingConfigMap: ""

networkPolicy:
  enabled: false
  # ingress:
  #   from: []

luaScripts: {}

## https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/configuration-file
config:
  service: |
    [SERVICE]
        Flush 1
        Daemon Off
        Log_Level info
        Parsers_File parsers.conf
        Parsers_File custom_parsers.conf
        HTTP_Server On
        HTTP_Listen 0.0.0.0
        HTTP_Port {{ .Values.service.port }}

  ## https://docs.fluentbit.io/manual/pipeline/inputs
  inputs: |
    [INPUT]
        Name tail
        Path /var/log/containers/*.log
        Parser docker
        Tag kube.*
        Mem_Buf_Limit 5MB
        Skip_Long_Lines On

    [INPUT]
        Name systemd
        Tag host.*
        Systemd_Filter _SYSTEMD_UNIT=kubelet.service
        Read_From_Tail On

  ## https://docs.fluentbit.io/manual/pipeline/filters
  filters: |
    [FILTER]
        Name kubernetes
        Match kube.*
        Merge_Log On
        Keep_Log Off
        K8S-Logging.Parser On
        K8S-Logging.Exclude On

  ## https://docs.fluentbit.io/manual/pipeline/outputs
  outputs: |
    [OUTPUT]
        Name es
        Match kube.*
        Host elasticsearch-master
        Logstash_Format On
        Retry_Limit False

    [OUTPUT]
        Name es
        Match host.*
        Host elasticsearch-master
        Logstash_Format On
        Logstash_Prefix node
        Retry_Limit False

  ## https://docs.fluentbit.io/manual/pipeline/parsers
  customParsers: |
    [PARSER]
        Name docker_no_time
        Format json
        Time_Keep Off
        Time_Key time
        Time_Format %Y-%m-%dT%H:%M:%S.%L

# The config volume is mounted by default, either to the existingConfigMap value, or the default of "fluent-bit.fullname"
volumeMounts:
  - name: config
    mountPath: /fluent-bit/etc/fluent-bit.conf
    subPath: fluent-bit.conf
  - name: config
    mountPath: /fluent-bit/etc/custom_parsers.conf
    subPath: custom_parsers.conf

daemonSetVolumes:
  - name: varlog
    hostPath:
      path: /var/log
  - name: varlibdockercontainers
    hostPath:
      path: /var/lib/docker/containers
  - name: etcmachineid
    hostPath:
      path: /etc/machine-id
      type: File

daemonSetVolumeMounts:
  - name: varlog
    mountPath: /var/log
  - name: varlibdockercontainers
    mountPath: /var/lib/docker/containers
    readOnly: true
  - name: etcmachineid
    mountPath: /etc/machine-id
    readOnly: true

args: []

command: []

initContainers: []
  # - name: do-something
  #   image: busybox
  #   command: ['do', 'something']



References

...