k8s服务配置

apiVersion: v1
kind: ConfigMap
metadata:
  name: k8s-logs-filebeat-config
data:
  filebeat.yml: |-
    filebeat.inputs:
    - type: container
      enabled: true
      symlinks: true
      paths:
        - /var/log/containers/java*.log
    multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}\s[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3}'
    multiline.negate: true
    multiline.match: after
    multiline.timeout: 10s
    processors:
      - drop_fields:
          fields: ["host", "ecs", "log", "agent", "input"]
          ignore_missing: false
    output.logstash:
      hosts: ["172.26.239.38:5000"]
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: k8s-logs
spec:
  selector:
    matchLabels:
      project: k8s
      app: filebeat
  template:
    metadata:
      labels:
        project: k8s
        app: filebeat
    spec:
      containers:
      - name: filebeat
        imagePullPolicy: IfNotPresent
        image: elastic/filebeat:7.4.1
        args: [
          "-c", "/etc/filebeat.yml",
          "-e",
        ]
        securityContext:
          runAsUser: 0
        volumeMounts:
        - name: filebeat-config
          mountPath: /etc/filebeat.yml
          subPath: filebeat.yml
        - name: k8s-docker
          mountPath:  /var/lib/docker/containers
          readOnly: true
        - name: k8s-pods
          mountPath: /var/log/pods
          readOnly: true
        - name: k8s-logs
          mountPath: /var/log/containers
          readOnly: true
      volumes:
      - name: k8s-docker
        hostPath:
          path:  /var/lib/docker/containers
      - name: k8s-pods
        hostPath:
          path: /var/log/pods
      - name: k8s-logs
        hostPath:
          path: /var/log/containers
      - name: filebeat-config
        configMap:
          name: k8s-logs-filebeat-config

logstsh配置

input {
        beats {
                port => 5000
        }

}


filter {

  if [type] == "log" {

     ruby {
         code => "event.set('timestamp', event.get('@timestamp').time.localtime + 8*60*60)"
     }
     ruby {
         code => "event.set('@timestamp',event.get('timestamp'))"
     }

    mutate{

     remove_field=>["@version"]
     remove_field=>["log"]
     remove_field=>["@version"]
     remove_field=>["input"]
     remove_field=>["fields"]
     remove_field=>["tags"]
     remove_field=>["host"]
     remove_field=>["agent"]
     remove_field=>["ecs"]
     remove_field=>["timestamp"]


    }

 }

}

## Add your filters / logstash plugins configuration here

output {

if [type] == "log"{

    if [json][http_activity_id]!="" {
        elasticsearch {
                hosts => "elasticsearch:9200"
                user => "elastic"
                password => ""
                index=>"visit_logs"   #设置索引名字


        }
    }

}else{


   elasticsearch {
                hosts => "elasticsearch:9200"
                user => "elastic"
                password => ""
                index=>"container_logs"   #设置索引名字


        }


}


}