k8s服务配置
apiVersion: v1
kind: ConfigMap
metadata:
name: k8s-logs-filebeat-config
data:
filebeat.yml: |-
filebeat.inputs:
- type: container
enabled: true
symlinks: true
paths:
- /var/log/containers/java*.log
multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}\s[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3}'
multiline.negate: true
multiline.match: after
multiline.timeout: 10s
processors:
- drop_fields:
fields: ["host", "ecs", "log", "agent", "input"]
ignore_missing: false
output.logstash:
hosts: ["172.26.239.38:5000"]
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: k8s-logs
spec:
selector:
matchLabels:
project: k8s
app: filebeat
template:
metadata:
labels:
project: k8s
app: filebeat
spec:
containers:
- name: filebeat
imagePullPolicy: IfNotPresent
image: elastic/filebeat:7.4.1
args: [
"-c", "/etc/filebeat.yml",
"-e",
]
securityContext:
runAsUser: 0
volumeMounts:
- name: filebeat-config
mountPath: /etc/filebeat.yml
subPath: filebeat.yml
- name: k8s-docker
mountPath: /var/lib/docker/containers
readOnly: true
- name: k8s-pods
mountPath: /var/log/pods
readOnly: true
- name: k8s-logs
mountPath: /var/log/containers
readOnly: true
volumes:
- name: k8s-docker
hostPath:
path: /var/lib/docker/containers
- name: k8s-pods
hostPath:
path: /var/log/pods
- name: k8s-logs
hostPath:
path: /var/log/containers
- name: filebeat-config
configMap:
name: k8s-logs-filebeat-config
logstsh配置
input {
beats {
port => 5000
}
}
filter {
if [type] == "log" {
ruby {
code => "event.set('timestamp', event.get('@timestamp').time.localtime + 8*60*60)"
}
ruby {
code => "event.set('@timestamp',event.get('timestamp'))"
}
mutate{
remove_field=>["@version"]
remove_field=>["log"]
remove_field=>["@version"]
remove_field=>["input"]
remove_field=>["fields"]
remove_field=>["tags"]
remove_field=>["host"]
remove_field=>["agent"]
remove_field=>["ecs"]
remove_field=>["timestamp"]
}
}
}
## Add your filters / logstash plugins configuration here
output {
if [type] == "log"{
if [json][http_activity_id]!="" {
elasticsearch {
hosts => "elasticsearch:9200"
user => "elastic"
password => ""
index=>"visit_logs" #设置索引名字
}
}
}else{
elasticsearch {
hosts => "elasticsearch:9200"
user => "elastic"
password => ""
index=>"container_logs" #设置索引名字
}
}
}