I am trying to parse checkpoint and fortinet logs through logstash and get log.syslog.structured_data field in kibana/logscale .Is there any way to remove it ? I know it represents structured data expressed in RFC 5424 messages but I want to exclude that field completely as it's a complete replica of @rawstring
my filter.conf is
grok {
match => {"message" => "%{SYSLOG5424PRI}%{GREEDYDATA:message}" }
overwrite => [ "message" ]
}
mutate {
remove_field => ["tags", "input", "log", "event", "@timestamp", "agent", "ecs", "@version", "host"]
}
kv {
field_split => " "
value_split => "="
}
mutate {
remove_field => ["message"]
rename => { "<(\d+)>date" => "date" }
add_field => { "logdate" => "%{date} %{time}"}
}
date {
match => ["logdate", "yyyy-MM-dd HH:mm:ss" ]
timezone => "Europe/London"
target => "@timestamp"
}
mutate {
remove_field => ["logdate", "date", "time"]
convert => { "rcvdbyte" => "integer"}
convert => { "sentbyte" => "integer"}
}
mutate{ remove_field => [ "[log][syslog][structured_data]"] }
}
}
I tried removing from filebeat.yml as well but it didn't work
- add_host_metadata:
when.not.contains.tags: forwarded
- add_cloud_metadata: ~
- add_docker_metadata: ~
- add_kubernetes_metadata: ~
- decode_json_fields:
fields: ["message"]
target: ""
overwrite_keys: true
- drop_fields:
fields: ["log.syslog.structured_data"]