Unverified Commit aab6fbbc authored by Tom Moulard's avatar Tom Moulard
Browse files

logstash: finally a good configuration file 

now tagging usefull log data using grok filter.
if you want to add a new nginx instance: copy the conf from nginx and
keep the type for both access and error.
parent 06e62b2c

logstash/logstash.conf

+84 −10
Original line number Diff line number Diff line 
# logstash.con

# Where you see:

# #        start_position => "beginning"

# You can un comment this line if the elasticsearch instance is new and you want

# to import all previous logs.



input {

    file {

        path => "/var/log/traefik/traefik.log"

        type => "traefik_traefik"

        type => "traefik_log"

#        start_position => "beginning"

    }



    file {

        path => "/var/log/traefik/access.log"

        type => "traefik_access"

#        start_position => "beginning"

    }

    file {

        path => "/var/log/nginx/access.log"

        type => "nginx_access"

#        start_position => "beginning"

    }



    file {

        path => "/var/log/nginx/error.log"

        type => "nginx_error"

#        start_position => "beginning"

    }

}



    file {

        path => "/var/log/nginx/access.log"

        type => "nginx_access"

filter {

    if [type] == "nginx_access" {

        grok {

            match => { "message" => ["%{IPORHOST:[nginx][access][remote_ip]} - %{DATA:[nginx][access][user_name]} \[%{HTTPDATE:[nginx][access][time]}\] \"%{WORD:[nginx][access][method]} %{DATA:[nginx][access][url]} HTTP/%{NUMBER:[nginx][access][http_version]}\" %{NUMBER:[nginx][access][response_code]} %{NUMBER:[nginx][access][body_sent][bytes]} \"%{DATA:[nginx][access][referrer]}\" \"%{DATA:[nginx][access][agent]}\""] }

            remove_field => "message"

        }

        mutate {

            add_field => { "read_timestamp" => "%{@timestamp}" }

        }

        date {

            match => [ "[nginx][access][time]", "dd/MMM/YYYY:H:m:s Z" ]

            remove_field => "[nginx][access][time]"

        }

        useragent {

            source => "[nginx][access][agent]"

            target => "[nginx][access][user_agent]"

            remove_field => "[nginx][access][agent]"

        }

#         This is not needed because traefik hides the real ip

#         geoip {

#             source => "[nginx][access][remote_ip]"

#             target => "[nginx][access][geoip]"

#         }

    }

    if [type] == "nginx_error" {

        grok {

            match => { "message" => ["%{DATA:[nginx][error][time]} \[%{DATA:[nginx][error][level]}\] %{NUMBER:[nginx][error][pid]}#%{NUMBER:[nginx][error][tid]}: (\*%{NUMBER:[nginx][error][connection_id]} )?%{GREEDYDATA:[nginx][error][message]}"] }

            remove_field => "message"

        }

        mutate {

            rename => { "@timestamp" => "read_timestamp" }

        }

        date {

            match => [ "[nginx][error][time]", "YYYY/MM/dd H:m:s" ]

            remove_field => "[nginx][error][time]"

        }

    }

    if [type] == "traefik_access" {

        json {

            source => "message"

        }

        date {

            match => [ "timestamp", "dd/MM/YYYY:KK:mm:ss Z"]

            target => "event_timestamp"

        }

        geoip {

            source => "ClientHost"

        }

        useragent {

            source => "request_User-Agent"

        }

    }

    if [type] == "traefik_log" {

        json {

            source => "message"

        }

        date {

            match => [ "timestamp", "dd/MM/YYYY:KK:mm:ss Z"]

            target => "event_timestamp"

        }

    }

    uuid {

        target => "uuid"

    }

}



output {

  elasticsearch {

    hosts    => [ 'elasticsearch' ]

    user     => 'elastic'

    password => 'changeme'

    hosts    => [ "elasticsearch" ]

    user     => "elastic"

    password => "changeme"

    index    => "logstash-%{+YYYY.MM.dd}"

  }

}