FINDOLOGIC GmbH

Boring Logfiles → Sexy Dashboards

Boring Logfiles

178.236.7.219 - - [27/Oct/2014:06:43:13 +0100] "GET /ps/amazon.de/index.php?shopkey=DEADBEEF&shopurl=https%3A%2F%2Fwww.amazon.de%2F&multishop_id=0&userip=173.194.112.2&referer=https%3A%2F%2Fwww.amazon.de%2FTechnik%2FFernseher&count=0&group[]=2&service=search&query=led HTTP/1.1" 200 12363 "-" "-"

Sexy Dashboards

Stack

● Elasticsearch

● Logstash

● Kibana

Infrastruktur

App 1 App 2 App 3

Logstash 1 Logstash 2 Logstash 3

ElasticSearch

Kibana

Logstash Architektur

Input Filter Output

● Grok● Search / Replace● Split● GeoIP● DNS● URL-Decode● ...

● ElasticSearch● Redis● CouchDB● ...

● Pipe● Socket● File● ...

Logstash stand-alone

● Logs zentralisieren

● In DB speichern

● Durchsuchbar machen

Setup

● Logstash herunterladen, extrahieren

● Elasticsearch installieren

● fertig

Georg?

● CTO bei FINDOLOGIC GmbH

● http://www.findologic.com

● g.sorst@findologic.com

● @piefke_schorsch

Hands on

● Grok

● kv

● Query

● Endpoint

● GeoIP

Logstash.confinput {

file {

path => "/home/georg/coding/logstash-apache/access.log"

start_position => beginning

}

#pipe {

# command => "ssh frontend1.findologic.com tail -f /var/log/apache2/access.log"

#}

}

filter {

grok {

match => { "message" => "%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent}" }

}

# split the query string: query=test&count=0&first=10 → request_query: test, request_count: 0, request_first: 10

kv {

field_split => "&?"

source => "request"

prefix => "request_"

}

# urldecode does not decode the + into whitespace so do this manually

mutate {

gsub => [

"request_query", "\+", " "

]

}

# URL-decode the query so it is easier to read. We don't really about the other request_* parameters

urldecode {

field => "request_query"

}

# extract the script file, eg. index.php

mutate {

gsub => [

# Remove the query string including the ? : /path/index.php?key=value -> /path/index.php

"request", "\?.*$", "",

# Remove everything up to the last / : /path/index.php -> index.php

"request", "^.*/", ""

]

}

# Default to index.php if no script file is given

if [request] == "" {

mutate {

replace => [ "request", "index.php" ]

}

}

# If the IP is syntactically valid resolve it geographically

if [request_userip] =~ "^\d+\.\d+\.\d+\.\d+$" {

geoip {

source => "request_userip"

target => "client_geoip"

}

}

}

output {

elasticsearch {

host => localhost

}

#stdout { codec => rubydebug }

}

Kibana Dashboard

{

"title": "Logstash Search",

"services": {

"query": {

"list": {

"0": {

"query": "*",

"alias": "",

"color": "#7EB26D",

"id": 0,

"pin": false,

"type": "lucene",

"enable": true

},

"1": {

"id": 1,

"color": "#EAB839",

"alias": "Suche",

"pin": false,

"type": "lucene",

"enable": true,

"query": "request:index.php"

},

"2": {

"id": 2,

"color": "#6ED0E0",

"alias": "Autocomplete",

"pin": false,

"type": "lucene",

"enable": true,

"query": "request:autocomplete.php"

},

"3": {

"id": 3,

"color": "#EF843C",

"alias": "",

"pin": false,

"type": "lucene",

"enable": true,

"query": "client_geoip.country_code2:DE OR client_geoip.country_code2:AT OR client_geoip.country_code2:CH"

}

},

"ids": [

0,

1,

2,

3

]

},

"filter": {

"list": {

"0": {

"type": "time",

"field": "@timestamp",

"from": "now-24h",

"to": "now",

"mandate": "must",

"active": true,

"alias": "",

"id": 0

},

"1": {

"type": "time",

"from": "2014-10-27T18:04:33.357Z",

"to": "2014-10-27T19:18:47.459Z",

"field": "@timestamp",

"mandate": "must",

"active": true,

"alias": "",

"id": 1

}

},

"ids": [

0,

1

]

}

},

"rows": [

{

"title": "Graph",

"height": "350px",

"editable": true,

"collapse": false,

"collapsable": true,

"panels": [

{

"span": 12,

"editable": true,

"group": [

"default"

],

"type": "histogram",

"mode": "count",

"time_field": "@timestamp",

"value_field": null,

"auto_int": true,

"resolution": 100,

"interval": "30s",

"fill": 3,

"linewidth": 3,

"timezone": "browser",

"spyable": true,

"zoomlinks": true,

"bars": true,

"stack": true,

"points": false,

"lines": false,

"legend": true,

"x-axis": true,

"y-axis": true,

"percentage": false,

"interactive": true,

"queries": {

"mode": "selected",

"ids": [

1,

2

]

},

"title": "Events over time",

"intervals": [

"auto",

"1s",

"1m",

"5m",

"10m",

"30m",

"1h",

"3h",

"12h",

"1d",

"1w",

"1M",

"1y"

],

"options": true,

"tooltip": {

"value_type": "cumulative",

"query_as_alias": true

},

"scale": 1,

"y_format": "none",

"grid": {

"max": null,

"min": 0

},

"annotate": {

"enable": false,

"query": "*",

"size": 20,

"field": "_type",

"sort": [

"_score",

"desc"

]

},

"pointradius": 5,

"show_query": true,

"legend_counts": true,

"zerofill": true,

"derivative": false

}

],

"notice": false

},

{

"title": "",

"height": "350px",

"editable": true,

"collapse": false,

"collapsable": true,

"panels": [

{

"error": false,

"span": 6,

"editable": true,

"type": "terms",

"loadingEditor": false,

"field": "request_query",

"exclude": [],

"missing": false,

"other": false,

"size": 10,

"order": "count",

"style": {

"font-size": "10pt"

},

"donut": false,

"tilt": false,

"labels": true,

"arrangement": "horizontal",

"chart": "pie",

"counter_pos": "above",

"spyable": true,

"queries": {

"mode": "all",

"ids": [

0,

1,

2,

3

]

},

"tmode": "terms",

"tstat": "total",

"valuefield": ""

},

{

"error": false,

"span": 6,

"editable": true,

"type": "bettermap",

"loadingEditor": false,

"field": "client_geoip.location",

"size": 1000,

"spyable": true,

"tooltip": "_id",

"queries": {

"mode": "selected",

"ids": [

3

]

}

}

],

"notice": false

},

{

"title": "Events",

"height": "350px",

"editable": true,

"collapse": false,

"collapsable": true,

"panels": [

{

"title": "All events",

"error": false,

"span": 12,

"editable": true,

"group": [

"default"

],

"type": "table",

"size": 100,

"pages": 5,

"offset": 0,

"sort": [

"@timestamp",

"desc"

],

"style": {

"font-size": "9pt"

},

"overflow": "min-height",

"fields": [],

"localTime": true,

"timeField": "@timestamp",

"highlight": [],

"sortable": true,

"header": true,

"paging": true,

"spyable": true,

"queries": {

"mode": "all",

"ids": [

0,

1,

2,

3

]

},

"field_list": true,

"status": "Stable",

"trimFactor": 300,

"normTimes": true,

"all_fields": false

}

],

"notice": false

}

],

"editable": true,

"failover": false,

"index": {

"interval": "day",

"pattern": "[logstash-]YYYY.MM.DD",

"default": "NO_TIME_FILTER_OR_INDEX_PATTERN_NOT_MATCHED",

"warm_fields": true

},

"style": "light",

"panel_hints": true,

"pulldowns": [

{

"type": "query",

"collapse": false,

"notice": false,

"query": "*",

"pinned": true,

"history": [

"client_geoip.country_code2:DE OR client_geoip.country_code2:AT OR client_geoip.country_code2:CH",

"request:autocomplete.php",

"request:index.php",

"*"

],

"remember": 10,

"enable": true

},

{

"type": "filtering",

"collapse": false,

"notice": true,

"enable": true

}

],

"nav": [

{

"type": "timepicker",

"collapse": false,

"notice": false,

"status": "Stable",

"time_options": [

"5m",

"15m",

"1h",

"6h",

"12h",

"24h",

"2d",

"7d",

"30d"

],

"refresh_intervals": [

"5s",

"10s",

"30s",

"1m",

"5m",

"15m",

"30m",

"1h",

"2h",

"1d"

],

"timefield": "@timestamp",

"now": false,

"filter_id": 0,

"enable": true

}

],

"loader": {

"save_gist": false,

"save_elasticsearch": true,

"save_local": true,

"save_default": true,

"save_temp": true,

"save_temp_ttl_enable": true,

"save_temp_ttl": "30d",

"load_gist": true,

"load_elasticsearch": true,

"load_elasticsearch_size": 20,

"load_local": true,

"hide": false

},

"refresh": false

}

Vergrößern zum Anzeigen