nginx log_format

1
2
3
4
5
6

log_format  ops '$remote_addr - $remote_user [$time_local] "$request" '
                '$status $body_bytes_sent "$http_referer" '
                '"$http_user_agent" $http_x_forwarded_for $request_time '
                '$upstream_addr $upstream_response_time '
                '"$content_type" '
                '$host';

日志用例

123.150.174.174 - - [15/Jul/2020:18:44:31 +0800] "POST /tencent&contenttype=json HTTP/1.1" 200 50 "-" "-" - 0.005 10.203.151.216:10042 0.005 "application/json" www.baidu.com

logstash配置

input {
  kafka {
    bootstrap_servers => "kafka_ip:9092"
    topics => ["topic_name"]
    consumer_threads => 8
    type => "nginx_log"
  }
}

filter {
  if [type] == "nginx_log" {
    json {
      source => "message"
    }

    mutate {
      remove_field => [ "input", "host", "input.type", "offset", "prospector" ]
    }

    grok {
      match => [
        "message", "%{IPORHOST:client} (%{USER:ident}|-) (%{USER:auth}|-) \[%{HTTPDATE:accept_date}\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:http_version})?|-)\" %{NUMBER:response} %{NUMBER:bytes:int} (%{QS:referrer}|-) %{QS:agent} (%{QS:x_forword}|-) %{NUMBER:request_time:float} (%{NOTSPACE:upstream_addr}|-) %{NUMBER:upstream_response_time:float} \"(%{NOTSPACE:content-type}|-)\" (%{NOTSPACE:host}|-)"
      ]
    }
	
    date {
      match => ["accept_date","dd/MMM/yyy:HH:mm:ss"]
    }	
	
    ruby {
      code => "
        event.set('locate_index',event.get('@timestamp').time.localtime.strftime('%Y.%m.%d'))
      "
    }    
  }
}


output {
  if [type] == "nginx_log" {
    elasticsearch {
      hosts => ["es_ip"]
        index => "nginx_log-%{locate_index}"
        retry_initial_interval => 60
      }
  }
}