logstash apache COMMONAPACHELOG

삽질 1. timezone 부분

logstash, elasticsearch(plugis등), kibana 골치 아프니 UTC로 모두 바꾸니 편함...ㅡㅡ;

[logstash]

dd/MMM/yyyy:HH:mm:ss Z 했었는데 처리가 안되서.. 아래와 같이 했더니 해결...

         -> https://discuss.elastic.co/t/how-to-set-timestamp-timezone/28401/10

[kibana]

Settings -> Advanced -> dateFormat:tz -> UTC 선택

[logconf/apache.conf]

input {
  stdin { }
}

filter {
  grok {
    patterns_dir => ["./patterns"]
    # 본인 로그 패턴에 따라 설정
    match => { "message" => "%{COMMONAPACHELOG} %{NUMBER:responseTime} %{ALL_STR:qq1} %{ALL_STR:qq2} %{QS:referrer} %{QS:agent}" }
  }
  date {
    match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss +0900" ]
    timezone => "UTC"
  }
}

output {
  stdout { codec => json }
}

[logconf/apache.conf]

# 파일 단위 -> elasticsearch
input {
  file {
    path => "/usr/local/tomcat/logstash/dump_logs/ssl/*"
    start_position => "beginning"
  }
}

filter {
  grok {
    patterns_dir => ["./patterns"]
    # 본인 로그 패턴에 따라 설정
    match => { "message" => "%{COMMONAPACHELOG} %{ALL_STR:qq1} %{ALL_STR:qq2} %{QS:referrer} %{QS:agent}" }
  }
  date {
    match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss +0900" ]
    timezone => "UTC"
  }
}

output {
  elasticsearch { hosts => ["localhost:9200"]
                  index => "apache-access-log-%{+YYYY-MM-dd}"
                  document_type => "web01"
                }
  #stdout { codec => rubydebug }
}

[haproxy]

input {
  #stdin { }
  file {
    type => "searchWas"
    path => "/usr/local/tomcat/logstash/dump_logs/haproxy/haproxy-searchWas.log*"
    start_position => "beginning"
  }
  file {
    type => "aqmp"
    path => "/usr/local/tomcat/logstash/dump_logs/haproxy/haproxy-aqmp.log*"
    start_position => "beginning"
  }
}

  
filter {
  if [type] == "searchWas" {
    grok {
      patterns_dir => ["./patterns"]
    # 본인 로그 패턴에 따라 설정
      match => { "message" => "%{HAPROXYHTTP}" }
    }
    date {
      match => [ "accept_date", "dd/MMM/YYYY:HH:mm:ss.SSS" ]
      timezone => "UTC"
    }
  } else if [type] == "aqmp" {
    grok {
      patterns_dir => ["./patterns"]
      match => { "message" => "%{HAPROXYTCP}" }
    }
    date {
      match => [ "accept_date", "dd/MMM/YYYY:HH:mm:ss.SSS" ]
      timezone => "UTC"
    }
  }
}

output {
  #stdout { codec => json }
  
  if [type] == "searchWas" {
    elasticsearch {  
      hosts => ["localhost:9200"]
      index => "haproxy-access-log-%{+YYYY-MM-dd}"
      document_type => "searchWas"
    }
  } else if [type] == "aqmp" {
    elasticsearch {  
      hosts => ["localhost:9200"]
      index => "haproxy-access-log-%{+YYYY-MM-dd}"
      document_type => "aqmp"
    }
  }
}

[logconf/test-db.conf] db -> elasticsearch 케이스

input {
        jdbc {
                jdbc_driver_library => "/usr/local/tomcat/logstash/lib/mysql-connector-java-5.1.38.jar"
                jdbc_driver_class => "com.mysql.jdbc.Driver"
                jdbc_connection_string => "jdbc:mysql://mariadb-dev:3306/log"
                jdbc_user => "log"
                jdbc_password => "test"
                statement => "SELECT * FROM ez_apache_access_log_201602"
                #schedule => "* * * * *"
                jdbc_paging_enabled => "true"
                jdbc_page_size => "50000"

        }
}

[patterns/custom.conf]

ALL_STR .*

test exec

$ bin/logstash -f logconf/apache.conf 
192.168.1.100 - - [17/Jul/2016:21:08:51 +0900] "GET /test/list HTTP/1.1" 200 17194 5657 -/- (-%) "http://ddakker.test.com/test/add?test=1" "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0)"

mappings -> 수동 index/document mapping 생성 예제

$ curl -XPUT http://localhost:9200/log_httpd -d '
{
  "mappings": {
    "web01": {
      "properties": {
        "req_query_string": {
          "type": "string",
          "fields": {
            "raw": {
              "type": "string",
              "index": "not_analyzed"
            }
          }
        },
        "req_dt": {
          "type": "string"
        },
        "referrer_query_string": {
          "type": "string",
          "fields": {
            "raw": {
              "type": "string",
              "index": "not_analyzed"
            }
          }
        },
        "referrer_domain": {
          "type": "string",
          "fields": {
            "raw": {
              "type": "string",
              "index": "not_analyzed"
            }
          }
        },
        "req_ext": {
          "type": "string"
        },
        "referrer_ext": {
          "type": "string"
        },
        "res_status": {
          "type": "string"
        },
        "req_mall_type": {
          "type": "string"
        },
        "@version": {
          "type": "string"
        },
        "ip": {
          "type": "string"
        },
        "req_uri": {
          "type": "string",
          "fields": {
            "raw": {
              "type": "string",
              "index": "not_analyzed"
            }
          }
        },
        "@timestamp": {
          "format": "strict_date_optional_time||epoch_millis",
          "type": "date"
        },
        "referrer_mall_type": {
          "type": "string",
          "fields": {
            "raw": {
              "type": "string",
              "index": "not_analyzed"
            }
          }
        },
        "method": {
          "type": "string"
        },
        "referrer_uri": {
          "type": "string",
          "fields": {
            "raw": {
              "type": "string",
              "index": "not_analyzed"
            }
          }
        }
      }
    }
  }
}'