cscli parsers install crowdsecurity/nginx-logsA generic parser for nginx, support both access and error logs. This parser support also ingress nginx controller default log_format
*note : * If you are aggregating logs from several domains, prefix your logline with the target FQDN. HTTP based scenarios should take this into account so that buckets are per source IP per target FQDN, limiting false positives due to logs multiplexing.
1filter: "evt.Parsed.program startsWith 'nginx'"2onsuccess: next_stage3name: crowdsecurity/nginx-logs4description: "Parse nginx access and error logs"5pattern_syntax:6 NGCUSTOMUSER: '[a-zA-Z0-9\.\@\-\+_%]+'7 NGCUSTOMURIPATH: "(?:/[A-Za-z0-9$.+!*'\\(\\)\\{\\},~:;=@\\#%&_\\-]*)+"8 NGCUSTOMURIPATHPARAM: '%{NGCUSTOMURIPATH}(?:%{URIPARAM})?'9 NPMPLUSFQDN: '%{IPORHOST}||%{DATA}'10 NGCACHESTATUS: 'HIT|MISS|BYPASS|EXPIRED|STALE|UPDATING|REVALIDATED|-'11 NUM_OR_DASH: '-|\d*'12nodes:13 - grok:14 pattern: '(%{IPORHOST:target_fqdn}(:%{INT:port})? )?%{IPORHOST:remote_addr} - (%{NGCUSTOMUSER:remote_user} )?\[%{HTTPDATE:time_local}\] "%{WORD:verb} %{DATA:request} HTTP/%{NUMBER:http_version}" %{NUMBER:status} %{NUMBER:body_bytes_sent} "%{NOTDQUOTE:http_referer}" "%{NOTDQUOTE:http_user_agent}"( %{NUMBER:request_length} %{NUMBER:request_time} \[%{DATA:proxy_upstream_name}\] \[%{DATA:proxy_alternative_upstream_name}\])?'15 apply_on: message16 statics:17 - meta: log_type18 value: http_access-log19 - target: evt.StrTime20 expression: evt.Parsed.time_local21 - grok:22 # and this one the error log23 pattern: '(%{IPORHOST:target_fqdn} )?%{NGINXERRTIME:time} \[%{LOGLEVEL:loglevel}\] %{NONNEGINT:pid}#%{NONNEGINT:tid}: (\*%{NONNEGINT:cid} )?%{GREEDYDATA:message}, client: %{IPORHOST:remote_addr}, server: %{DATA:target_fqdn}, request: "%{WORD:verb} ([^/]+)?%{NGCUSTOMURIPATHPARAM:request}( HTTP/%{NUMBER:http_version})?", host: "%{IPORHOST}(:%{NONNEGINT})?"'24 apply_on: message25 statics:26 - meta: log_type27 value: http_error-log28 - target: evt.StrTime29 expression: evt.Parsed.time30 pattern_syntax:31 NO_DOUBLE_QUOTE: '[^"]+'32 onsuccess: next_stage33 nodes:34 - filter: "evt.Parsed.message contains 'was not found in'"35 pattern_syntax:36 USER_NOT_FOUND: 'user "%{NO_DOUBLE_QUOTE:username}" was not found in "%{NO_DOUBLE_QUOTE}"'37 grok:38 pattern: '%{USER_NOT_FOUND}'39 apply_on: message40 statics:41 - meta: sub_type42 value: "auth_fail"43 - meta: username44 expression: evt.Parsed.username45 - filter: "evt.Parsed.message contains 'password mismatch'"46 pattern_syntax:47 PASSWORD_MISMATCH: 'user "%{NO_DOUBLE_QUOTE:username}": password mismatch'48 grok:49 pattern: '%{PASSWORD_MISMATCH}'50 apply_on: message51 statics:52 - meta: sub_type53 value: "auth_fail"54 - meta: username55 expression: evt.Parsed.username56 - filter: "evt.Parsed.message contains 'limiting requests, excess'"57 statics:58 - meta: sub_type59 value: "req_limit_exceeded"60 ## Parse malformed requests61 - grok:62 pattern: '(%{IPORHOST:target_fqdn} )?%{IPORHOST:remote_addr} - (%{NGUSER:remote_user})? \[%{HTTPDATE:time_local}\] "%{DATA:request}" %{NUMBER:status} %{NUMBER:body_bytes_sent} "%{NOTDQUOTE:http_referer}" "%{NOTDQUOTE:http_user_agent}"( %{NUMBER:request_length} %{NUMBER:request_time} \[%{DATA:proxy_upstream_name}\] \[%{DATA:proxy_alternative_upstream_name}\])?'63 apply_on: message64 statics:65 - meta: log_type66 value: http_access-log67 - target: evt.StrTime68 expression: evt.Parsed.time_local69 ## Catch NPMplus access logs that might be sent to nginx parser70 - grok:71 pattern: '\[%{HTTPDATE:time_local}\] %{NPMPLUSFQDN:target_fqdn} %{IPORHOST:remote_addr} %{NUMBER:request_time} "%{WORD:verb} %{DATA:request} HTTP/%{NUMBER:http_version}" %{NUMBER:status} %{NUMBER:body_bytes_sent} %{NUMBER:bytes_sent} %{DATA:http_referer} %{GREEDYDATA:http_user_agent}'72 apply_on: message73 statics:74 - meta: log_type75 value: http_access-log76 - target: evt.StrTime77 expression: evt.Parsed.time_local78 ## Catch NPMplus malformed access logs79 - grok:80 pattern: '\[%{HTTPDATE:time_local}\] %{NPMPLUSFQDN:target_fqdn} %{IPORHOST:remote_addr} %{NUMBER:request_time} "%{DATA:request}" %{NUMBER:status} %{NUMBER:body_bytes_sent} %{NUMBER:bytes_sent} %{DATA:http_referer} %{GREEDYDATA:http_user_agent}'81 apply_on: message82 statics:83 - meta: log_type84 value: http_access-log85 - target: evt.StrTime86 expression: evt.Parsed.time_local87 ## Catch nginx-proxy-manager logs that might be tagged as npm88 - grok:89 pattern: '\[%{HTTPDATE:time_local}\]( %{NGCACHESTATUS:upstream_cache_status} %{NUM_OR_DASH:upstream_status})? %{NUMBER:status} - %{WORD:verb} %{WORD:scheme} %{IPORHOST:target_fqdn} \"%{NOTDQUOTE:request}\" \[Client %{IPORHOST:remote_addr}\] \[Length %{NUMBER:body_bytes_sent}\] \[Gzip %{DATA:gzip_ratio}\]( \[Sent-to %{DATA:target_server}\])? \"%{NOTDQUOTE:http_user_agent}\" \"%{NOTDQUOTE:http_referer}\"'90 apply_on: message91 statics:92 - meta: log_type93 value: http_access-log94 - target: evt.StrTime95 expression: evt.Parsed.time_local96 # these ones apply for both grok patterns97statics:98 - meta: service99 value: http100 - meta: source_ip101 expression: "evt.Parsed.remote_addr"102 - meta: http_status103 expression: "evt.Parsed.status"104 - meta: http_path105 expression: "evt.Parsed.request"106 - meta: http_verb107 expression: "evt.Parsed.verb"108 - meta: http_user_agent109 expression: "evt.Parsed.http_user_agent"110 - meta: target_fqdn111 expression: "evt.Parsed.target_fqdn"112