1filter: "evt.Parsed.program startsWith 'nginx'"
2onsuccess: next_stage
3name: crowdsecurity/nginx-logs
4description: "Parse nginx access and error logs"
5pattern_syntax:
6  NGCUSTOMUSER: '[a-zA-Z0-9\.\@\-\+_%]+'
7  NGCUSTOMURIPATH: "(?:/[A-Za-z0-9$.+!*'\\(\\)\\{\\},~:;=@\\#%&_\\-]*)+"
8  NGCUSTOMURIPATHPARAM: '%{NGCUSTOMURIPATH}(?:%{URIPARAM})?'
9  NPMPLUSFQDN: '%{IPORHOST}||%{DATA}'
10  NGCACHESTATUS: 'HIT|MISS|BYPASS|EXPIRED|STALE|UPDATING|REVALIDATED|-'
11  NUM_OR_DASH: '-|\d*'
12nodes:
13  - grok:
14      pattern: '(%{IPORHOST:target_fqdn}(:%{INT:port})? )?%{IPORHOST:remote_addr} - (%{NGCUSTOMUSER:remote_user} )?\[%{HTTPDATE:time_local}\] "%{WORD:verb} %{DATA:request} HTTP/%{NUMBER:http_version}" %{NUMBER:status} %{NUMBER:body_bytes_sent} "%{NOTDQUOTE:http_referer}" "%{NOTDQUOTE:http_user_agent}"( %{NUMBER:request_length} %{NUMBER:request_time} \[%{DATA:proxy_upstream_name}\] \[%{DATA:proxy_alternative_upstream_name}\])?'
15      apply_on: message
16      statics:
17        - meta: log_type
18          value: http_access-log
19        - target: evt.StrTime
20          expression: evt.Parsed.time_local
21  - grok:
22      # and this one the error log
23      pattern: '(%{IPORHOST:target_fqdn} )?%{NGINXERRTIME:time} \[%{LOGLEVEL:loglevel}\] %{NONNEGINT:pid}#%{NONNEGINT:tid}: (\*%{NONNEGINT:cid} )?%{GREEDYDATA:message}, client: %{IPORHOST:remote_addr}, server: %{DATA:target_fqdn}, request: "%{WORD:verb} ([^/]+)?%{NGCUSTOMURIPATHPARAM:request}( HTTP/%{NUMBER:http_version})?", host: "%{IPORHOST}(:%{NONNEGINT})?"'
24      apply_on: message
25      statics:
26        - meta: log_type
27          value: http_error-log
28        - target: evt.StrTime
29          expression: evt.Parsed.time
30    pattern_syntax:
31      NO_DOUBLE_QUOTE: '[^"]+'
32    onsuccess: next_stage
33    nodes:
34      - filter: "evt.Parsed.message contains 'was not found in'"
35        pattern_syntax:
36          USER_NOT_FOUND: 'user "%{NO_DOUBLE_QUOTE:username}" was not found in "%{NO_DOUBLE_QUOTE}"'
37        grok:
38          pattern: '%{USER_NOT_FOUND}'
39          apply_on: message
40        statics:
41          - meta: sub_type
42            value: "auth_fail"
43          - meta: username
44            expression: evt.Parsed.username
45      - filter: "evt.Parsed.message contains 'password mismatch'"
46        pattern_syntax:
47          PASSWORD_MISMATCH: 'user "%{NO_DOUBLE_QUOTE:username}": password mismatch'
48        grok:
49          pattern: '%{PASSWORD_MISMATCH}'
50          apply_on: message
51        statics:
52          - meta: sub_type
53            value: "auth_fail"
54          - meta: username
55            expression: evt.Parsed.username
56      - filter: "evt.Parsed.message contains 'limiting requests, excess'"
57        statics:
58          - meta: sub_type
59            value: "req_limit_exceeded"
60  ## Parse malformed requests
61  - grok:
62      pattern: '(%{IPORHOST:target_fqdn} )?%{IPORHOST:remote_addr} - (%{NGUSER:remote_user})? \[%{HTTPDATE:time_local}\] "%{DATA:request}" %{NUMBER:status} %{NUMBER:body_bytes_sent} "%{NOTDQUOTE:http_referer}" "%{NOTDQUOTE:http_user_agent}"( %{NUMBER:request_length} %{NUMBER:request_time} \[%{DATA:proxy_upstream_name}\] \[%{DATA:proxy_alternative_upstream_name}\])?'
63      apply_on: message
64    statics:
65      - meta: log_type
66        value: http_access-log
67      - target: evt.StrTime
68        expression: evt.Parsed.time_local
69  ## Catch NPMplus access logs that might be sent to nginx parser
70  - grok:
71      pattern: '\[%{HTTPDATE:time_local}\] %{NPMPLUSFQDN:target_fqdn} %{IPORHOST:remote_addr} %{NUMBER:request_time} "%{WORD:verb} %{DATA:request} HTTP/%{NUMBER:http_version}" %{NUMBER:status} %{NUMBER:body_bytes_sent} %{NUMBER:bytes_sent} %{DATA:http_referer} %{GREEDYDATA:http_user_agent}'
72      apply_on: message
73    statics:
74      - meta: log_type
75        value: http_access-log
76      - target: evt.StrTime
77        expression: evt.Parsed.time_local
78  ## Catch NPMplus malformed access logs
79  - grok:
80      pattern: '\[%{HTTPDATE:time_local}\] %{NPMPLUSFQDN:target_fqdn} %{IPORHOST:remote_addr} %{NUMBER:request_time} "%{DATA:request}" %{NUMBER:status} %{NUMBER:body_bytes_sent} %{NUMBER:bytes_sent} %{DATA:http_referer} %{GREEDYDATA:http_user_agent}'
81      apply_on: message
82    statics:
83      - meta: log_type
84        value: http_access-log
85      - target: evt.StrTime
86        expression: evt.Parsed.time_local
87  ## Catch nginx-proxy-manager logs that might be tagged as npm
88  - grok:
89      pattern: '\[%{HTTPDATE:time_local}\]( %{NGCACHESTATUS:upstream_cache_status} %{NUM_OR_DASH:upstream_status})? %{NUMBER:status} - %{WORD:verb} %{WORD:scheme} %{IPORHOST:target_fqdn} \"%{NOTDQUOTE:request}\" \[Client %{IPORHOST:remote_addr}\] \[Length %{NUMBER:body_bytes_sent}\] \[Gzip %{DATA:gzip_ratio}\]( \[Sent-to %{DATA:target_server}\])? \"%{NOTDQUOTE:http_user_agent}\" \"%{NOTDQUOTE:http_referer}\"'
90      apply_on: message
91    statics:
92      - meta: log_type
93        value: http_access-log
94      - target: evt.StrTime
95        expression: evt.Parsed.time_local
96    # these ones apply for both grok patterns
97statics:
98  - meta: service
99    value: http
100  - meta: source_ip
101    expression: "evt.Parsed.remote_addr"
102  - meta: http_status
103    expression: "evt.Parsed.status"
104  - meta: http_path
105    expression: "evt.Parsed.request"
106  - meta: http_verb
107    expression: "evt.Parsed.verb"
108  - meta: http_user_agent
109    expression: "evt.Parsed.http_user_agent"
110  - meta: target_fqdn
111    expression: "evt.Parsed.target_fqdn"
112