1filter: "evt.Parsed.program startsWith 'nginx'"
2onsuccess: next_stage
3name: crowdsecurity/nginx-logs
4description: "Parse nginx access and error logs"
5pattern_syntax:
6  NGCUSTOMUSER: '[a-zA-Z0-9\.\@\-\+_%]+'
7  NGCUSTOMURIPATH: "(?:/[A-Za-z0-9$.+!*'\\(\\)\\{\\},~:;=@\\#%&_\\-]*)+"
8  NGCUSTOMURIPATHPARAM: '%{NGCUSTOMURIPATH}(?:%{URIPARAM})?'
9nodes:
10  - grok:
11      pattern: '(%{IPORHOST:target_fqdn}(:%{INT:port})? )?%{IPORHOST:remote_addr} - %{NGCUSTOMUSER:remote_user}? \[%{HTTPDATE:time_local}\] "%{WORD:verb} %{DATA:request} HTTP/%{NUMBER:http_version}" %{NUMBER:status} %{NUMBER:body_bytes_sent} "%{NOTDQUOTE:http_referer}" "%{NOTDQUOTE:http_user_agent}"( %{NUMBER:request_length} %{NUMBER:request_time} \[%{DATA:proxy_upstream_name}\] \[%{DATA:proxy_alternative_upstream_name}\])?'
12      apply_on: message
13      statics:
14        - meta: log_type
15          value: http_access-log
16        - target: evt.StrTime
17          expression: evt.Parsed.time_local
18  - grok:
19      # and this one the error log
20      pattern: '(%{IPORHOST:target_fqdn} )?%{NGINXERRTIME:time} \[%{LOGLEVEL:loglevel}\] %{NONNEGINT:pid}#%{NONNEGINT:tid}: (\*%{NONNEGINT:cid} )?%{GREEDYDATA:message}, client: %{IPORHOST:remote_addr}, server: %{DATA:target_fqdn}, request: "%{WORD:verb} ([^/]+)?%{NGCUSTOMURIPATHPARAM:request}( HTTP/%{NUMBER:http_version})?", host: "%{IPORHOST}(:%{NONNEGINT})?"'
21      apply_on: message
22      statics:
23        - meta: log_type
24          value: http_error-log
25        - target: evt.StrTime
26          expression: evt.Parsed.time
27    pattern_syntax:
28      NO_DOUBLE_QUOTE: '[^"]+'
29    onsuccess: next_stage
30    nodes:
31      - filter: "evt.Parsed.message contains 'was not found in'"
32        pattern_syntax:
33          USER_NOT_FOUND: 'user "%{NO_DOUBLE_QUOTE:username}" was not found in "%{NO_DOUBLE_QUOTE}"'
34        grok:
35          pattern: '%{USER_NOT_FOUND}'
36          apply_on: message
37        statics:
38          - meta: sub_type
39            value: "auth_fail"
40          - meta: username
41            expression: evt.Parsed.username
42      - filter: "evt.Parsed.message contains 'password mismatch'"
43        pattern_syntax:
44          PASSWORD_MISMATCH: 'user "%{NO_DOUBLE_QUOTE:username}": password mismatch'
45        grok:
46          pattern: '%{PASSWORD_MISMATCH}'
47          apply_on: message
48        statics:
49          - meta: sub_type
50            value: "auth_fail"
51          - meta: username
52            expression: evt.Parsed.username
53      - filter: "evt.Parsed.message contains 'limiting requests, excess'"
54        statics:
55          - meta: sub_type
56            value: "req_limit_exceeded"
57  ## Parse malformed requests
58  - grok:
59      pattern: '(%{IPORHOST:target_fqdn} )?%{IPORHOST:remote_addr} - (%{NGUSER:remote_user})? \[%{HTTPDATE:time_local}\] "%{DATA:request}" %{NUMBER:status} %{NUMBER:body_bytes_sent} "%{NOTDQUOTE:http_referer}" "%{NOTDQUOTE:http_user_agent}"( %{NUMBER:request_length} %{NUMBER:request_time} \[%{DATA:proxy_upstream_name}\] \[%{DATA:proxy_alternative_upstream_name}\])?'
60      apply_on: message
61    statics:
62      - meta: log_type
63        value: http_access-log
64      - target: evt.StrTime
65        expression: evt.Parsed.time_local
66    # these ones apply for both grok patterns
67statics:
68  - meta: service
69    value: http
70  - meta: source_ip
71    expression: "evt.Parsed.remote_addr"
72  - meta: http_status
73    expression: "evt.Parsed.status"
74  - meta: http_path
75    expression: "evt.Parsed.request"
76  - meta: http_verb
77    expression: "evt.Parsed.verb"
78  - meta: http_user_agent
79    expression: "evt.Parsed.http_user_agent"
80  - meta: target_fqdn
81    expression: "evt.Parsed.target_fqdn"
82