1name: crowdsecurity/appsec-bot-challenge-exclude-ai-crawlers
2
3
4inband:
5 pre_eval:
6 - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/gptbot.json")
7 apply:
8 - ExemptFromChallenge("gptbot")
9 - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/openai-searchbot.json")
10 apply:
11 - ExemptFromChallenge("openai-searchbot")
12 - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/openai-chatgpt-user.json")
13 apply:
14 - ExemptFromChallenge("openai-chatgpt-user")
15 - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/perplexitybot.json")
16 apply:
17 - ExemptFromChallenge("perplexitybot")
18 - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/anthropic.json")
19 apply:
20 - ExemptFromChallenge("anthropic")
21 - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/commoncrawl.json")
22 apply:
23 - ExemptFromChallenge("commoncrawl")
24 - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/mistralai-index.json")
25 apply:
26 - ExemptFromChallenge("mistralai-index")
27 - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/mistralai-user.json")
28 apply:
29 - ExemptFromChallenge("mistralai-user")
30 - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/openai-adsbot.json")
31 apply:
32 - ExemptFromChallenge("openai-adsbot")
33 - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/perplexity-user.json")
34 apply:
35 - ExemptFromChallenge("perplexity-user")
36 - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/duckassistbot.json")
37 apply:
38 - ExemptFromChallenge("duckassistbot")
39 - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/youbot.json")
40 apply:
41 - ExemptFromChallenge("youbot")
42
43
44data:
45 - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/gptbot.json
46 dest_file: legit_bots/gptbot.json
47 type: bots
48 - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/openai-searchbot.json
49 dest_file: legit_bots/openai-searchbot.json
50 type: bots
51 - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/openai-chatgpt-user.json
52 dest_file: legit_bots/openai-chatgpt-user.json
53 type: bots
54 - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/perplexitybot.json
55 dest_file: legit_bots/perplexitybot.json
56 type: bots
57 - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/anthropic.json
58 dest_file: legit_bots/anthropic.json
59 type: bots
60 - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/commoncrawl.json
61 dest_file: legit_bots/commoncrawl.json
62 type: bots
63 - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/mistralai-index.json
64 dest_file: legit_bots/mistralai-index.json
65 type: bots
66 - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/mistralai-user.json
67 dest_file: legit_bots/mistralai-user.json
68 type: bots
69 - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/openai-adsbot.json
70 dest_file: legit_bots/openai-adsbot.json
71 type: bots
72 - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/perplexity-user.json
73 dest_file: legit_bots/perplexity-user.json
74 type: bots
75 - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/duckassistbot.json
76 dest_file: legit_bots/duckassistbot.json
77 type: bots
78 - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/youbot.json
79 dest_file: legit_bots/youbot.json
80 type: bots
81
