Hub WAF configuration

Name: appsec-bot-challenge-exclude-search-engines
Author: crowdsecurity

More info

1name: crowdsecurity/appsec-bot-challenge-exclude-search-engines

2# Skip the challenge for verified search-engine crawlers (matched against their published IP/rDNS ranges).

3# Do not edit this file to add your own exclusions, create a custom config.

4inband:

5 pre_eval:

6 - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/googlebot.json")

7 apply:

8 - ExemptFromChallenge("googlebot")

9 - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/bingbot.json")

10 apply:

11 - ExemptFromChallenge("bingbot")

12 - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/applebot.json")

13 apply:

14 - ExemptFromChallenge("applebot")

15 - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/amazonbot.json")

16 apply:

17 - ExemptFromChallenge("amazonbot")

18 - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/yandex.json")

19 apply:

20 - ExemptFromChallenge("yandex")

21 - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/baidu.json")

22 apply:

23 - ExemptFromChallenge("baidu")

24 - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/yahoo.json")

25 apply:

26 - ExemptFromChallenge("yahoo")

27 - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/sogou.json")

28 apply:

29 - ExemptFromChallenge("sogou")

30 - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/qwant.json")

31 apply:

32 - ExemptFromChallenge("qwant")

33 - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/babbar.json")

34 apply:

35 - ExemptFromChallenge("babbar")

36 - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/duckduckbot.json")

37 apply:

38 - ExemptFromChallenge("duckduckbot")

39 - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/ahrefs.json")

40 apply:

41 - ExemptFromChallenge("ahrefs")

42 - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/semrush.json")

43 apply:

44 - ExemptFromChallenge("semrush")

45 - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/lumar.json")

46 apply:

47 - ExemptFromChallenge("lumar")

48 - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/seznam.json")

49 apply:

50 - ExemptFromChallenge("seznam")

51 - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/naver.json")

52 apply:

53 - ExemptFromChallenge("naver")

54 - filter: MatchKnownBot(req.RemoteAddr, req.UserAgent(), req.URL.Path, "legit_bots/internet-archive.json")

55 apply:

56 - ExemptFromChallenge("internet-archive")

58# Declares the datafiles so crowdsec downloads them.

59data:

60 - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/googlebot.json

61 dest_file: legit_bots/googlebot.json

62 type: bots

63 - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/bingbot.json

64 dest_file: legit_bots/bingbot.json

65 type: bots

66 - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/applebot.json

67 dest_file: legit_bots/applebot.json

68 type: bots

69 - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/amazonbot.json

70 dest_file: legit_bots/amazonbot.json

71 type: bots

72 - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/yandex.json

73 dest_file: legit_bots/yandex.json

74 type: bots

75 - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/baidu.json

76 dest_file: legit_bots/baidu.json

77 type: bots

78 - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/yahoo.json

79 dest_file: legit_bots/yahoo.json

80 type: bots

81 - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/sogou.json

82 dest_file: legit_bots/sogou.json

83 type: bots

84 - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/qwant.json

85 dest_file: legit_bots/qwant.json

86 type: bots

87 - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/babbar.json

88 dest_file: legit_bots/babbar.json

89 type: bots

90 - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/duckduckbot.json

91 dest_file: legit_bots/duckduckbot.json

92 type: bots

93 - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/ahrefs.json

94 dest_file: legit_bots/ahrefs.json

95 type: bots

96 - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/semrush.json

97 dest_file: legit_bots/semrush.json

98 type: bots

99 - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/lumar.json

100 dest_file: legit_bots/lumar.json

101 type: bots

102 - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/seznam.json

103 dest_file: legit_bots/seznam.json

104 type: bots

105 - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/naver.json

106 dest_file: legit_bots/naver.json

107 type: bots

108 - source_url: https://hub-data.crowdsec.net/whitelists/benign_bots/legit_bots/internet-archive.json

109 dest_file: legit_bots/internet-archive.json

110 type: bots

111

Hub WAF configuration

« appsec-bot-challenge-exclude-search-engines »v0.2 by crowdsecurityWAF configuration

« appsec-bot-challenge-exclude-search-engines »
v0.2 by crowdsecurity
WAF configuration