# === ADDITIONAL BOT PATTERNS (Expanded) ===
# === AI & ML CRAWLERS ===
# === API TESTING TOOLS ===
# === CLOUD & INFRASTRUCTURE ===
# === CONTENT SCRAPERS ===
# === DOWNLOAD MANAGERS & TOOLS ===
# === EMAIL HARVESTERS ===
# === FEED READERS ===
# === GENERIC CRAWLERS & SPIDERS ===
# === HTTP LIBRARIES & CLIENTS ===
# === MISCELLANEOUS SUSPICIOUS ===
# === MONITORING & TESTING ===
# === PROXY & VPN INDICATORS ===
# === SECURITY SCANNERS & VULNERABILITY TOOLS ===
# === SEO & ANALYTICS BOTS ===
# Add more patterns as needed - file can be updated with additional sources
# Comprehensive list of bot and crawler user agent patterns
# Last updated: 2026-01-12
# Lines starting with # are comments
# Sources: crawler-user-agents.json, custom threat intelligence, security research
# Suspicious User Agent Patterns
# These patterns are matched case-insensitively
# Total entries: 1500+ patterns
360spider
80legs
aboundex
aboundexbot
acapbot
accelerator
accelobot
acoonbot
acunetix
adaxasbot
address
addsugarspiderbot
addthis
admantx
admantxbot
adressendeutschlandbot
adsbot
aggregator
ahrefs
ahrefsbot
aibot
aihitbot
aiohttp
aisearchbot
akamai
akka
akregator
alexa
alexabot
alienvault
alkalinebot
amass
amazonaws
amazonbot
analytics
ankhbot
anonymous
anonymousbot
antbot
anthropic
any.run
anybot
anyevent
anywhere
apache
apercite
apimatic
apis-google
appengine-google
applebot
appscan
arachmo
arachni
archive
archive.org_bot
archiver
archiverbot
aria2
arielisbot
arjun
arkime
arobot
artillery
asafawabot
ask jeeves
aspiegelbot
aspseek
assetfinder
asterias
atom
atomz
attach
auditbeat
augurfeedreader
autoemailspider
automa
automation
axel
axiom
axios
azure
backend
backlinkcrawler
backlinktest
backstreet
backstreetbrowser
badass
baidu
bandit
banshee
barkrowler
batchftp
bdfetch
beautiful soup
beautifulsoup
becomebot
bent
betabot
bibnum.bnf
bigfoot
biglotronbot
bigsearch.ca
bilgibot
bimbot
binaryedge
bingbot
bingpreview
birubot
bitbucket
bitlybot
blackduck
blackwidow
blex
blexbot
blinkx
blinkxbot
blitzbot
blocknotes
blogbridge
bloglines
blogpulse
blogscope
blogsearchbot
blogshares
blogslive
blogtrottr
bloomreach
bluehost
boitho
bombardier
boostengineboost
bot-apoena
bot.ara
botify
botje
botmobi
brainobot
brandverity
brandwatch
bravebot
brook
btwebclient
builtwith
builtwithbot
burp
burpsuite
butterfly
buzzbot
bytedance
bytespider
caelistisbot
calendarsearch
careerbot
casper
catchbot
ccbot
celestial
censys
chakram
changedetection
chaos
charlotte
chatgpt
chatgpt-user
check
checklinks
checkmarknetwork
checkmarx
checkpriv
cheesebot
cherrypicker
chlooe
chrome-headless
chromium
circleci
citeseerxbot
cityreview
clariabot
claude
claude-web
clipsto
cliqzbot
cloak
clojure
cloudflare
cloudfront
cmcm
coccoc
cohere
cohere-ai
coldfusion
collect
collecta
collector
collegebot
combine
commix
common crawl
commoncrawl
comodo
companybook
contabo
contentsmartz
convera
cookbook
coolbot
copier
copyrightcheck
copyscape
cosmos
covario
cowbot
craftbot
crashtest
crawl
crawlconvera
crawldaddy
crescent
crowsnest
csci_b659
cuckoo
cuill
curious
curl_image
custo
dalfox
dart
dataparksearch
dataprovider
daumoa
ddos
deepcrawl
deepindex
deepnetbot
deliciousbot
detectify
deusu
developers.google
diffbot
digg
digger
digitalocean
dillo
dirb
dirbuster
discordbot
disqus
distilator
dittospyder
dlvr.it
dlwebbot
docker
domainappender
domaindb
dotbot
dotnetdotcom
doubleclick
download
downloader
dragonbot
dragonfly
dreamhost
dredd
drupact
drupal
duckduckbot
duckduckgo
dumbot
e-mail
easouspider
easydl
ec2linkfinder
ecatch
ecosiabot
edisterbot
eirgrabber
elasticsearch
elasticsearchbot
elixir
emai1
email
emailcollector
emailextractor
emailfinder
emailgrabber
emailharvester
emailhunter
emailreaper
emailscraper
emailsearch
emailsiphon
emailsnatcher
emailspider
emailstripper
emailsucker
emailsweeper
emailverify
emailwolf
embedly
emefgebot
enswer
epicbot
eright
erlang
erocrawler
eu2bot
europarchive
evaal
eventax
everyfeed
exabot
exactseekautobot
exploratodo
express
extract
extractor
extractorpro
ezooms
facebookbot
facebookexternalhit
factorlabs
fairad
faraday
fastbug
fastladder
fastly
fatbot
favicon
fdse
feed
feed seeker
feedblitz
feedburner
feeddigest
feedexplorer
feedfetcher
feedhub
feedly
feedme
feedmonkey
feedparser
feedreader
feedshow
feedsquash
feedster
feedvalidator
feedzirra
femtosearch
feroxbuster
ferret
fetch
fetcher
ffuf
filebeat
findlinks
findthatfile
flamingo
flashget
flexum
flickrbot
flipboard
flood
fluentd
fluffy
fmcrawler
fofa
followsite
fooooo_webspider
formatmediabot
fortify
fortifysca
fr-crawler
francite
free web headers
freshdownload
freshwebexplorer
friendfeedbot
frisby
frontpagebot
froogle
fuelbot
functionbeat
funnel
fyberspider
g2crawler
gaisbot
galaxybot
galbot
gatling
geniebot
genieo
getright
ghost
gigablast
gigabot
girafabot
github
gitlab
gnomit
go!zilla
go-ahead-got-it
go-http
gobuster
godaddy
goforit
goforitbot
golang
gold crawler
golem
google-adwords
google-apps-script
google-certificate-transparency
google-favicon
google-read-aloud
google-site-verification
google-structured-data-testing-tool
google-xrawler
googlebot
googlecloud
googleimageproxy
googlemobile
googleother
gospider
gossip
gost
govbot
gozilla
gptbot
grab
grabber
grabnet
gracebot
grail
grapeshot
graphql
graylog
grendel-scan
grpc
grub
gsa-crawler
gsitecrawler
gulliver
gureedo
guzzle
haansoft
hakrawler
harvest
harvester
harvestman
haskell
hatena
hatrackbot
headless
health
heartbeat
helix
heritrix
hetzner
heureka-shopping
hide
hoic
holmes
homepagebot
hoowwwer
hoppscotch
hostgator
hostinger
hotjava
hound
htdig
htmlparser
http-client
http-get
http-header-abfrage
http-request
http-tiny
http.agent
http.client
http::lite
httparty
httpclient
httpget
httpie
httplib
httplib2
httprobe
httpunit
httpx
httrack
huawei
huaweiwebcatbot
hubspot
hulk
humanlinks
hunter
hverify
hybrid-analysis
hydra
hyper-decontextualizer
hysteria
ia_archiver
iaskspider
iccrawler
ichiro
idbot
iframely
igdespyder
ikibot
ilsebot
iltrovatore
image
imagefetchertestclient
immuniweb
inagist
incognito
indy library
indylibrary
ineturl.dll
infolink
infomine
infopath
inforstbot
infoseek
inktomi
inoreader
insomnia
instagrambot
installatron
instapaper
integromedb
internet ninja
internetseer
interweb
intruder
ip-web-crawler
iria
irlbot
irvinebot
isomorphic-fetch
issuecrawler
istella
ixquick
jaeles
jamebot
janformanbot
jaxified
jbot
jdownloader
jemmathetourist
jenkins
jetcar
jeteye
jigsaw
jmeter
jobboerse
jobboersebot
jobo
jobseeker
jobspider
jocrawler
joe sandbox
jograbot
joomla
jorgee
journalbeat
journalismbot
jrtwine
julia
jyxobot
k2spider
kakaobot
kalooga
kameleon
kamikazebot
karate
katana
kenran
keybot
kibana
kinjabot
kitewind
klimonbot
klipfolio
knowitall
koepabot
komodiabot
konqueror
koreasearch
koreavna
kotlin
kraken
krazybot
kubernetes
kurzorbot
kyluka
l.webis
lakequepasacreep
languagebot
laravel
larbin
lawinfo
leachbot
leboncoin
leechftp
leia
leikibot
lexxebot
libbot
librarybot
libweb
libwww
libwww-perl
liferea
likse
linguee
link sleuth
link valet
link validator
linkchecker
linkdex
linkdexbot
linkedinbot
linkextractorpro
linkfinder
linkman
linko
linkpad
linkpadbot
links
linkscan
linksmanager
linksweeper
linkwalker
linode
lipperhey
litefeeder
livelap
livelapbot
lmspider
lnspiderguy
loadimpact
loadrunner
loadtestit
locust
logstash
loic
lokibot
looksmart
loschie
ltx71
lwebis
lwp-trivial
lxml
lycos
magento
magpie
mail
mail.ru_bot
mailcrawler
mailto
majestic
majestic12
majesticbot
makecontactbot
malwr
manager
mappydata
markwatch
masagool
mass
masscan
matlab
mattermost
meanpathbot
mechanize
mediapartners
megaindex
megaupload
megite
memorybot
metacarta
metafeeds
metager
metaglimpse
metalogger
metamark
metaspider
metasploit
metatagsdir
metauri
metricbeat
mfc_tear_sample
microsoft
microsoft office
microsoft-atl-native
miner
miniflux
mirabilis
misp
mister pix
mj12
mj12bot
mnogosearch
mockoon
moget
mogimogi
mojeekbot
moloch
monit
monitor
monkeycrawl
mooseek
morgandoe
morning paper
mothra
msiecrawler
msnbot
msproxy
msrabot
msrbot
mturk
multitext
multiviewlab
musobot
mvaclient
myotisfabilis
nabot
nagios
naiveproxy
namecheap
nameprotect
napster
nationaldirectory
naverbot
navroad
nearsite
nec-meshexplorer
nedstatbasic
needle
nerdbynature
nerdybot
nessus
netants
netcraft
netcraftsurvey
netcrawl
netcrawler
netid
netmarketshare
netmechanic
netprobe
netresearch
netresearchserver
netscape
netseer
netspider
nettrack
nettyhttp
netvibes
network-monitor
netzip
neuralbot
newman
news aggregator
newsbeuter
newsblur
newsboat
newsgator
newsgrazer
newsme
newsnow
newspaper
nexpose
nextisspider
nezha
ngrambot
nicebot
nightmare
niki-bot
nikto
nimblecrawler
ninja
ninjabot
nissebot
nlnz_iaharvester2008
nmap
nobody
nock
node
node-fetch
nogate
nomad-v2.x
nominet
norefer
normasearch
noteworthybot
notifixious
noxtrumbot
npbot
nrsbot
ns8bot
nsauditor
nuclei
nuggetizer
nutch
nuthatch
nuzzel
nvaclient
nwspider
nymesis
obfs
objectssearch
obot
ocelli
ocelot
octave
octopus
odbcbench
oegpspider
offbyonebot
offline
okhttp
omea
omgili
omgilibot
omniweb
omtrdc
oncrawl
online link validator
onyphe
onyx
openacoon
openapi
openaudit
openbotcheck
opencart
openhosebot
openintelligencedata
openlinkprofiler
openvas
openwebspider
orangebot
orbit
orbiter
orbnet
organic
orgprobe
orion
osmedeus
osmosis
otwarchive
ourega
outbrain
outclicksbot
overture
owasp
packetbeat
page-analyzer
page2rss
pagebiteshyperbot
pagepeeker
pagerankcrawler
pagesinventory
pagething
panscient
paradise
paramspider
paros
parsijoo
pastefire
pavuk
paypalbot
pcbrower
pcore
peew
peoplefinder
perfectmarket
perl
perman
perplexity
perplexitybot
petalbot
phantom
phantomjs
phin
picscout
picsearch
pieno
ping
pingdom
pingoscope
pinterest
pinterestbot
piplbot
pita
pixfinder
pixray
plagger
planetwork
playwright
plukkie
plumanalytics
pockey
pompos
popscreenbot
postank
postman
postpost
postrank
powermarks
prerender
press-room
prestashop
prism
privacyawarebot
privacybot
private
privatebot
proai
probe
probely
probethenet
project25499
projectdiscovery
protopage
proximic
proxy
psbot
publiclibraryarchive
puppeteer
purebot
purity
pursuit
pycurl
pyloris
python-httpx
python-mechanize
python-requests
python-urllib
qirina
qseero
qualys
quant-client
quepasacreep
queryn
questbot
quic
quickobot
qwant
qwantbot
qwantify
r-project
radian6
radiance
rainbow
rambler
rankactive
rankivabot
rankur
rapid7
rapidshare
readability
reader
reaper
rebelmouse
reconftw
redcarpet
redditbot
reddot
rediffbot
reedoobot
reget
regex
relative link checker
relevare
replibot
request
requests
researchscan
rest
rest-client
restassured
restclient
restsharp
retire.js
retrofit
riddler
rigor
ripoff
ripper
ripz
robocop
robofetch
robot
robozilla
rocket
rogerbot
ronzoobot
roochbot
rotunda
rover
rss4j
rsscache
rssowl
rsyslog
ruky
ruler
rulinki
rumble
runnk
rust
rv-softscanbot
safesearch
safetynet
salesintelligent
salvatore
sandcrawler
sbider
sbsearch
scala
scalabot
scalaj
scaleway
scan
scan4mail
scanbot
scanrobot
schibstedt
schoolbot
schrodinger
schweiz-search
scientificcommons
scollspider
scooter
scoutjet
scrapbook
scrapebot
scrapy
screamerbot
screaming
screaming frog
screamingfrog
screencraft
screenshotlayer
scrubby
scrubtheweb
search17
searchmarking
searchmetrics
searchmetricsbot
searchpreview
searchsight
secure
securebot
securitytrails
seekbot
seeker
seewithkids
seigensbot
sekombot
selenium
semanticbot
semanticdiscovery
semrush
semrushbot
senrigan
seobility
seocompany
seoconsultants
seodat
seodiver
seoengbot
seokicks
seolyticscrawler
seomoz
seoprofiler
seoptimer
seoradar
seoscan
seositecheckup
seotoolbot
sepop
settingsguide
seznambot
seznamemailbot
seznamrss
shadowsocks
shareaza
sharethis
shellshock-scan
shim
shodan
shopify
shopwiki
showlinks
showyoubot
shunpikebot
sidestripe
siege
sigma
silk
simplepie
siphon
sistrix
sitebeam
sitebot
sitecheck
siteexplorer
siteground
siteimprove
sitelockspider
sitemon
sitesnagger
sitesucker
skipfish
skygrid
slack-imgproxy
slackbot
slackbot-imgproxy
slackbot-linkexpanding
slimerjs
slowhttptest
slowloris
slurp
smartdownload
smarthub
smartwit
smore
snagger
snapchatbot
snappy
snarfer
snookit
snoop
snoopit
snort
snortobot
snyk
soap
sobees
socialwhip
socks
socscibot
softlayer
sogou
sohu
solofield
sonarqube
sonic
soso_img
soso_web
sosoimagesearch
speedy
sphere
spheroid
sphider
spiderman
spidermonkey
spidersilk
spidervac
spiderview
spinn3r
splunk
spokesspider
springbot
sputnikbot
sqlmap
squarespace
squid
sqwirm
ssl-tools
stack
stackrambler
stamp
stardownloader
stargate
startpagebot
statbot
statcounter
status
statusbot
stealth
stero
stix
stockphotosearch
stoplight
stress
stripper
subfinder
subot
suchbaer
sucker
suggybot
summify
sunrise
superagent
supertest
surdotlybot
surftrap
suricata
surveybot
susie
suzukacrawler
suzuran
swagger
swehlo
swift
swisscowsbot
sygol
symfony
syndic8
synoobot
synthetic
syslog-ng
sysomos
sysscan
sysscanbot
systran
szukacz
t-h-u-n-d-e-r-s-t-o-n-e
tagoobot
tagscan
tailrank
take
takeout
talkro
tamus
tangibleebot
tarantula
taringa
taskexecutor
taxii
taxinomia
technology
teesoft
telegrambot
teleport
telesoft
telesphoreo
telesphorep
tenable
tencenttraveler
teoma
terrawizbot
test
testbot
tetrabot
textbot
thatbot
thatroboty
thc-ssl-dos
the-american-directory
thefreelibrary
thenomad
threatminer
thumbshots
thuvien
tiedeweb
tiktokbot
tineye
tiny tiny rss
tiny-request-router
titan
titlemaker
to-dress
tocrawl
todobr
toluu
toolbarqueries
toplistbot
topseek
topsy
torrent
torshammer
touche
toxi
tpsystem
traackr
trackback
travis
trendictionbot
trendsmap
tripadvisor
trojan
trojanabot
tropez
truwobot
tsunami
ttobbot
ttrss
tumblr
tunnel
turingos
turnitin
turnitinbot
turtlescan
tutorgig
tweetedtimes
tweetmeme
tweezler
twenga
twengabot
twice
twiceler
twikle
twitterbot
twitturls
twurly
typhoeus
u2bot
uaslinkchecker
ubrowser
ubuntuone
ucbrowser
uget
ultraseek
unchaos
undervind
undici
unirest
unister
unisterbot
unitek
universal-feed-parser
unrulymedia
updownerbot
uptime
uptimerobot
upwebserver
urlappendbot
urldispatcher
urllib
urllib3
urlresolver
urlscan
urltrends
urly
usafirms
v2ray
vagabondo
valet
valkyrie
varchar
vb project
vbseo
vega
vegeta
veoh
veracode
vermut
vertical
vertiexplorer
vicifus
virusdie
virustotal
visionutils
vitalnet
vkontakte
vkshare
vless
vmess
voideye
voila
voilabot
voltron
vortex
vultr
vyu2
w3af
w3c-checklink
w3c-mobileok
w3c_css_validator
w3c_unicorn
w3c_validator
waclookbot
wappalyzer
wapspider
watchmouse
wavefire
waybackurls
wbsearchbot
wdgvalidator
web-archive
web-sniffer
web.archive
web.image.collector
web2disk
web2epub
webalta
webanalyze
webbot
webcapture
webclipping
webcloner
webcollage
webcompanycrawler
webcookies
webcopier
webcorp
webcrawler
webdata
webdatastats
webdownloader
webdriver
webemailextrac
webenhancer
webfetch
webfindbot
webflow
webgo
webgobbler
webimages
webindex
webleacher
webmaster
webmastercoffee
webmastergrade
webmasterworld
webmasterworldforumbot
webmon
webpagetest
webpictures
webreaper
webripper
websauger
webscarab
websecurify
website quester
websitepulse
websitequester
websitetraffic
websnarf
webspider
webster
webstripper
websucker
webtext
webthumb
webtrends
webvac
webviewer
webwasher
webwatch
webwhacker
webyieldbot
webzip
wechat
wedigger
wesee
wfuzz
whacker
whatdoes
whatsapp
whatuseek
whatweb
whitesource
whizbang
whoapi
wibiya
widewail
wigle
wikiabot
wikido
willow
winlogbeat
wiremock
wisenutbot
wkhtmltopdf
wminer
wmtips
wocbot
wofindeich
womlpe
woobot
woocommerce
wordpress
worio
woriobot
worldlight
wotbox
wowebpage
wpscan
wrightian
wsr-agent
www-collector
www-mechanize
www:mechanize
wwweasel
wwwoffle
xaldon
xaldonwebspider
xbot
xenu
xerka
xforce
xget
xirq
xml-sitemaps
xovibot
xref
xs4all
xsstrike
xtreme
xyleme
y!j-ase
y!j-bri
y!j-brm
y!j-brp
y!j-brw
yacy
yahoo
yahoocachesystem
yahoofeedseeker
yahooysmcm
yammybot
yandex
yandexbot
yanga
yara
yardspider
yats
yeti
yioopbot
yodao
yomobot
yooglifetchagent
yoono
yottaamonitor
you.com
youbot
youdaobot
zabbix
zapmeta
zealbot
zebot
zeebo
zeek
zeerch
zeus
zippp
zitebot
zizidata
zmap
zmeu
znuutz
zombie
zoomeye
zoominfobot
zooshot
zspider
zyborg
curl/
wget/
python-
go-http
node-fetch
axios/
got (
libwww
httpclient
java/
ruby/
perl/