grex "adbeat" "admantx" "ahrefs" "appinsights" "aspiegel" "awariobot" "awariosmartbot" "barkrowler" "blexbot" "bomborabot" "buck" "bvbot" "bytespider" "ccbot" "censysinspect" "checkhost" "cincraw" "claudebot" "clickagy" "cocolyzebot" "criteobot" "crawler" "curl" "df bot 1.0" "domainstatsbot" "domcopbot" "dotbot" "embed.ly" "facebookexternalhit" "friendlycrawler" "grapeshotcrawler" "gulperbot" "httrack" "ias_crawler" "internet-structure" "ioncrawl" "keys-so-bot" "linguee" "linkfluence" "magpie-crawler" "mediatoolkitbot" "megaindex" "mj12bot" "nimbostratus" "node" "omgili" "onalyticabot" "panscient.com" "petalbot" "postman" "proximic" "riddler" "rogerbot" "sbl-bot" "seekport" "semantic-visions" "semanticbot" "serpstatbot" "sogou" "sqlmap" "traackr" "trendictionbot" "ttd-content" "voluumdsp" "wc-test-dev-bot" "webtechbot" "wget" "whatcms" "zgrab"f(?:acebookexternalhit|riendlycrawler)|i(?:nternet\\-structure|oncrawl)|(?:semantic\\-vision|nimbostratu|whatcm)s|(?:grapeshot|magpie\\-|ias_)crawler|(?:(?:m(?:ediatoolkit|j12)|do(?:m(?:ainstats|cop)|t)|(?:awariosmar|serpsta)t|trendiction|onalytica|c(?:ocolyze|riteo|laude|c)|webtech|bombora|gulper|petal|roger|b(?:lex|v))bo|(?:wc\\-test\\-dev|sbl)\\-bo|c(?:ensysinspec|heckhos)|ttd\\-conten|(?:semantic|awario)bo|keys\\-so\\-bo|seekpor|adbea|wge)t|p(?:anscient\.com|roximic|ostman)|a(?:ppinsight|href)s|(?:lin(?:kfluenc|gue)|nod)e|df bot 1\.0|bytespider|(?:barkro|cra)wler|megaindex|voluumdsp|clickagy|embed\.ly|(?:aspiege|cur)l|cincraw|httrack|admantx|traackr|riddler|omgili|sqlmap|sogou|zgrab|buck