Benchmark framework + bug fixes (#31)

[test] Add tests for matching and Engine based on Alex top 1000 domains
[perf] Add `bench` module to measure performance + memory consumption in Node.js
[perf] Avoid string copy in tokenizer
[perf] Avoid copy of filters during Engine initialization
[perf] Add minified version of bundle using Closure Compiler
[fix] Fix network filter option parsing
[fix] Fix `isAnchoredByHostname` in network filter matching
[fix] Re-work optimization framework to be easier to understand and extend
(also fix bugs in incorrect optimizations)
[breaking] `Engine.match` will now return the original filter matching
the request instead of a pretty-printed version (caveat: when
optimizations kick-in, the original filter might not be available anymore.)
[breaking] Build artifacts are now located in `dist` folder.
Remove dependency to babel and let TypeScript compile to ES3 instead
Simplify TypeScript config + make more strict
Simplify rollup config
Update dependencies
Make use of new `tldts` package for URL parsing
Add type definitions in build artifacts to allow the adblocker to be used
in a TypeScript project, as a library.
Update filters list assets + add script to ease future updates.
[fix] Cosmetic filter tokenization (ignoring styles) + add tests

.eslintrc

@@ -0,0 +1,12 @@
+{
+    "extends": "airbnb",
+    "env": {
+        "browser": true,
+        "es6": true,
+        "webextensions": true
+    },
+    "rules": {
+        "no-console": "off",
+        "no-continue": "off"
+    }
+}

.gitignore

@@ -1,3 +1,12 @@
 build/
 coverage/
 node_modules/
+.tern-port
+bench/dataset/chromedriver
+bench/dataset/ext.zip
+bench/dataset/requests.json
+bench/.bench.json
+example/background.bundle.js
+example/background.js
+example/content-script.bundle.js
+example/content-script.js

.travis.yml

@@ -1,28 +1,31 @@
 sudo: false
 dist: trusty
 language: node_js
+
 node_js:
-  - 7
+  - 6
   - 8
   - 9
   - node
 
-cache:
-  directories:
-    - node_modules
-
 notifications:
   email:
     on_failure: change
 
+before_install:
+  - npm install -g npm@latest
+
 install:
-  - npm install
+  - npm ci
 
 script:
-  - npm run test
+  - npm test
 
 before_deploy:
+  - npm run clean
+  - npm run build
   - npm run bundle
+  - npm run minify
 
 deploy:
   provider: npm

CHANGELOG.md

@@ -2,6 +2,35 @@
 
 ## *not released*
 
+## 0.2.0
+
+*2018-09-25*
+
+  * Add benchmark module + various improvements [#31](https://github.com/cliqz-oss/adblocker/pull/31)
+    * [test] Add tests for matching and Engine based on Alex top 1000 domains
+    * [perf] Add `bench` module to measure performance + memory consumption in Node.js
+    * [perf] Avoid string copy in tokenizer
+    * [perf] Avoid copy of filters during Engine initialization
+    * [perf] Add minified version of bundle using Closure Compiler
+    * [fix] Fix network filter option parsing
+    * [fix] Fix `isAnchoredByHostname` in network filter matching
+    * [fix] Re-work optimization framework to be easier to understand and extend
+      (also fix bugs in incorrect optimizations)
+    * [breaking] `Engine.match` will now return the original filter matching
+      the request instead of a pretty-printed version (caveat: when
+      optimizations kick-in, the original filter might not be available anymore.)
+    * [breaking] Build artifacts are now located in `dist` folder.
+    * Remove dependency to babel and let TypeScript compile to ES3 instead
+    * Simplify TypeScript config + make more strict
+    * Simplify rollup config
+    * Update dependencies
+    * Make use of new `tldts` package for URL parsing
+    * Add type definitions in build artifacts to allow the adblocker to be used
+      in a TypeScript project, as a library.
+    * Update filters list assets + add script to ease future updates.
+    * [fix] Cosmetic filter tokenization (ignoring styles) + add tests
+
+
 ## 0.1.13
 
 *2018-07-24*

assets/easylist-downloads.adblockplus.org/antiadblockfilters.txt

@@ -1,8 +1,8 @@
 [Adblock Plus 2.0]
-! Checksum: 25Q1CoQsDWVdOu09X4Cubg
-! Version: 201711280910
+! Checksum: l5tURbTn+U93MFhI6uz1PA
+! Version: 201808201230
 ! Title: Adblock Warning Removal List
-! Last modified: 28 Nov 2017 09:10 UTC
+! Last modified: 20 Aug 2018 12:30 UTC
 ! Expires: 1 days (update frequency)
 ! Homepage: https://easylist.to/
 !
@@ -34,6 +34,7 @@ koolhydraatarmerecepten.info##body > [style^="z-index: 1000000; "]
 ! *** antiadblockfilters:antiadblockfilters/antiadblock_english.txt ***
 flightradar24.com###bottomRightPanel
 videofun.me###flowplayer > div[class][style]:first-child
+abc-fonts.me###galepro-core-adb-enabled
 yocast.tv###ra9e
 flightradar24.com###responsiveBottomPanel
 myanimelist.net##._unit
@@ -52,6 +53,7 @@ toolslib.net##.row > .col-md-7 > .panel-default
 hearthhead.com,thottbot.com,wowhead.com##.sidebar-bg
 biggestplayer.me##.soret
 spanishdict.com##.whitelist
+pkmods.com##body > div[style*="z-index"][style*="background"]
 /adblock-alerter/js/adblock_alerter.js
 /adblock-img.
 play44.net,videobug.net,videofun.me,vidzur.com,yucache.net##[style^="position: absolute; width: 100%;"]
@@ -98,6 +100,7 @@ shadbase.com##img[alt="adblock"]
 @@/advertisement.js$script,~third-party,domain=fun-animes.fr
 @@/noadblock-nice-message/assets/js/ads.js?$script,~third-party
 !
+@@||abcbourse.com/scripts/ads.js$script,~third-party
 @@||ausujet.com/skins/common/ads.js
 @@||cyberdean.fr/js/advertisement.js
 @@||dbz-fantasy.com/ads.css
@@ -126,9 +129,11 @@ shadbase.com##img[alt="adblock"]
 @@||ps3-infos.fr/adsense/openads/ads/ads.js
 @@||ptitchef.com/js/ads.js
 @@||static.koreus.com/js/advert.js
+@@||cdn4.tennistemple.com/js/advert.js|$script,~third-party
 @@||trackr.fr/sites/all/themes/contrib/touch/js/miroirs/adwordstracking.js
 @@||universfreebox.com/js/advertisement.js
 @@||uplea.com/js/advertisement.js
+@@||peimg.fr/js/ads.js$script,domain=pubeco.fr
 @@||wamiz.com/css/ads.css
 @@||zagaz.com/img/blogads.css
 !
@@ -154,6 +159,7 @@ shadbase.com##img[alt="adblock"]
 ###adblocker_modal_overlay
 ###advertisementjsalert
 ###anti_adblock
+###detectadblock
 ###fnAdblockingOverlay
 ###message_adblock
 ##.AdblockMessage
@@ -193,25 +199,18 @@ shadbase.com##img[alt="adblock"]
 ##.top-bar-adblock
 ##[class][data-sitename][data-header-version] > div[id^="detection-block"]
 !
-streamwatching.fr###adblockplus
 jaitoutcompris.com###antiBlock
-vide-greniers.org###content > #pageContectFull > [id][style^="background-color:"]
-venez.fr###dv_warn_block
 iphonesoft.fr,iphonetweak.fr###msgNoPub
 retinaboys.com###screen_block
-doodle.com##.doodlead
-fan-de-cinema.com##.fancybox-overlay
-clkme.me,clkmein.com,cllkme.com,corneey.com,destyy.com,festyy.com,gestyy.com,jigbig.tk,newsfortrump.ga,sh.st,shorte.st,skiip.me,skipp.io,viid.me,wiid.me##.information-container
+lagrosseradio.com###warn
+ceesty.com,clkme.me,clkmein.com,cllkme.com,corneey.com,destyy.com,festyy.com,gestyy.com,jigbig.tk,newsfortrump.ga,sh.st,shorte.st,skiip.me,skipp.io,viid.me,wiid.me##.information-container
 etr.fr##.no_block
 radiox.com##.notifyjs-corner
-vide-greniers.org##[href="/explications-publicite"]
-tunnl.com##[id^="blockedAddons"]
 papergeek.fr##[id^="padb_widget_"]
 forumdz.com##body .breadcrumb + center > [color="red"] > b
 forum.pcastuces.com##div > [src] + [id] + [class][style^="display: block;"]
-prixdubaril.com##div > div[class^="sweet-"][style]
-phonandroid.com##div[class][style*="none repeat scroll 0% 0%;"]
 bd-sanctuary.com,cine-sanctuary.com,comics-sanctuary.com,manga-sanctuary.com##div[style^="background:#444;"]
+pausecafein.fr###adb-pop
 !
 #@##adblocktest
 !
@@ -222,7 +221,6 @@ ot-boutique.fr#@##publicite
 iwin.com#@##sponsorText
 fairytailmx.com#@##sponsored-ad
 forum.virtualdjing.com#@#.adFrame
-abcbourse.com#@#.adsbox
 dealerdunet.fr#@#.text_ads
 comptoir-info.com#@#.textad
 !
@@ -236,30 +234,38 @@ usgang.ch###rectangle-wrapper
 ! *** antiadblockfilters:antiadblockfilters/antiadblock_latvian.txt ***
 ! *** antiadblockfilters:antiadblockfilters/antiadblock_romanian.txt ***
 ! *** antiadblockfilters:antiadblockfilters/antiadblock_russian.txt ***
-d3.ru##section[data-uid="ad"]
-d3.ru##section[data-uid="extraAd"]
-d3.ru##.b-promo__placeholder
-mycrib.ru,piggy-bank.online##[data-type="anti-abp"]
-fightnews.info##body > blockquote
-newdeaf-online.net,newsland.com,logist.uxxo.ru##body > div[id]:not([class])[style="display: block;"]
-psjailbreak.ru##div[id^="advertur_"] + div[id][style^="width"]
-glav.su##body > div[id]:not([class])[style*="background"][style*="display: block"]
-sd-company.su###dom_adblock
-webdesign-master.ru###adbd
+drive2.ru##a[href][target="_blank"] > img[src][style^="width: 100%; height: auto;"][style*="vertical-align: middle; max-width: 240px;"]
+delfi.ee###ab--notification-ribbon
+minimultik.ru###abDetected
+patrioty.org.ua###ad_pu_modal
+fastpic.ru###adblock-detect-message
 vesti-ukr.com###adblockpopup
-mindspace.ru###adblock_screen
 happy-hack.ru###admitad-cookie-check-popup
+chipinfo.pro###ads-row
 wotactions.com###ads_warning
 firebit.net###adv-300 + div[class][align="center"][style^="background-color"]
+novostroy-m.ru###advert_result
 cxem.net###antiAdBlock
 elhow.ru###b-info-wrapper
+webdesign-master.ru###blocka
 ma.by###bodysubcont > div[style^="padding"]:not([id]):not([class])
+pln-pskov.ru###check_ad_block
+scrin.org##center > div[id][style="display: block;"]
+helpsetup.ru###d930
 delfi.ee###delfiTopBar
+sd-company.su###dom_adblock
+forum.bestflowers.ru###exposeMask
+pastenow.ru###google-block
+blackdesert-info.ru###header > div[style*="z-index"][style*="999999"]
 info-mage.ru###helpme[style="display: block;"]
+poncy.ru###hide-adblock
 kissvk.com###madBlockModal,.modal-backdrop
 gisclub.tv,teremok.org.ua,webos-forums.ru###mdl_adb
 ostroh.info###modal.modal-bg
+7days2die.info###nafikblock
 all-episodes.net###odin
+center-yf.ru,lektsii.org,studopedia.org,studopedia.ru,studopedia.su###opaco,#popup
+forummikrotik.ru###phpbb-ads-ab
 kompravda.eu,kp.ru###popapIsBlockedAd
 pb.wtf,piratbit.org,piratbit.ru###result > .request
 mnogoto4ka.ru###sb-site > div[class][style*="z-index"][style*="999999"]
@@ -268,86 +274,126 @@ kaermorhen.ru###top-ban-container
 numberempire.com###toplink > h3[style="color: red"]
 ganoderma-mushrooms.com,korrektor.name,varicream.org###turn_advert.adblock
 czx.to###vdl_ctnr
+vsaduidoma.com##.ABD_display_wrapper
 eurabota.com##.abtop
 sprashivalka.com##.ad
+topcraft.ru##.ad-info
 cc-fan.ru,cc-fan.tv,cn-fan.ru,fox-fan.ru,goodfon.ru,satsis.info,swordmaster.org##.adb
-hentaiz.org##.adblocker
 vsadu.ru##.adb-block
+andevice.ru##.adb-def
 tuchkatv.ru##.adb_m
-hentaiz.org,life.ru,mastergrad.com,forum.qrz.ru##.adblock
+brazzers-hdx.club##.adblck
+forum.qrz.ru,hentaiz.org,life.ru,mastergrad.com,tvfeed.in##.adblock
 exler.ru##.adblock-pls
 zakon.kz##.adblock_alert
-nulled.cc##.adblock_floating_message
+ogorod.ru##.adblock_popup_warning
+hentaiz.org##.adblocker
 kubik3.ru##.adbs
 print-post.com##.addbl
 24video.xxx##.adt
+rusrek.com##.alert
 benfilm.net##.alert-block
-ria.ru##.antiblock
+stroi-help.ru##.alert_warn
 rufilmtv.org##.attention
+mel.fm##.b-ad-space__placeholder
 ixbt.com##.b-blockadblock
+d3.ru##.b-promo__placeholder
 cs-amba.ru##.banka
 electric-house.ru##.blok_forum > div[style]
+it-actual.ru##.blok > div[class^="blok-"]
 audio-hi-fi.ru,electro-sila.ru##.blok_new > div[style]
+rutor1.ru##.box_in > .text > p > font
+tsn.ua##.c-aside-embed
 kinoafisha.info,tvcook.ru##.cat
-autonews.ru,rbc.ru##.checkad_info
+autonews.ru,rbc.ru,rbcplus.ru,sportrbc.ru##.checkad_info
 rusopen.com##.classblsi.addshr
-www.qrz.ru##.content_row > div[class][-abp-properties="background*z-index"]:not([style])
 smotrisport.online,smotrisport.tv##.dont-add-our-site-pls
+goha.ru#@#.goha_ads
+goha.ru#@#.goha_ads_acceptable
+themoviebay.ru##.is-danger.notification
+ivi.ru##.ivi-footer ~ div[class] > style + div[class]
 go.mail.ru##.js-adb-msg
-rbcplus.ru##.js-checkad-warning
 space.utema.ru##.main-content-padding > div[class]:not([itemprop])
 megatest.online##.msg-AdBlock
 ykt.ru##.n-banner--adblocked
-animeland.su,itcats.ru,noadsradio.ru,sharkgame.ru,w10.jeanzzz.ru##.no-adb
+animeland.su,itcats.ru,nekto.me,noadsradio.ru,sharkgame.ru,w10.jeanzzz.ru##.no-adb
+mail.rambler.ru##.notification-warning
 delfi.lv##.notification-widget
 radio.yandex.by,radio.yandex.kz,radio.yandex.ru,radio.yandex.ua##.page-root .notify
 radio.yandex.by,radio.yandex.kz,radio.yandex.ru,radio.yandex.ua##.page-root .notify *
 music.yandex.by,music.yandex.kz,music.yandex.ru,music.yandex.ua##.page-root > .notify
 music.yandex.by,music.yandex.kz,music.yandex.ru,music.yandex.ua##.page-root > .notify *
+alexfilm.cc##.panel-primary
 darkteam.net##.pls_rmv
+animedub.ru##.poverh
+webdesign-master.ru##.preloader + style + div[class] > p
 pythonworld.ru##.pythonworld-contributor
 fanfics.me##.red.nodata
+101.ru##.showMessageAdBlock
 playunturned.ru##.side_left > center
 playunturned.ru##.side_right > center
-101.ru##.showMessageAdBlock
 objavlenija.respublika.cz##.sweet-alert,.sweet-overlay
-hs-manacost.ru##.td-a-rec
+xn--80aikhbrhr.net,кинокрад.net##.warning-adblock-box
+forum.bestflowers.ru##.xenOverlay
 /api/scripts/mainjs?token$script,domain=liveresult.ru
-/forum/docs/requests.js$script,domain=nnm-club.lib|nnm-club.name|nnm-club.tv|nnmclub.to
+/cookie_check.min.js$domain=acetv.me
+/forum/docs/requests.js$script,domain=nnm-club-me.appspot.com|nnm-club.lib|nnm-club.me|nnm-club.name|nnm-club.tv|nnmclub.to|nnmclub.tv|nontorclub.in.net
 /fuckadblock.js$domain=ace-stream.tv
+@@||goha.ru/app/video/adblock.teaser.js|$script,domain=goha.ru
 @@||sports.ru^*/adfox.$xmlhttprequest
+mycrib.ru,piggy-bank.online##[data-type="anti-abp"]
+lr-club.com##a[href="https://adblockplus.org/ru/faq_basics#disable"]
 pic4you.ru,pic5you.ru##body > #bb
 forum.vip-cxema.org##body > #document_modal.modal,#some_ad_block_key_popup
 it-enginer.ru##body > #modal[style*="blur"]
 swaego.ru##body > .special-message-wrapper
 ukr.net##body > .yellow-die
+in-trend.biz##body > [id][style^="z-index"]
+fightnews.info##body > blockquote
+olegon.ru##body > div:not([id]):not([class])[style^="color"][style*="position: fixed; width: 99%"]
 factorio.su##body > div[class][style*="background"][style*="display: block"]
 gdekluet.ru##body > div[class^="stop-block"]
+glav.su##body > div[id]:not([class])[style*="background"][style*="display: block"]
+logist.uxxo.ru,newdeaf-online.net,newsland.com##body > div[id]:not([class])[style="display: block;"]
 factorio.su##body > div[id][class][style*="background"][style*="visible"]
+gta.com.ua##body > div[id][style*="background"][style*="visible"]
+portfel.info##body > div[id][style="display: block;"]
 tut.by##body > div[style*="position: fixed;"][style*="right: 0;"]
-bychico.net,dyrik.ru,ecosounds.net,fainaidea.com,in-drive.ru,obozrevatel.com,oknotivi.ru,sandero.ru,tehnobzor.ru##body > div[style*="z-index"][style*="999999"]
+1snau.ru,bychico.net,dyrik.ru,ecosounds.net,fainaidea.com,in-drive.ru,musecube.org,obozrevatel.com,oknotivi.ru,pogodnik.com,sandero.ru,tehnobzor.ru##body > div[style*="z-index"][style*="999999"]
+torrnado.win##body > div > .a-overlay
 dugtor.ru##body > noindex
+lostfilm.tv##div[class^="alarm"]
 gismeteo.by,gismeteo.ru##div[class^="stopblock"]
+finfront.ru##div[class^="swal2"]
+www.kinopoisk.ru##div[class$="_visible_show"]
 sport.rbc.ru##div[data-name="adb_warn_sport"]
 eset-key.ru,politonline.ru,showmehow.ru##div[data-type="anti-abp"]
-lostfilm.tv##div[class^="alarm"]
-pvcalc.ru##div[style^="position:fixed; top:0px; left:0px; opacity: 0.9;"]
+savevideo.me##div[id^="ads_notice_"]
 acomics.ru##div[style*="/design/common/pic/please"]
+pvcalc.ru##div[style^="position:fixed; top:0px; left:0px; opacity: 0.9;"]
 ph4.ru##div[style^="width: 728px; height: 90px;"]
+7sim.net##html > body ~ div
+d3.ru##section[data-uid="ad"]
+d3.ru##section[data-uid="extraAd"]
 ph4.ru##table[width][height="90"][style]
 bigpuzzle.ru##td[valign="top"][width="200px"][height="600"] > div[style]
+www.qrz.ru#?#.content_row > div[class]:-abp-has(> .inner > a[href="/help/adblock/"])
+newkaliningrad.ru#?#div[class]:-abp-has(> [class]:-abp-contains(/adblock!/i))
 ||adscat.ru^$third-party
 ||animeteatr.ru/a-detector/
+||anivisual.net/mim*.js|$script,domain=anivisual.net
 ||avito.ru^*/some-pretty-script.js
 ||fairtop.in/template/js/fdetect.js
 ||i140.ru/js/etxm*.js|$script
 ||ipprof.ru/wp-content/plugins/ad-blocking-advisor/
 ||kontrolnaya-rabota.ru^*/fuckadblock.js$script
 ||kontrolnaya-rabota.ru^*/toastr.$script,stylesheet
 ||mhelp.kz/wp-content/plugins/wp-adblock-dedect/
+||olegon.ru/jscookie.js
 ||pwnews.net/des/antiblock.js$script
 ||rusnext.ru/advar.js$script
 ||sdamgia.ru/img/blockadblock_
 ||shellcat.ru^$third-party
+||stroi-help.ru/forum/script_new.js
 ! *** antiadblockfilters:antiadblockfilters/antiadblock_slovak.txt ***
 ! *** antiadblockfilters:antiadblockfilters/antiadblock_spanish.txt ***