From 087da6407d68e95eafa7aa846a5a984e6d20980f Mon Sep 17 00:00:00 2001 From: Raymond Hill Date: Sun, 15 Aug 2021 10:43:36 -0400 Subject: [PATCH] Add support for nodejs flavors The main nodejs flavor is "npm", which is to be used to lint/test and the publication of an official npm package -- and by design it has dependencies on mocha, eslint, etc. A new flavor "dig" has been created with minimal dependencies and which purpose is to easily allow to write specialized code to investigate local code changes in uBO -- and it's not meant for publication. Consequently, "make nodejs" has been replaced with "make npm", and a new "dig" target has been added to the makefile, to be used for instrumenting local code changes for investigation purpose. --- .github/workflows/main.yml | 6 +- Makefile | 24 ++-- platform/nodejs/README.md | 2 +- platform/nodejs/index.js | 14 +- platform/nodejs/test.js | 160 --------------------- platform/{nodejs => npm}/.eslintrc.json | 0 platform/{nodejs => npm}/package-lock.json | 0 platform/{nodejs => npm}/package.json | 0 platform/npm/test.js | 46 ++++++ platform/{nodejs => npm}/tests/snfe.js | 0 tools/make-dig.sh | 17 +++ tools/make-nodejs.sh | 49 +------ tools/make-npm.sh | 60 ++++++++ 13 files changed, 152 insertions(+), 226 deletions(-) delete mode 100644 platform/nodejs/test.js rename platform/{nodejs => npm}/.eslintrc.json (100%) rename platform/{nodejs => npm}/package-lock.json (100%) rename platform/{nodejs => npm}/package.json (100%) create mode 100644 platform/npm/test.js rename platform/{nodejs => npm}/tests/snfe.js (100%) create mode 100755 tools/make-dig.sh create mode 100755 tools/make-npm.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f20612527..e44e460c0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -39,7 +39,7 @@ jobs: tools/make-chromium.sh ${{ steps.release_info.outputs.VERSION }} tools/make-firefox.sh ${{ steps.release_info.outputs.VERSION }} tools/make-thunderbird.sh ${{ steps.release_info.outputs.VERSION }} - tools/make-nodejs.sh ${{ steps.release_info.outputs.VERSION }} + tools/make-npm.sh ${{ steps.release_info.outputs.VERSION }} - name: Upload Chromium package uses: actions/upload-release-asset@v1 env: @@ -73,6 +73,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: dist/build/uBlock0_${{ steps.release_info.outputs.VERSION }}.nodejs.tgz - asset_name: uBlock0_${{ steps.release_info.outputs.VERSION }}.nodejs.tgz + asset_path: dist/build/uBlock0_${{ steps.release_info.outputs.VERSION }}.npm.tgz + asset_name: uBlock0_${{ steps.release_info.outputs.VERSION }}.npm.tgz asset_content_type: application/octet-stream diff --git a/Makefile b/Makefile index 1e910700f..3fedbf04a 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all clean test lint chromium firefox nodejs +.PHONY: all clean test lint chromium firefox npm sources := $(wildcard src/* src/*/* src/*/*/* src/*/*/*/*) platform := $(wildcard platform/* platform/*/*) @@ -7,7 +7,7 @@ assets := $(wildcard submodules/uAssets/* \ submodules/uAssets/*/*/* \ submodules/uAssets/*/*/*/*) -all: chromium firefox nodejs +all: chromium firefox npm dist/build/uBlock0.chromium: tools/make-chromium.sh $(sources) $(platform) $(assets) tools/make-chromium.sh @@ -21,17 +21,23 @@ dist/build/uBlock0.firefox: tools/make-firefox.sh $(sources) $(platform) $(asset # Build the extension for Firefox. firefox: dist/build/uBlock0.firefox -dist/build/uBlock0.nodejs: tools/make-nodejs.sh $(sources) $(platform) $(assets) - tools/make-nodejs.sh +dist/build/uBlock0.npm: tools/make-nodejs.sh $(sources) $(platform) $(assets) + tools/make-npm.sh # Build the Node.js package. -nodejs: dist/build/uBlock0.nodejs +npm: dist/build/uBlock0.npm -lint: nodejs - cd dist/build/uBlock0.nodejs && npm install && npm run lint +lint: npm + cd dist/build/uBlock0.npm && npm install && npm run lint -test: nodejs - cd dist/build/uBlock0.nodejs && npm install && npm run test +test: npm + cd dist/build/uBlock0.npm && npm install && npm run test + +dist/build/uBlock0.dig: tools/make-nodejs.sh $(sources) $(platform) $(assets) + tools/make-dig.sh + +dig: dist/build/uBlock0.dig + cd dist/build/uBlock0.dig && npm install && npm run test # Update submodules. update-submodules: diff --git a/platform/nodejs/README.md b/platform/nodejs/README.md index 41fb6adfd..79a6a398f 100644 --- a/platform/nodejs/README.md +++ b/platform/nodejs/README.md @@ -102,7 +102,7 @@ and use this capability now if you figure out the details. You can directly use specific APIs exposed by this package, here are some of them, which are used internally by uBO's SNFE. -### `HNTrieContainer` +### HNTrieContainer A well optimised [compressed trie](https://en.wikipedia.org/wiki/Trie#Compressing_tries) container specialized to specifically store and lookup hostnames. diff --git a/platform/nodejs/index.js b/platform/nodejs/index.js index 3267f5507..e7282f8f8 100644 --- a/platform/nodejs/index.js +++ b/platform/nodejs/index.js @@ -95,8 +95,7 @@ function pslInit(raw) { return globals.publicSuffixList; } - const require = createRequire(import.meta.url); // jshint ignore:line - raw = require('./data/effective_tld_names.json'); + raw = readFileSync(resolve(__dirname, './assets/thirdparties/publicsuffix.org/list/effective_tld_names.dat'), 'utf8') if ( typeof raw !== 'string' || raw.trim() === '' ) { console.error('Unable to populate public suffix list'); return; @@ -203,14 +202,14 @@ class MockStorage { } const fctx = new FilteringContext(); -let snfeInstance = null; +let snfeProxyInstance = null; class StaticNetFilteringEngine { constructor() { - if ( snfeInstance !== null ) { + if ( snfeProxyInstance !== null ) { throw new Error('Only a single instance is supported.'); } - snfeInstance = this; + snfeProxyInstance = this; } async useLists(lists) { @@ -253,6 +252,11 @@ class StaticNetFilteringEngine { return instance; } + + static release() { + useLists([]); + snfeProxyInstance = null; + } } /******************************************************************************/ diff --git a/platform/nodejs/test.js b/platform/nodejs/test.js deleted file mode 100644 index 15e3db0c4..000000000 --- a/platform/nodejs/test.js +++ /dev/null @@ -1,160 +0,0 @@ -/******************************************************************************* - - uBlock Origin - a browser extension to block requests. - Copyright (C) 2014-present Raymond Hill - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see {http://www.gnu.org/licenses/}. - - Home: https://github.com/gorhill/uBlock -*/ - -/* eslint-disable-next-line no-redeclare */ -/* globals process */ - -'use strict'; - -/******************************************************************************/ - -import { strict as assert } from 'assert'; -import { spawn } from "child_process"; -import { createRequire } from 'module'; -import { promisify } from 'util'; - -import { - enableWASM, - StaticNetFilteringEngine, -} from './index.js'; - -import HNTrieContainer from './js/hntrie.js'; - -/******************************************************************************/ - -function fetch(listName) { - return new Promise(resolve => { - const require = createRequire(import.meta.url); // jshint ignore:line - resolve(require(`./data/${listName}.json`)); - }); -} - -function testSNFE(engine) { - let result = 0; - - // Tests - // Not blocked - result = engine.matchRequest({ - originURL: 'https://www.bloomberg.com/', - url: 'https://www.bloomberg.com/tophat/assets/v2.6.1/that.css', - type: 'stylesheet' - }); - if ( result !== 0 ) { - console.log(engine.toLogData()); - } - - // Blocked - result = engine.matchRequest({ - originURL: 'https://www.bloomberg.com/', - url: 'https://securepubads.g.doubleclick.net/tag/js/gpt.js', - type: 'script' - }); - if ( result !== 0 ) { - console.log(engine.toLogData()); - } - - // Unblocked - result = engine.matchRequest({ - originURL: 'https://www.bloomberg.com/', - url: 'https://sourcepointcmp.bloomberg.com/ccpa.js', - type: 'script' - }); - if ( result !== 0 ) { - console.log(engine.toLogData()); - } -} - -async function doSNFE() { - const engine = await StaticNetFilteringEngine.create(); - - await engine.useLists([ - fetch('easylist').then(raw => ({ name: 'easylist', raw })), - fetch('easyprivacy').then(raw => ({ name: 'easyprivacy', raw })), - ]); - - testSNFE(engine); - - const serialized = await engine.serialize(); - engine.useLists([]); - - assert.notDeepEqual(await engine.serialize(), serialized); - - testSNFE(engine); - - await engine.deserialize(serialized); - - assert.deepEqual(await engine.serialize(), serialized); - - testSNFE(engine); -} - -async function doHNTrie() { - const trieContainer = new HNTrieContainer(); - - const aTrie = trieContainer.createOne(); - aTrie.add('example.org'); - aTrie.add('example.com'); - - const anotherTrie = trieContainer.createOne(); - anotherTrie.add('foo.invalid'); - anotherTrie.add('bar.invalid'); - - // matches() return the position at which the match starts, or -1 when - // there is no match. - - // Matches: return 4 - console.log("aTrie.matches('www.example.org')", aTrie.matches('www.example.org')); - - // Does not match: return -1 - console.log("aTrie.matches('www.foo.invalid')", aTrie.matches('www.foo.invalid')); - - // Does not match: return -1 - console.log("anotherTrie.matches('www.example.org')", anotherTrie.matches('www.example.org')); - - // Matches: return 0 - console.log("anotherTrie.matches('foo.invalid')", anotherTrie.matches('foo.invalid')); -} - -async function spawnMocha() { - await promisify(spawn)('mocha', [ '--experimental-vm-modules', '--no-warnings', 'tests' ], { stdio: [ 'inherit', 'inherit', 'inherit' ] }); -} - -async function main() { - try { - const result = await enableWASM(); - if ( result !== true ) { - console.log('Failed to enable all WASM code paths'); - } - } catch(ex) { - console.log(ex); - } - - await doSNFE(); - await doHNTrie(); - - if ( process.argv[2] === '--mocha' ) { - await spawnMocha(); - } -} - -main(); - -/******************************************************************************/ diff --git a/platform/nodejs/.eslintrc.json b/platform/npm/.eslintrc.json similarity index 100% rename from platform/nodejs/.eslintrc.json rename to platform/npm/.eslintrc.json diff --git a/platform/nodejs/package-lock.json b/platform/npm/package-lock.json similarity index 100% rename from platform/nodejs/package-lock.json rename to platform/npm/package-lock.json diff --git a/platform/nodejs/package.json b/platform/npm/package.json similarity index 100% rename from platform/nodejs/package.json rename to platform/npm/package.json diff --git a/platform/npm/test.js b/platform/npm/test.js new file mode 100644 index 000000000..5fb73cfb4 --- /dev/null +++ b/platform/npm/test.js @@ -0,0 +1,46 @@ +/******************************************************************************* + + uBlock Origin - a browser extension to block requests. + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* eslint-disable-next-line no-redeclare */ +/* globals process */ + +'use strict'; + +/******************************************************************************/ + +import { spawn } from "child_process"; +import { promisify } from 'util'; + +/******************************************************************************/ + +async function spawnMocha() { + await promisify(spawn)('mocha', [ '--experimental-vm-modules', '--no-warnings', 'tests' ], { stdio: [ 'inherit', 'inherit', 'inherit' ] }); +} + +async function main() { + if ( process.argv[2] === '--mocha' ) { + await spawnMocha(); + } +} + +main(); + +/******************************************************************************/ diff --git a/platform/nodejs/tests/snfe.js b/platform/npm/tests/snfe.js similarity index 100% rename from platform/nodejs/tests/snfe.js rename to platform/npm/tests/snfe.js diff --git a/tools/make-dig.sh b/tools/make-dig.sh new file mode 100755 index 000000000..2ced63c9f --- /dev/null +++ b/tools/make-dig.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# +# This script assumes a linux environment + +set -e + +DES="dist/build/uBlock0.dig" + +./tools/make-nodejs.sh $DES +./tools/make-assets.sh $DES + +cp -R platform/dig/* $DES/ + +cd $DES +npm run build + +echo "*** uBlock0.dig: Package done." diff --git a/tools/make-nodejs.sh b/tools/make-nodejs.sh index 58462a06e..ad6a92f8c 100755 --- a/tools/make-nodejs.sh +++ b/tools/make-nodejs.sh @@ -4,18 +4,7 @@ set -e -DES=dist/build/uBlock0.nodejs - -TMPDIR=tmp -mkdir -p $TMPDIR - -# Save existing npm dependencies if present so that we do not have to fetch -# them all again -if [ -d "$DES/node_modules" ]; then - mv "$DES/node_modules" "$TMPDIR/node_modules" -fi - -rm -rf $DES +DES=$1 mkdir -p $DES/js cp src/js/base64-custom.js $DES/js @@ -47,42 +36,6 @@ node -pe "JSON.stringify(Array.from(fs.readFileSync('src/js/wasm/biditrie.wasm') node -pe "JSON.stringify(Array.from(fs.readFileSync('src/lib/publicsuffixlist/wasm/publicsuffixlist.wasm')))" \ > $DES/lib/publicsuffixlist/wasm/publicsuffixlist.wasm.json -git submodule update --depth 1 --init -UASSETS=submodules/uAssets - -# https://github.com/uBlockOrigin/uBlock-issues/issues/1664#issuecomment-888332409 -THIRDPARTY=$UASSETS/thirdparties/publicsuffix.org -mkdir -p $DES/data -node -pe "JSON.stringify(fs.readFileSync('$THIRDPARTY/list/effective_tld_names.dat', 'utf8'))" \ - > $DES/data/effective_tld_names.json -THIRDPARTY=$UASSETS/thirdparties/easylist-downloads.adblockplus.org -node -pe "JSON.stringify(fs.readFileSync('$THIRDPARTY/easylist.txt', 'utf8'))" \ - > $DES/data/easylist.json -node -pe "JSON.stringify(fs.readFileSync('$THIRDPARTY/easyprivacy.txt', 'utf8'))" \ - > $DES/data/easyprivacy.json - -cp platform/nodejs/.*.json $DES/ cp platform/nodejs/*.js $DES/ -cp platform/nodejs/*.json $DES/ cp platform/nodejs/README.md $DES/ cp LICENSE.txt $DES/ -cp -R platform/nodejs/tests $DES/ - -cd $DES -npm run build -tarballname=$(npm pack 2> /dev/null) -if [ "$1" ]; then - echo "*** uBlock0.nodejs: Creating versioned package..." - mv $tarballname ../uBlock0_"$1".nodejs.tgz -else - echo "*** uBlock0.nodejs: Creating plain package..." - mv $tarballname ../uBlock0.nodejs.tgz -fi -cd - - -# Restore saved npm dependencies -if [ -d "$TMPDIR/node_modules" ]; then - mv "$TMPDIR/node_modules" "$DES/node_modules" -fi - -echo "*** uBlock0.nodejs: Package done." diff --git a/tools/make-npm.sh b/tools/make-npm.sh new file mode 100755 index 000000000..c83479bc6 --- /dev/null +++ b/tools/make-npm.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# +# This script assumes a linux environment + +set -e + +DES="dist/build/uBlock0.npm" + +TMPDIR=tmp +mkdir -p $TMPDIR + +# Save existing npm dependencies if present so that we do not have to fetch +# them all again +if [ -d "$DES/node_modules" ]; then + mv "$DES/node_modules" "$TMPDIR/node_modules" +fi + +rm -rf $DES + +./tools/make-nodejs.sh $DES + +UASSETS=submodules/uAssets + +# https://github.com/uBlockOrigin/uBlock-issues/issues/1664#issuecomment-888332409 +mkdir -p $DES/assets/thirdparties/publicsuffix.org/list +THIRDPARTY=$UASSETS/thirdparties/publicsuffix.org +node -pe "JSON.stringify(fs.readFileSync('$THIRDPARTY/list/effective_tld_names.dat', 'utf8'))" \ + > $DES/assets/thirdparties/publicsuffix.org/list/effective_tld_names.dat + +mkdir -p $DES/data +THIRDPARTY=$UASSETS/thirdparties/easylist-downloads.adblockplus.org +node -pe "JSON.stringify(fs.readFileSync('$THIRDPARTY/easylist.txt', 'utf8'))" \ + > $DES/data/easylist.json +node -pe "JSON.stringify(fs.readFileSync('$THIRDPARTY/easyprivacy.txt', 'utf8'))" \ + > $DES/data/easyprivacy.json + +# Target-specific +cp platform/npm/*.json $DES/ +cp platform/npm/.*.json $DES/ +cp platform/npm/*.js $DES/ +cp -R platform/npm/tests $DES/ + +cd $DES +npm run build +tarballname=$(npm pack 2> /dev/null) +if [ "$1" ]; then + echo "*** uBlock0.npm: Creating versioned package..." + mv $tarballname ../uBlock0_$1.npm.tgz +else + echo "*** uBlock0.npm: Creating plain package..." + mv $tarballname ../uBlock0.npm.tgz +fi +cd - + +# Restore saved npm dependencies +if [ -d "$TMPDIR/node_modules" ]; then + mv "$TMPDIR/node_modules" "$DES/node_modules" +fi + +echo "*** uBlock0.npm: Package done."