/******************************************************************************* uBlock Origin - a browser extension to block requests. Copyright (C) 2014-present Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see {http://www.gnu.org/licenses/}. Home: https://github.com/gorhill/uBlock */ /* global punycode, publicSuffixList */ 'use strict'; /******************************************************************************/ µBlock.getBytesInUse = async function() { const promises = []; let bytesInUse; // Not all platforms implement this method. promises.push( vAPI.storage.getBytesInUse instanceof Function ? vAPI.storage.getBytesInUse(null) : undefined ); if ( navigator.storage instanceof Object && navigator.storage.estimate instanceof Function ) { promises.push(navigator.storage.estimate()); } const results = await Promise.all(promises); const processCount = count => { if ( typeof count !== 'number' ) { return; } if ( bytesInUse === undefined ) { bytesInUse = 0; } bytesInUse += count; return bytesInUse; }; processCount(results[0]); if ( results.length > 1 && results[1] instanceof Object ) { processCount(results[1].usage); } µBlock.storageUsed = bytesInUse; return bytesInUse; }; /******************************************************************************/ µBlock.saveLocalSettings = (( ) => { const saveAfter = 4 * 60 * 1000; const onTimeout = ( ) => { const µb = µBlock; if ( µb.localSettingsLastModified > µb.localSettingsLastSaved ) { µb.saveLocalSettings(); } vAPI.setTimeout(onTimeout, saveAfter); }; vAPI.setTimeout(onTimeout, saveAfter); return function() { this.localSettingsLastSaved = Date.now(); return vAPI.storage.set(this.localSettings); }; })(); /******************************************************************************/ µBlock.saveUserSettings = function() { vAPI.storage.set(this.userSettings); }; /******************************************************************************/ µBlock.loadHiddenSettings = async function() { const bin = await vAPI.storage.get('hiddenSettings'); if ( bin instanceof Object === false ) { return; } const hs = bin.hiddenSettings; if ( hs instanceof Object ) { const hsDefault = this.hiddenSettingsDefault; for ( const key in hsDefault ) { if ( hsDefault.hasOwnProperty(key) && hs.hasOwnProperty(key) && typeof hs[key] === typeof hsDefault[key] ) { this.hiddenSettings[key] = hs[key]; } } if ( typeof this.hiddenSettings.suspendTabsUntilReady === 'boolean' ) { this.hiddenSettings.suspendTabsUntilReady = this.hiddenSettings.suspendTabsUntilReady ? 'yes' : 'unset'; } } self.log.verbosity = this.hiddenSettings.consoleLogLevel; this.fireDOMEvent('hiddenSettingsChanged'); }; // Note: Save only the settings which values differ from the default ones. // This way the new default values in the future will properly apply for those // which were not modified by the user. µBlock.saveHiddenSettings = function() { const bin = { hiddenSettings: {} }; for ( const prop in this.hiddenSettings ) { if ( this.hiddenSettings.hasOwnProperty(prop) && this.hiddenSettings[prop] !== this.hiddenSettingsDefault[prop] ) { bin.hiddenSettings[prop] = this.hiddenSettings[prop]; } } vAPI.storage.set(bin); this.saveImmediateHiddenSettings(); self.log.verbosity = this.hiddenSettings.consoleLogLevel; }; /******************************************************************************/ µBlock.hiddenSettingsFromString = function(raw) { const out = Object.assign({}, this.hiddenSettingsDefault); const lineIter = new this.LineIterator(raw); while ( lineIter.eot() === false ) { const line = lineIter.next(); const matches = /^\s*(\S+)\s+(.+)$/.exec(line); if ( matches === null || matches.length !== 3 ) { continue; } const name = matches[1]; if ( out.hasOwnProperty(name) === false ) { continue; } const value = matches[2]; switch ( typeof out[name] ) { case 'boolean': if ( value === 'true' ) { out[name] = true; } else if ( value === 'false' ) { out[name] = false; } break; case 'string': out[name] = value; break; case 'number': out[name] = parseInt(value, 10); if ( isNaN(out[name]) ) { out[name] = this.hiddenSettingsDefault[name]; } break; default: break; } } return out; }; µBlock.stringFromHiddenSettings = function() { const out = []; for ( const key of Object.keys(this.hiddenSettings).sort() ) { out.push(key + ' ' + this.hiddenSettings[key]); } return out.join('\n'); }; /******************************************************************************/ // These settings must be available immediately on startup, without delay // through the vAPI.localStorage. Add/remove settings as needed. µBlock.saveImmediateHiddenSettings = function() { const props = [ 'consoleLogLevel', 'disableWebAssembly', 'suspendTabsUntilReady', ]; const toSave = {}; for ( const prop of props ) { if ( this.hiddenSettings[prop] !== this.hiddenSettingsDefault[prop] ) { toSave[prop] = this.hiddenSettings[prop]; } } if ( Object.keys(toSave).length !== 0 ) { vAPI.localStorage.setItem( 'immediateHiddenSettings', JSON.stringify(toSave) ); } else { vAPI.localStorage.removeItem('immediateHiddenSettings'); } }; /******************************************************************************/ µBlock.savePermanentFirewallRules = function() { vAPI.storage.set({ dynamicFilteringString: this.permanentFirewall.toString() }); }; /******************************************************************************/ µBlock.savePermanentURLFilteringRules = function() { vAPI.storage.set({ urlFilteringString: this.permanentURLFiltering.toString() }); }; /******************************************************************************/ µBlock.saveHostnameSwitches = function() { vAPI.storage.set({ hostnameSwitchesString: this.permanentSwitches.toString() }); }; /******************************************************************************/ µBlock.saveWhitelist = function() { vAPI.storage.set({ netWhitelist: this.arrayFromWhitelist(this.netWhitelist) }); this.netWhitelistModifyTime = Date.now(); }; /******************************************************************************* TODO(seamless migration): The code related to 'remoteBlacklist' can be removed when I am confident all users have moved to a version of uBO which no longer depends on the property 'remoteBlacklists, i.e. v1.11 and beyond. **/ µBlock.loadSelectedFilterLists = async function() { const bin = await vAPI.storage.get('selectedFilterLists'); if ( bin instanceof Object && Array.isArray(bin.selectedFilterLists) ) { this.selectedFilterLists = bin.selectedFilterLists; return; } // https://github.com/gorhill/uBlock/issues/747 // Select default filter lists if first-time launch. const lists = await this.assets.metadata(); this.saveSelectedFilterLists(this.autoSelectRegionalFilterLists(lists)); }; µBlock.saveSelectedFilterLists = function(newKeys, append = false) { const oldKeys = this.selectedFilterLists.slice(); if ( append ) { newKeys = newKeys.concat(oldKeys); } const newSet = new Set(newKeys); // Purge unused filter lists from cache. for ( const oldKey of oldKeys ) { if ( newSet.has(oldKey) === false ) { this.removeFilterList(oldKey); } } newKeys = Array.from(newSet); this.selectedFilterLists = newKeys; return vAPI.storage.set({ selectedFilterLists: newKeys }); }; /******************************************************************************/ µBlock.applyFilterListSelection = function(details) { let selectedListKeySet = new Set(this.selectedFilterLists); let externalLists = this.userSettings.externalLists; // Filter lists to select if ( Array.isArray(details.toSelect) ) { if ( details.merge ) { for ( let i = 0, n = details.toSelect.length; i < n; i++ ) { selectedListKeySet.add(details.toSelect[i]); } } else { selectedListKeySet = new Set(details.toSelect); } } // Imported filter lists to remove if ( Array.isArray(details.toRemove) ) { const removeURLFromHaystack = (haystack, needle) => { return haystack.replace( new RegExp( '(^|\\n)' + this.escapeRegex(needle) + '(\\n|$)', 'g'), '\n' ).trim(); }; for ( let i = 0, n = details.toRemove.length; i < n; i++ ) { const assetKey = details.toRemove[i]; selectedListKeySet.delete(assetKey); externalLists = removeURLFromHaystack(externalLists, assetKey); this.removeFilterList(assetKey); } } // Filter lists to import if ( typeof details.toImport === 'string' ) { // https://github.com/gorhill/uBlock/issues/1181 // Try mapping the URL of an imported filter list to the assetKey // of an existing stock list. const assetKeyFromURL = url => { const needle = url.replace(/^https?:/, ''); const assets = this.availableFilterLists; for ( const assetKey in assets ) { const asset = assets[assetKey]; if ( asset.content !== 'filters' ) { continue; } if ( typeof asset.contentURL === 'string' ) { if ( asset.contentURL.endsWith(needle) ) { return assetKey; } continue; } if ( Array.isArray(asset.contentURL) === false ) { continue; } for ( let i = 0, n = asset.contentURL.length; i < n; i++ ) { if ( asset.contentURL[i].endsWith(needle) ) { return assetKey; } } } return url; }; const importedSet = new Set(this.listKeysFromCustomFilterLists(externalLists)); const toImportSet = new Set(this.listKeysFromCustomFilterLists(details.toImport)); for ( const urlKey of toImportSet ) { if ( importedSet.has(urlKey) ) { continue; } const assetKey = assetKeyFromURL(urlKey); if ( assetKey === urlKey ) { importedSet.add(urlKey); } selectedListKeySet.add(assetKey); } externalLists = Array.from(importedSet).sort().join('\n'); } const result = Array.from(selectedListKeySet); if ( externalLists !== this.userSettings.externalLists ) { this.userSettings.externalLists = externalLists; vAPI.storage.set({ externalLists: externalLists }); } this.saveSelectedFilterLists(result); }; /******************************************************************************/ µBlock.listKeysFromCustomFilterLists = function(raw) { const out = new Set(); const reIgnore = /^[!#]/; const reValid = /^[a-z-]+:\/\/\S+/; const lineIter = new this.LineIterator(raw); while ( lineIter.eot() === false ) { const location = lineIter.next().trim(); if ( reIgnore.test(location) || !reValid.test(location) ) { continue; } out.add(location); } return Array.from(out); }; /******************************************************************************/ µBlock.saveUserFilters = function(content) { // https://github.com/gorhill/uBlock/issues/1022 // Be sure to end with an empty line. content = content.trim(); if ( content !== '' ) { content += '\n'; } this.removeCompiledFilterList(this.userFiltersPath); return this.assets.put(this.userFiltersPath, content); }; µBlock.loadUserFilters = function() { return this.assets.get(this.userFiltersPath); }; µBlock.appendUserFilters = async function(filters, options) { filters = filters.trim(); if ( filters.length === 0 ) { return; } // https://github.com/uBlockOrigin/uBlock-issues/issues/372 // Auto comment using user-defined template. let comment = ''; if ( options instanceof Object && options.autoComment === true && this.hiddenSettings.autoCommentFilterTemplate.indexOf('{{') !== -1 ) { const d = new Date(); comment = '! ' + this.hiddenSettings.autoCommentFilterTemplate .replace('{{date}}', d.toLocaleDateString()) .replace('{{time}}', d.toLocaleTimeString()) .replace('{{origin}}', options.origin); } const details = await this.loadUserFilters(); if ( details.error ) { return; } // The comment, if any, will be applied if and only if it is different // from the last comment found in the user filter list. if ( comment !== '' ) { const pos = details.content.lastIndexOf(comment); if ( pos === -1 || details.content.indexOf('\n!', pos + 1) !== -1 ) { filters = '\n' + comment + '\n' + filters; } } // https://github.com/chrisaljoudi/uBlock/issues/976 // If we reached this point, the filter quite probably needs to be // added for sure: do not try to be too smart, trying to avoid // duplicates at this point may lead to more issues. await this.saveUserFilters(details.content.trim() + '\n' + filters); const compiledFilters = this.compileFilters( filters, { assetKey: this.userFiltersPath } ); const snfe = this.staticNetFilteringEngine; const cfe = this.cosmeticFilteringEngine; const acceptedCount = snfe.acceptedCount + cfe.acceptedCount; const discardedCount = snfe.discardedCount + cfe.discardedCount; this.applyCompiledFilters(compiledFilters, true); const entry = this.availableFilterLists[this.userFiltersPath]; const deltaEntryCount = snfe.acceptedCount + cfe.acceptedCount - acceptedCount; const deltaEntryUsedCount = deltaEntryCount - (snfe.discardedCount + cfe.discardedCount - discardedCount); entry.entryCount += deltaEntryCount; entry.entryUsedCount += deltaEntryUsedCount; vAPI.storage.set({ 'availableFilterLists': this.availableFilterLists }); this.staticNetFilteringEngine.freeze(); this.redirectEngine.freeze(); this.staticExtFilteringEngine.freeze(); this.selfieManager.destroy(); // https://www.reddit.com/r/uBlockOrigin/comments/cj7g7m/ // https://www.reddit.com/r/uBlockOrigin/comments/cnq0bi/ if ( options.killCache ) { browser.webRequest.handlerBehaviorChanged(); } }; µBlock.createUserFilters = function(details) { this.appendUserFilters(details.filters, details); // https://github.com/gorhill/uBlock/issues/1786 this.cosmeticFilteringEngine.removeFromSelectorCache(details.pageDomain); }; /******************************************************************************/ µBlock.autoSelectRegionalFilterLists = function(lists) { const selectedListKeys = [ this.userFiltersPath ]; for ( const key in lists ) { if ( lists.hasOwnProperty(key) === false ) { continue; } const list = lists[key]; if ( list.off !== true ) { selectedListKeys.push(key); continue; } if ( this.listMatchesEnvironment(list) ) { selectedListKeys.push(key); list.off = false; } } return selectedListKeys; }; /******************************************************************************/ µBlock.getAvailableLists = async function() { let oldAvailableLists = {}, newAvailableLists = {}; // User filter list. newAvailableLists[this.userFiltersPath] = { group: 'user', title: vAPI.i18n('1pPageName') }; // Custom filter lists. const importedListKeys = this.listKeysFromCustomFilterLists( this.userSettings.externalLists ); for ( const listKey of importedListKeys ) { const entry = { content: 'filters', contentURL: listKey, external: true, group: 'custom', submitter: 'user', title: '' }; newAvailableLists[listKey] = entry; this.assets.registerAssetSource(listKey, entry); } // Convert a no longer existing stock list into an imported list. const customListFromStockList = assetKey => { const oldEntry = oldAvailableLists[assetKey]; if ( oldEntry === undefined || oldEntry.off === true ) { return; } let listURL = oldEntry.contentURL; if ( Array.isArray(listURL) ) { listURL = listURL[0]; } const newEntry = { content: 'filters', contentURL: listURL, external: true, group: 'custom', submitter: 'user', title: oldEntry.title || '' }; newAvailableLists[listURL] = newEntry; this.assets.registerAssetSource(listURL, newEntry); importedListKeys.push(listURL); this.userSettings.externalLists += '\n' + listURL; this.userSettings.externalLists = this.userSettings.externalLists.trim(); vAPI.storage.set({ externalLists: this.userSettings.externalLists }); this.saveSelectedFilterLists([ listURL ], true); }; // Load previously saved available lists -- these contains data // computed at run-time, we will reuse this data if possible. const [ bin, entries ] = await Promise.all([ vAPI.storage.get('availableFilterLists'), this.assets.metadata(), ]); oldAvailableLists = bin && bin.availableFilterLists || {}; for ( const assetKey in entries ) { if ( entries.hasOwnProperty(assetKey) === false ) { continue; } const entry = entries[assetKey]; if ( entry.content !== 'filters' ) { continue; } newAvailableLists[assetKey] = Object.assign({}, entry); } // Load set of currently selected filter lists. const listKeySet = new Set(this.selectedFilterLists); for ( const listKey in newAvailableLists ) { if ( newAvailableLists.hasOwnProperty(listKey) ) { newAvailableLists[listKey].off = !listKeySet.has(listKey); } } //finalize(); // Final steps: // - reuse existing list metadata if any; // - unregister unreferenced imported filter lists if any. // Reuse existing metadata. for ( const assetKey in oldAvailableLists ) { const oldEntry = oldAvailableLists[assetKey]; const newEntry = newAvailableLists[assetKey]; // List no longer exists. If a stock list, try to convert to // imported list if it was selected. if ( newEntry === undefined ) { this.removeFilterList(assetKey); if ( assetKey.indexOf('://') === -1 ) { customListFromStockList(assetKey); } continue; } if ( oldEntry.entryCount !== undefined ) { newEntry.entryCount = oldEntry.entryCount; } if ( oldEntry.entryUsedCount !== undefined ) { newEntry.entryUsedCount = oldEntry.entryUsedCount; } // This may happen if the list name was pulled from the list // content. // https://github.com/chrisaljoudi/uBlock/issues/982 // There is no guarantee the title was successfully extracted from // the list content. if ( newEntry.title === '' && typeof oldEntry.title === 'string' && oldEntry.title !== '' ) { newEntry.title = oldEntry.title; } } // Remove unreferenced imported filter lists. for ( const assetKey in newAvailableLists ) { const newEntry = newAvailableLists[assetKey]; if ( newEntry.submitter !== 'user' ) { continue; } if ( importedListKeys.indexOf(assetKey) !== -1 ) { continue; } delete newAvailableLists[assetKey]; this.assets.unregisterAssetSource(assetKey); this.removeFilterList(assetKey); } return newAvailableLists; }; /******************************************************************************/ µBlock.loadFilterLists = (( ) => { const loadedListKeys = []; let loadingPromise; const t0 = Date.now(); const onDone = function() { log.info(`loadFilterLists() took ${Date.now()-t0} ms`); this.staticNetFilteringEngine.freeze(); this.staticExtFilteringEngine.freeze(); this.redirectEngine.freeze(); vAPI.net.unsuspend(); vAPI.storage.set({ 'availableFilterLists': this.availableFilterLists }); vAPI.messaging.broadcast({ what: 'staticFilteringDataChanged', parseCosmeticFilters: this.userSettings.parseAllABPHideFilters, ignoreGenericCosmeticFilters: this.userSettings.ignoreGenericCosmeticFilters, listKeys: loadedListKeys }); this.selfieManager.destroy(); this.lz4Codec.relinquish(); this.compiledFormatChanged = false; loadingPromise = undefined; }; const applyCompiledFilters = function(assetKey, compiled) { const snfe = this.staticNetFilteringEngine; const sxfe = this.staticExtFilteringEngine; let acceptedCount = snfe.acceptedCount + sxfe.acceptedCount, discardedCount = snfe.discardedCount + sxfe.discardedCount; this.applyCompiledFilters(compiled, assetKey === this.userFiltersPath); if ( this.availableFilterLists.hasOwnProperty(assetKey) ) { const entry = this.availableFilterLists[assetKey]; entry.entryCount = snfe.acceptedCount + sxfe.acceptedCount - acceptedCount; entry.entryUsedCount = entry.entryCount - (snfe.discardedCount + sxfe.discardedCount - discardedCount); } loadedListKeys.push(assetKey); }; const onFilterListsReady = function(lists) { this.availableFilterLists = lists; vAPI.net.suspend(); this.redirectEngine.reset(); this.staticExtFilteringEngine.reset(); this.staticNetFilteringEngine.reset(); this.selfieManager.destroy(); this.staticFilteringReverseLookup.resetLists(); // We need to build a complete list of assets to pull first: this is // because it *may* happens that some load operations are synchronous: // This happens for assets which do not exist, ot assets with no // content. const toLoad = []; for ( const assetKey in lists ) { if ( lists.hasOwnProperty(assetKey) === false ) { continue; } if ( lists[assetKey].off ) { continue; } toLoad.push( this.getCompiledFilterList(assetKey).then(details => { applyCompiledFilters.call( this, details.assetKey, details.content ); }) ); } return Promise.all(toLoad); }; return function() { if ( loadingPromise instanceof Promise === false ) { loadedListKeys.length = 0; loadingPromise = Promise.all([ this.getAvailableLists().then(lists => onFilterListsReady.call(this, lists) ), this.loadRedirectResources(), ]).then(( ) => { onDone.call(this); }); } return loadingPromise; }; })(); /******************************************************************************/ µBlock.getCompiledFilterList = async function(assetKey) { const compiledPath = 'compiled/' + assetKey; if ( this.compiledFormatChanged === false ) { let compiledDetails = await this.assets.get(compiledPath); if ( compiledDetails.content !== '' ) { compiledDetails.assetKey = assetKey; return compiledDetails; } } const rawDetails = await this.assets.get(assetKey); // Compiling an empty string results in an empty string. if ( rawDetails.content === '' ) { rawDetails.assetKey = assetKey; return rawDetails; } this.extractFilterListMetadata(assetKey, rawDetails.content); // Fetching the raw content may cause the compiled content to be // generated somewhere else in uBO, hence we try one last time to // fetch the compiled content in case it has become available. let compiledDetails = await this.assets.get(compiledPath); if ( compiledDetails.content === '' ) { compiledDetails.content = this.compileFilters( rawDetails.content, { assetKey: assetKey } ); this.assets.put(compiledPath, compiledDetails.content); } compiledDetails.assetKey = assetKey; return compiledDetails; }; /******************************************************************************/ // https://github.com/gorhill/uBlock/issues/3406 // Lower minimum update period to 1 day. µBlock.extractFilterListMetadata = function(assetKey, raw) { const listEntry = this.availableFilterLists[assetKey]; if ( listEntry === undefined ) { return; } // Metadata expected to be found at the top of content. const head = raw.slice(0, 1024); // https://github.com/gorhill/uBlock/issues/313 // Always try to fetch the name if this is an external filter list. if ( listEntry.title === '' || listEntry.group === 'custom' ) { const matches = head.match(/(?:^|\n)(?:!|# )[\t ]*Title[\t ]*:([^\n]+)/i); if ( matches !== null ) { // https://bugs.chromium.org/p/v8/issues/detail?id=2869 // orphanizeString is to work around String.slice() // potentially causing the whole raw filter list to be held in // memory just because we cut out the title as a substring. listEntry.title = this.orphanizeString(matches[1].trim()); } } // Extract update frequency information const matches = head.match(/(?:^|\n)(?:!|# )[\t ]*Expires[\t ]*:[\t ]*(\d+)[\t ]*(h)?/i); if ( matches !== null ) { let v = Math.max(parseInt(matches[1], 10), 1); if ( matches[2] !== undefined ) { v = Math.ceil(v / 24); } if ( v !== listEntry.updateAfter ) { this.assets.registerAssetSource(assetKey, { updateAfter: v }); } } }; /******************************************************************************/ µBlock.removeCompiledFilterList = function(assetKey) { this.assets.remove('compiled/' + assetKey); }; µBlock.removeFilterList = function(assetKey) { this.removeCompiledFilterList(assetKey); this.assets.remove(assetKey); }; /******************************************************************************/ µBlock.compileFilters = function(rawText, details) { const writer = new this.CompiledLineIO.Writer(); // Populate the writer with information potentially useful to the // client compilers. if ( details ) { if ( details.assetKey ) { writer.properties.set('assetKey', details.assetKey); } } // Useful references: // https://adblockplus.org/en/filter-cheatsheet // https://adblockplus.org/en/filters const staticNetFilteringEngine = this.staticNetFilteringEngine; const staticExtFilteringEngine = this.staticExtFilteringEngine; const reIsWhitespaceChar = /\s/; const reMaybeLocalIp = /^[\d:f]/; const reIsLocalhostRedirect = /\s+(?:0\.0\.0\.0|broadcasthost|localhost|local|ip6-\w+)\b/; const reLocalIp = /^(?:0\.0\.0\.0|127\.0\.0\.1|::1|fe80::1%lo0)/; const lineIter = new this.LineIterator(this.processDirectives(rawText)); while ( lineIter.eot() === false ) { // rhill 2014-04-18: The trim is important here, as without it there // could be a lingering `\r` which would cause problems in the // following parsing code. let line = lineIter.next().trim(); if ( line.length === 0 ) { continue; } // Strip comments const c = line.charAt(0); if ( c === '!' || c === '[' ) { continue; } // Parse or skip cosmetic filters // All cosmetic filters are caught here if ( staticExtFilteringEngine.compile(line, writer) ) { continue; } // Whatever else is next can be assumed to not be a cosmetic filter // Most comments start in first column if ( c === '#' ) { continue; } // Catch comments somewhere on the line // Remove: // ... #blah blah blah // ... # blah blah blah // Don't remove: // ...#blah blah blah // because some ABP filters uses the `#` character (URL fragment) const pos = line.indexOf('#'); if ( pos !== -1 && reIsWhitespaceChar.test(line.charAt(pos - 1)) ) { line = line.slice(0, pos).trim(); } // https://github.com/gorhill/httpswitchboard/issues/15 // Ensure localhost et al. don't end up in the ubiquitous blacklist. // With hosts files, we need to remove local IP redirection if ( reMaybeLocalIp.test(c) ) { // Ignore hosts file redirect configuration // 127.0.0.1 localhost // 255.255.255.255 broadcasthost if ( reIsLocalhostRedirect.test(line) ) { continue; } line = line.replace(reLocalIp, '').trim(); } if ( line.length === 0 ) { continue; } staticNetFilteringEngine.compile(line, writer); } return writer.toString(); }; /******************************************************************************/ // https://github.com/gorhill/uBlock/issues/1395 // Added `firstparty` argument: to avoid discarding cosmetic filters when // applying 1st-party filters. µBlock.applyCompiledFilters = function(rawText, firstparty) { if ( rawText === '' ) { return; } let reader = new this.CompiledLineIO.Reader(rawText); this.staticNetFilteringEngine.fromCompiledContent(reader); this.staticExtFilteringEngine.fromCompiledContent(reader, { skipGenericCosmetic: this.userSettings.ignoreGenericCosmeticFilters, skipCosmetic: !firstparty && !this.userSettings.parseAllABPHideFilters }); }; /******************************************************************************/ // https://github.com/AdguardTeam/AdguardBrowserExtension/issues/917 µBlock.processDirectives = function(content) { const reIf = /^!#(if|endif)\b([^\n]*)/gm; const stack = []; const shouldDiscard = ( ) => stack.some(v => v); const parts = []; let beg = 0, discard = false; while ( beg < content.length ) { const match = reIf.exec(content); if ( match === null ) { break; } switch ( match[1] ) { case 'if': let expr = match[2].trim(); const target = expr.charCodeAt(0) === 0x21 /* '!' */; if ( target ) { expr = expr.slice(1); } const token = this.processDirectives.tokens.get(expr); const startDiscard = token === 'false' && target === false || token !== undefined && vAPI.webextFlavor.soup.has(token) === target; if ( discard === false && startDiscard ) { parts.push(content.slice(beg, match.index)); discard = true; } stack.push(startDiscard); break; case 'endif': stack.pop(); const stopDiscard = shouldDiscard() === false; if ( discard && stopDiscard ) { beg = match.index + match[0].length + 1; discard = false; } break; default: break; } } if ( stack.length === 0 && parts.length !== 0 ) { parts.push(content.slice(beg)); content = parts.join('\n'); } return content.trim(); }; µBlock.processDirectives.tokens = new Map([ [ 'ext_ublock', 'ublock' ], [ 'env_chromium', 'chromium' ], [ 'env_edge', 'edge' ], [ 'env_firefox', 'firefox' ], [ 'env_mobile', 'mobile' ], [ 'env_safari', 'safari' ], [ 'cap_html_filtering', 'html_filtering' ], [ 'cap_user_stylesheet', 'user_stylesheet' ], [ 'false', 'false' ], ]); /******************************************************************************/ µBlock.loadRedirectResources = async function() { try { const success = await this.redirectEngine.resourcesFromSelfie(); if ( success === true ) { return true; } const fetchPromises = [ this.redirectEngine.loadBuiltinResources() ]; const userResourcesLocation = this.hiddenSettings.userResourcesLocation; if ( userResourcesLocation !== 'unset' ) { for ( const url of userResourcesLocation.split(/\s+/) ) { fetchPromises.push(this.assets.fetchText(url)); } } const results = await Promise.all(fetchPromises); if ( Array.isArray(results) === false ) { return results; } let content = ''; for ( let i = 1; i < results.length; i++ ) { const result = results[i]; if ( result instanceof Object === false || typeof result.content !== 'string' || result.content === '' ) { continue; } content += '\n\n' + result.content; } this.redirectEngine.resourcesFromString(content); this.redirectEngine.selfieFromResources(); } catch(ex) { log.info(ex); return false; } return true; }; /******************************************************************************/ µBlock.loadPublicSuffixList = async function() { if ( this.hiddenSettings.disableWebAssembly === false ) { publicSuffixList.enableWASM(); } try { const result = await this.assets.get(`compiled/${this.pslAssetKey}`); if ( publicSuffixList.fromSelfie(result.content, this.base64) ) { return; } } catch (ex) { console.error(ex); return; } const result = await this.assets.get(this.pslAssetKey); if ( result.content !== '' ) { this.compilePublicSuffixList(result.content); } }; µBlock.compilePublicSuffixList = function(content) { publicSuffixList.parse(content, punycode.toASCII); this.assets.put( 'compiled/' + this.pslAssetKey, publicSuffixList.toSelfie(µBlock.base64) ); }; /******************************************************************************/ // This is to be sure the selfie is generated in a sane manner: the selfie will // be generated if the user doesn't change his filter lists selection for // some set time. µBlock.selfieManager = (( ) => { const µb = µBlock; let createTimer; let destroyTimer; // As of 2018-05-31: // JSON.stringify-ing ourselves results in a better baseline // memory usage at selfie-load time. For some reasons. const create = async function() { await Promise.all([ µb.assets.put( 'selfie/main', JSON.stringify({ magic: µb.systemSettings.selfieMagic, availableFilterLists: µb.availableFilterLists, }) ), µb.redirectEngine.toSelfie('selfie/redirectEngine'), µb.staticExtFilteringEngine.toSelfie( 'selfie/staticExtFilteringEngine' ), µb.staticNetFilteringEngine.toSelfie( 'selfie/staticNetFilteringEngine' ), ]); µb.lz4Codec.relinquish(); µb.selfieIsInvalid = false; }; const loadMain = async function() { const details = await µb.assets.get('selfie/main'); if ( details instanceof Object === false || typeof details.content !== 'string' || details.content === '' ) { return false; } let selfie; try { selfie = JSON.parse(details.content); } catch(ex) { } if ( selfie instanceof Object === false || selfie.magic !== µb.systemSettings.selfieMagic ) { return false; } µb.availableFilterLists = selfie.availableFilterLists; return true; }; const load = async function() { if ( µb.selfieIsInvalid ) { return false; } try { const results = await Promise.all([ loadMain(), µb.redirectEngine.fromSelfie('selfie/redirectEngine'), µb.staticExtFilteringEngine.fromSelfie( 'selfie/staticExtFilteringEngine' ), µb.staticNetFilteringEngine.fromSelfie( 'selfie/staticNetFilteringEngine' ), ]); if ( results.every(v => v) ) { return µb.loadRedirectResources(); } } catch (reason) { log.info(reason); } destroy(); return false; }; const destroy = function() { µb.cacheStorage.remove('selfie'); // TODO: obsolete, remove eventually. µb.assets.remove(/^selfie\//); µb.selfieIsInvalid = true; createTimer = vAPI.setTimeout(( ) => { createTimer = undefined; create(); }, µb.hiddenSettings.selfieAfter * 60000); }; const destroyAsync = function() { if ( destroyTimer !== undefined ) { return; } if ( createTimer !== undefined ) { clearTimeout(createTimer); createTimer = undefined; } destroyTimer = vAPI.setTimeout( ( ) => { destroyTimer = undefined; destroy(); }, 1019 ); µb.selfieIsInvalid = true; }; return { load, destroy: destroyAsync }; })(); /******************************************************************************/ // https://github.com/gorhill/uBlock/issues/531 // Overwrite user settings with admin settings if present. // // Admin settings match layout of a uBlock backup. Not all data is // necessarily present, i.e. administrators may removed entries which // values are left to the user's choice. µBlock.restoreAdminSettings = async function() { let data; try { const json = await vAPI.adminStorage.getItem('adminSettings'); if ( typeof json === 'string' && json !== '' ) { data = JSON.parse(json); } } catch (ex) { console.error(ex); } if ( data instanceof Object === false ) { return; } const bin = {}; let binNotEmpty = false; // https://github.com/uBlockOrigin/uBlock-issues/issues/666 // Allows an admin to set their own 'assets.json' file, with their // own set of stock assets. if ( typeof data.assetsBootstrapLocation === 'string' && data.assetsBootstrapLocation !== '' ) { µBlock.assetsBootstrapLocation = data.assetsBootstrapLocation; } if ( typeof data.userSettings === 'object' ) { for ( const name in this.userSettings ) { if ( this.userSettings.hasOwnProperty(name) === false ) { continue; } if ( data.userSettings.hasOwnProperty(name) === false ) { continue; } bin[name] = data.userSettings[name]; binNotEmpty = true; } } // 'selectedFilterLists' is an array of filter list tokens. Each token // is a reference to an asset in 'assets.json'. if ( Array.isArray(data.selectedFilterLists) ) { bin.selectedFilterLists = data.selectedFilterLists; binNotEmpty = true; } if ( Array.isArray(data.whitelist) ) { bin.netWhitelist = data.whitelist; binNotEmpty = true; } else if ( typeof data.netWhitelist === 'string' ) { bin.netWhitelist = data.netWhitelist.split('\n'); binNotEmpty = true; } if ( typeof data.dynamicFilteringString === 'string' ) { bin.dynamicFilteringString = data.dynamicFilteringString; binNotEmpty = true; } if ( typeof data.urlFilteringString === 'string' ) { bin.urlFilteringString = data.urlFilteringString; binNotEmpty = true; } if ( typeof data.hostnameSwitchesString === 'string' ) { bin.hostnameSwitchesString = data.hostnameSwitchesString; binNotEmpty = true; } if ( binNotEmpty ) { vAPI.storage.set(bin); } if ( typeof data.userFilters === 'string' ) { this.saveUserFilters(data.userFilters); } }; /******************************************************************************/ // https://github.com/gorhill/uBlock/issues/2344 // Support mutliple locales per filter list. // https://github.com/gorhill/uBlock/issues/3210 // Support ability to auto-enable a filter list based on user agent. µBlock.listMatchesEnvironment = function(details) { // Matches language? if ( typeof details.lang === 'string' ) { let re = this.listMatchesEnvironment.reLang; if ( re === undefined ) { const match = /^[a-z]+/.exec(self.navigator.language); if ( match !== null ) { re = new RegExp('\\b' + match[0] + '\\b'); this.listMatchesEnvironment.reLang = re; } } if ( re !== undefined && re.test(details.lang) ) { return true; } } // Matches user agent? if ( typeof details.ua === 'string' ) { let re = new RegExp('\\b' + this.escapeRegex(details.ua) + '\\b', 'i'); if ( re.test(self.navigator.userAgent) ) { return true; } } return false; }; /******************************************************************************/ µBlock.scheduleAssetUpdater = (function() { let timer, next = 0; return function(updateDelay) { if ( timer ) { clearTimeout(timer); timer = undefined; } if ( updateDelay === 0 ) { next = 0; return; } const now = Date.now(); // Use the new schedule if and only if it is earlier than the previous // one. if ( next !== 0 ) { updateDelay = Math.min(updateDelay, Math.max(next - now, 0)); } next = now + updateDelay; timer = vAPI.setTimeout(( ) => { timer = undefined; next = 0; this.assets.updateStart({ delay: this.hiddenSettings.autoUpdateAssetFetchPeriod * 1000 || 120000 }); }, updateDelay); }; })(); /******************************************************************************/ µBlock.assetObserver = function(topic, details) { // Do not update filter list if not in use. if ( topic === 'before-asset-updated' ) { if ( details.type === 'filters' ) { if ( this.availableFilterLists.hasOwnProperty(details.assetKey) === false || this.selectedFilterLists.indexOf(details.assetKey) === -1 ) { return; } } return true; } // Compile the list while we have the raw version in memory if ( topic === 'after-asset-updated' ) { // Skip selfie-related content. if ( details.assetKey.startsWith('selfie/') ) { return; } const cached = typeof details.content === 'string' && details.content !== ''; if ( this.availableFilterLists.hasOwnProperty(details.assetKey) ) { if ( cached ) { if ( this.selectedFilterLists.indexOf(details.assetKey) !== -1 ) { this.extractFilterListMetadata( details.assetKey, details.content ); this.assets.put( 'compiled/' + details.assetKey, this.compileFilters( details.content, { assetKey: details.assetKey } ) ); } } else { this.removeCompiledFilterList(details.assetKey); } } else if ( details.assetKey === this.pslAssetKey ) { if ( cached ) { this.compilePublicSuffixList(details.content); } } vAPI.messaging.broadcast({ what: 'assetUpdated', key: details.assetKey, cached: cached }); // https://github.com/gorhill/uBlock/issues/2585 // Whenever an asset is overwritten, the current selfie is quite // likely no longer valid. this.selfieManager.destroy(); return; } // Update failed. if ( topic === 'asset-update-failed' ) { vAPI.messaging.broadcast({ what: 'assetUpdated', key: details.assetKey, failed: true }); return; } // Reload all filter lists if needed. if ( topic === 'after-assets-updated' ) { if ( details.assetKeys.length !== 0 ) { // https://github.com/gorhill/uBlock/pull/2314#issuecomment-278716960 if ( this.hiddenSettings.userResourcesLocation !== 'unset' || vAPI.webextFlavor.soup.has('devbuild') ) { this.redirectEngine.invalidateResourcesSelfie(); } this.loadFilterLists(); } if ( this.userSettings.autoUpdate ) { this.scheduleAssetUpdater(this.hiddenSettings.autoUpdatePeriod * 3600000 || 25200000); } else { this.scheduleAssetUpdater(0); } vAPI.messaging.broadcast({ what: 'assetsUpdated', assetKeys: details.assetKeys }); return; } // New asset source became available, if it's a filter list, should we // auto-select it? if ( topic === 'builtin-asset-source-added' ) { if ( details.entry.content === 'filters' ) { if ( details.entry.off !== true || this.listMatchesEnvironment(details.entry) ) { this.saveSelectedFilterLists([ details.assetKey ], true); } } return; } };