uBlock/js/storage.js

678 lines
22 KiB
JavaScript
Raw Normal View History

2014-06-24 00:42:43 +02:00
/*******************************************************************************
µBlock - a Chromium browser extension to block requests.
Copyright (C) 2014 Raymond Hill
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see {http://www.gnu.org/licenses/}.
Home: https://github.com/gorhill/uBlock
*/
/* global chrome, µBlock, punycode, publicSuffixList */
/******************************************************************************/
µBlock.getBytesInUse = function() {
var getBytesInUseHandler = function(bytesInUse) {
µBlock.storageUsed = bytesInUse;
};
chrome.storage.local.getBytesInUse(null, getBytesInUseHandler);
};
/******************************************************************************/
µBlock.saveLocalSettings = function() {
chrome.storage.local.set(this.localSettings, function() {
µBlock.getBytesInUse();
});
};
/******************************************************************************/
µBlock.loadLocalSettings = function() {
var settingsLoaded = function(store) {
µBlock.localSettings = store;
};
chrome.storage.local.get(this.localSettings, settingsLoaded);
};
/******************************************************************************/
// Save local settings regularly. Not critical.
µBlock.asyncJobs.add(
'autoSaveLocalSettings',
null,
µBlock.saveLocalSettings.bind(µBlock),
2 * 60 * 1000,
true
);
/******************************************************************************/
µBlock.saveUserSettings = function() {
chrome.storage.local.set(this.userSettings, function() {
µBlock.getBytesInUse();
});
};
/******************************************************************************/
µBlock.loadUserSettings = function(callback) {
2014-06-24 00:42:43 +02:00
var settingsLoaded = function(store) {
µBlock.userSettings = store;
if ( typeof callback === 'function' ) {
2014-09-28 18:05:46 +02:00
callback(store);
}
};
chrome.storage.local.get(this.userSettings, settingsLoaded);
};
/******************************************************************************/
µBlock.saveWhitelist = function() {
var bin = {
'netWhitelist': this.stringFromWhitelist(this.netWhitelist)
};
chrome.storage.local.set(bin, function() {
µBlock.getBytesInUse();
});
this.netWhitelistModifyTime = Date.now();
};
/******************************************************************************/
µBlock.loadWhitelist = function(callback) {
var onWhitelistLoaded = function(store) {
2014-08-02 17:40:27 +02:00
var µb = µBlock;
// Backward compatibility after fix to #5
// TODO: remove once all users are up to date with latest version.
if ( store.netExceptionList ) {
if ( store.netWhitelist === '' ) {
store.netWhitelist = Object.keys(store.netExceptionList).join('\n');
if ( store.netWhitelist !== '' ) {
chrome.storage.local.set({ 'netWhitelist': store.netWhitelist });
}
}
chrome.storage.local.remove('netExceptionList');
}
µb.netWhitelist = µb.whitelistFromString(store.netWhitelist);
µb.netWhitelistModifyTime = Date.now();
if ( typeof callback === 'function' ) {
callback();
}
2014-06-24 00:42:43 +02:00
};
var bin = {
'netWhitelist': '',
'netExceptionList': ''
};
chrome.storage.local.get(bin, onWhitelistLoaded);
2014-07-17 16:52:43 +02:00
};
/******************************************************************************/
2014-07-13 02:32:44 +02:00
µBlock.saveUserFilters = function(content, callback) {
return this.assets.put(this.userFiltersPath, content, callback);
};
/******************************************************************************/
µBlock.loadUserFilters = function(callback) {
return this.assets.get(this.userFiltersPath, callback);
};
/******************************************************************************/
µBlock.appendUserFilters = function(content) {
2014-09-08 23:46:58 +02:00
var µb = this;
2014-07-13 02:32:44 +02:00
var onSaved = function(details) {
if ( details.error ) {
return;
}
2014-09-25 21:44:18 +02:00
µb.mergeFilterText(content);
µb.netFilteringEngine.freeze();
µb.cosmeticFilteringEngine.freeze();
µb.destroySelfie();
µb.toSelfieAsync();
2014-07-13 02:32:44 +02:00
};
2014-09-08 23:46:58 +02:00
2014-07-13 02:32:44 +02:00
var onLoaded = function(details) {
if ( details.error ) {
return;
}
2014-07-13 18:57:20 +02:00
if ( details.content.indexOf(content.trim()) !== -1 ) {
2014-07-13 02:32:44 +02:00
return;
}
2014-09-08 23:46:58 +02:00
µb.saveUserFilters(details.content + '\n' + content, onSaved);
2014-07-13 02:32:44 +02:00
};
2014-09-08 23:46:58 +02:00
2014-07-13 02:32:44 +02:00
if ( content.length > 0 ) {
this.loadUserFilters(onLoaded);
}
};
/******************************************************************************/
2014-07-25 22:12:20 +02:00
µBlock.getAvailableLists = function(callback) {
var availableLists = {};
var redirections = {};
2014-06-24 00:42:43 +02:00
2014-07-25 22:12:20 +02:00
// selected lists
var onSelectedListsLoaded = function(store) {
var µb = µBlock;
2014-07-25 22:12:20 +02:00
var lists = store.remoteBlacklists;
var locations = Object.keys(lists);
var oldLocation, newLocation;
var availableEntry, storedEntry;
2014-07-25 22:12:20 +02:00
while ( oldLocation = locations.pop() ) {
newLocation = redirections[oldLocation] || oldLocation;
availableEntry = availableLists[newLocation];
if ( availableEntry === undefined ) {
2014-07-25 22:12:20 +02:00
continue;
}
storedEntry = lists[oldLocation];
availableEntry.off = storedEntry.off || false;
µb.assets.setHomeURL(newLocation, availableEntry.homeURL);
if ( storedEntry.entryCount !== undefined ) {
availableEntry.entryCount = storedEntry.entryCount;
}
if ( storedEntry.entryUsedCount !== undefined ) {
availableEntry.entryUsedCount = storedEntry.entryUsedCount;
2014-07-25 22:12:20 +02:00
}
// This may happen if the list name was pulled from the list content
if ( availableEntry.title === '' && storedEntry.title !== '' ) {
availableEntry.title = storedEntry.title;
}
2014-06-24 00:42:43 +02:00
}
2014-07-25 22:12:20 +02:00
callback(availableLists);
2014-06-24 00:42:43 +02:00
};
2014-07-25 22:12:20 +02:00
// built-in lists
var onBuiltinListsLoaded = function(details) {
var location, locations;
try {
locations = JSON.parse(details.content);
} catch (e) {
locations = {};
}
var entry;
2014-07-25 22:12:20 +02:00
for ( location in locations ) {
if ( locations.hasOwnProperty(location) === false ) {
continue;
}
entry = locations[location];
availableLists['assets/thirdparties/' + location] = entry;
if ( entry.old !== undefined ) {
redirections[entry.old] = location;
delete entry.old;
}
2014-06-24 00:42:43 +02:00
}
2014-07-25 22:12:20 +02:00
// Now get user's selection of lists
2014-06-24 00:42:43 +02:00
chrome.storage.local.get(
2014-07-25 22:12:20 +02:00
{ 'remoteBlacklists': availableLists },
onSelectedListsLoaded
2014-06-24 00:42:43 +02:00
);
};
2014-07-25 22:12:20 +02:00
// permanent lists
var location;
var lists = this.permanentLists;
for ( location in lists ) {
if ( lists.hasOwnProperty(location) === false ) {
continue;
}
availableLists[location] = lists[location];
}
// custom lists
var c;
var locations = this.userSettings.externalLists.split('\n');
for ( var i = 0; i < locations.length; i++ ) {
location = locations[i].trim();
c = location.charAt(0);
if ( location === '' || c === '!' || c === '#' ) {
continue;
}
// Coarse validation
if ( /[^0-9A-Za-z!*'();:@&=+$,\/?%#\[\]_.~-]/.test(location) ) {
continue;
}
availableLists[location] = {
title: '',
group: 'custom',
external: true
};
}
// get built-in block lists.
this.assets.get('assets/ublock/filter-lists.json', onBuiltinListsLoaded);
};
/******************************************************************************/
2014-09-08 23:46:58 +02:00
µBlock.loadFilterLists = function(callback) {
2014-07-25 22:12:20 +02:00
var µb = this;
var blacklistLoadCount;
2014-09-08 23:46:58 +02:00
if ( typeof callback !== 'function' ) {
callback = this.noopFunc;
}
2014-07-25 22:12:20 +02:00
var loadBlacklistsEnd = function() {
2014-09-08 23:46:58 +02:00
µb.netFilteringEngine.freeze();
µb.cosmeticFilteringEngine.freeze();
2014-07-25 22:12:20 +02:00
chrome.storage.local.set({ 'remoteBlacklists': µb.remoteBlacklists });
2014-09-08 23:46:58 +02:00
µb.messaging.announce({ what: 'loadUbiquitousBlacklistCompleted' });
µb.toSelfieAsync();
callback();
2014-07-25 22:12:20 +02:00
};
2014-06-24 00:42:43 +02:00
var mergeBlacklist = function(details) {
2014-09-25 21:44:18 +02:00
µb.mergeFilterList(details);
2014-06-24 00:42:43 +02:00
blacklistLoadCount -= 1;
if ( blacklistLoadCount === 0 ) {
loadBlacklistsEnd();
}
};
2014-07-25 22:12:20 +02:00
var loadBlacklistsStart = function(lists) {
µb.remoteBlacklists = lists;
2014-09-08 23:46:58 +02:00
µb.netFilteringEngine.reset();
µb.cosmeticFilteringEngine.reset();
µb.destroySelfie();
2014-07-25 22:12:20 +02:00
var locations = Object.keys(lists);
blacklistLoadCount = locations.length;
2014-06-24 00:42:43 +02:00
if ( blacklistLoadCount === 0 ) {
loadBlacklistsEnd();
return;
}
// Load each preset blacklist which is not disabled.
var location;
2014-07-25 22:12:20 +02:00
while ( location = locations.pop() ) {
2014-06-24 00:42:43 +02:00
// rhill 2013-12-09:
// Ignore list if disabled
// https://github.com/gorhill/httpswitchboard/issues/78
2014-07-25 22:12:20 +02:00
if ( lists[location].off ) {
2014-06-24 00:42:43 +02:00
blacklistLoadCount -= 1;
continue;
}
µb.assets.get(location, mergeBlacklist);
2014-06-24 00:42:43 +02:00
}
};
2014-07-25 22:12:20 +02:00
this.getAvailableLists(loadBlacklistsStart);
2014-06-24 00:42:43 +02:00
};
/******************************************************************************/
2014-09-25 21:44:18 +02:00
µBlock.mergeFilterList = function(details) {
// console.log('µBlock > mergeFilterList from "%s": "%s..."', details.path, details.content.slice(0, 40));
2014-06-24 00:42:43 +02:00
2014-09-25 21:44:18 +02:00
var netFilteringEngine = this.netFilteringEngine;
var cosmeticFilteringEngine = this.cosmeticFilteringEngine;
var duplicateCount = netFilteringEngine.duplicateCount + cosmeticFilteringEngine.duplicateCount;
var acceptedCount = netFilteringEngine.acceptedCount + cosmeticFilteringEngine.acceptedCount;
2014-06-24 00:42:43 +02:00
2014-09-25 21:44:18 +02:00
this.mergeFilterText(details.content);
// For convenience, store the number of entries for this
// blacklist, user might be happy to know this information.
duplicateCount = netFilteringEngine.duplicateCount + cosmeticFilteringEngine.duplicateCount - duplicateCount;
acceptedCount = netFilteringEngine.acceptedCount + cosmeticFilteringEngine.acceptedCount - acceptedCount;
var filterListMeta = this.remoteBlacklists[details.path];
filterListMeta.entryCount = acceptedCount;
filterListMeta.entryUsedCount = acceptedCount - duplicateCount;
// Try to extract a human-friendly name (works only for
// ABP-compatible filter lists)
if ( filterListMeta.title === '' ) {
var matches = details.content.slice(0, 1024).match(/(?:^|\n)!\s*Title:([^\n]+)/i);
if ( matches !== null ) {
filterListMeta.title = matches[1].trim();
}
}
2014-09-25 21:44:18 +02:00
};
/******************************************************************************/
µBlock.mergeFilterText = function(rawText) {
var rawEnd = rawText.length;
2014-06-24 00:42:43 +02:00
// Useful references:
// https://adblockplus.org/en/filter-cheatsheet
// https://adblockplus.org/en/filters
2014-09-08 23:46:58 +02:00
var netFilteringEngine = this.netFilteringEngine;
var cosmeticFilteringEngine = this.cosmeticFilteringEngine;
2014-08-21 16:56:36 +02:00
var parseCosmeticFilters = this.userSettings.parseAllABPHideFilters;
2014-09-25 21:44:18 +02:00
var reIsCosmeticFilter = /#@?#/;
2014-09-19 16:59:44 +02:00
var reLocalhost = /(?:^|\s)(?:localhost\.localdomain|localhost|local|broadcasthost|0\.0\.0\.0|127\.0\.0\.1|::1|fe80::1%lo0)(?=\s|$)/g;
2014-06-24 00:42:43 +02:00
var reAsciiSegment = /^[\x21-\x7e]+$/;
var matches;
var lineBeg = 0, lineEnd, currentLineBeg;
var line, c;
while ( lineBeg < rawEnd ) {
lineEnd = rawText.indexOf('\n', lineBeg);
2014-09-25 21:44:18 +02:00
if ( lineEnd === -1 ) {
2014-06-24 00:42:43 +02:00
lineEnd = rawText.indexOf('\r', lineBeg);
2014-09-25 21:44:18 +02:00
if ( lineEnd === -1 ) {
2014-06-24 00:42:43 +02:00
lineEnd = rawEnd;
}
}
// rhill 2014-04-18: The trim is important here, as without it there
// could be a lingering `\r` which would cause problems in the
// following parsing code.
line = rawText.slice(lineBeg, lineEnd).trim();
currentLineBeg = lineBeg;
lineBeg = lineEnd + 1;
// Strip comments
c = line.charAt(0);
if ( c === '!' || c === '[' ) {
continue;
}
2014-09-25 21:44:18 +02:00
// Parse or skip cosmetic filters
2014-08-21 16:56:36 +02:00
if ( parseCosmeticFilters ) {
2014-09-08 23:46:58 +02:00
if ( cosmeticFilteringEngine.add(line) ) {
2014-06-24 00:42:43 +02:00
continue;
}
2014-09-25 21:44:18 +02:00
} else if ( reIsCosmeticFilter.test(line) ) {
continue;
2014-06-24 00:42:43 +02:00
}
if ( c === '#' ) {
continue;
}
// https://github.com/gorhill/httpswitchboard/issues/15
// Ensure localhost et al. don't end up in the ubiquitous blacklist.
line = line
.replace(/\s+#.*$/, '')
.toLowerCase()
.replace(reLocalhost, '')
.trim();
// The filter is whatever sequence of printable ascii character without
// whitespaces
matches = reAsciiSegment.exec(line);
2014-09-19 16:59:44 +02:00
if ( matches === null ) {
2014-09-25 21:44:18 +02:00
//console.debug('µBlock.mergeFilterList(): skipping "%s"', lineRaw);
2014-06-24 00:42:43 +02:00
continue;
}
// Bypass anomalies
// For example, when a filter contains whitespace characters, or
// whatever else outside the range of printable ascii characters.
if ( matches[0] !== line ) {
2014-09-25 21:44:18 +02:00
// console.error('"%s" !== "%s"', matches[0], line);
2014-06-24 00:42:43 +02:00
continue;
}
2014-09-19 16:59:44 +02:00
netFilteringEngine.add(matches[0]);
2014-06-24 00:42:43 +02:00
}
};
/******************************************************************************/
// `switches` contains the preset blacklists for which the switch must be
// revisited.
µBlock.reloadPresetBlacklists = function(switches, update) {
2014-09-15 20:28:07 +02:00
var µb = µBlock;
var onFilterListsReady = function() {
µb.loadUpdatableAssets({ update: update, psl: update });
};
2014-06-24 00:42:43 +02:00
// Toggle switches, if any
if ( switches !== undefined ) {
2014-09-15 20:28:07 +02:00
var filterLists = this.remoteBlacklists;
2014-06-24 00:42:43 +02:00
var i = switches.length;
while ( i-- ) {
2014-09-15 20:28:07 +02:00
if ( filterLists.hasOwnProperty(switches[i].location) === false ) {
2014-06-24 00:42:43 +02:00
continue;
}
2014-09-15 20:28:07 +02:00
filterLists[switches[i].location].off = !!switches[i].off;
2014-06-24 00:42:43 +02:00
}
// Save switch states
2014-09-15 20:28:07 +02:00
chrome.storage.local.set({ 'remoteBlacklists': filterLists }, onFilterListsReady);
} else {
onFilterListsReady();
2014-06-24 00:42:43 +02:00
}
};
/******************************************************************************/
µBlock.loadPublicSuffixList = function(callback) {
2014-09-08 23:46:58 +02:00
if ( typeof callback !== 'function' ) {
callback = this.noopFunc;
}
2014-06-24 00:42:43 +02:00
var applyPublicSuffixList = function(details) {
// TODO: Not getting proper suffix list is a bit serious, I think
// the extension should be force-restarted if it occurs..
if ( !details.error ) {
publicSuffixList.parse(details.content, punycode.toASCII);
}
2014-09-08 23:46:58 +02:00
callback();
2014-06-24 00:42:43 +02:00
};
2014-09-08 23:46:58 +02:00
this.assets.get(this.pslPath, applyPublicSuffixList);
2014-06-24 00:42:43 +02:00
};
/******************************************************************************/
// Load updatable assets
2014-09-08 23:46:58 +02:00
µBlock.loadUpdatableAssets = function(details) {
var µb = this;
details = details || {};
var update = details.update !== false;
2014-08-21 04:34:22 +02:00
this.assets.autoUpdate = update || this.userSettings.autoUpdate;
this.assets.autoUpdateDelay = this.updateAssetsEvery;
2014-08-21 01:39:49 +02:00
2014-09-08 23:46:58 +02:00
var onFiltersReady = function() {
if ( update ) {
µb.updater.restart();
}
};
var onPSLReady = function() {
µb.loadFilterLists(onFiltersReady);
};
if ( details.psl !== false ) {
this.loadPublicSuffixList(onPSLReady);
} else {
this.loadFilterLists(onFiltersReady);
2014-08-21 01:39:49 +02:00
}
2014-06-24 00:42:43 +02:00
};
/******************************************************************************/
2014-09-08 23:46:58 +02:00
µBlock.toSelfie = function() {
var selfie = {
magic: this.selfieMagic,
publicSuffixList: publicSuffixList.toSelfie(),
filterLists: this.remoteBlacklists,
netFilteringEngine: this.netFilteringEngine.toSelfie(),
2014-09-11 22:00:50 +02:00
cosmeticFilteringEngine: this.cosmeticFilteringEngine.toSelfie()
2014-09-08 23:46:58 +02:00
};
chrome.storage.local.set({ selfie: selfie });
// console.log('µBlock.toSelfie> made a selfie!');
};
// This is to be sure the selfie is generated in a sane manner: the selfie will
// be generated if the user doesn't change his filter lists selection for
// some set time.
2014-09-25 21:44:18 +02:00
2014-09-08 23:46:58 +02:00
µBlock.toSelfieAsync = function(after) {
if ( typeof after !== 'number' ) {
after = this.selfieAfter;
}
this.asyncJobs.add(
'toSelfie',
null,
this.toSelfie.bind(this),
after,
false
);
};
/******************************************************************************/
µBlock.fromSelfie = function(callback) {
var µb = this;
if ( typeof callback !== 'function' ) {
callback = this.noopFunc;
}
var onSelfieReady = function(store) {
var selfie = store.selfie;
if ( typeof selfie !== 'object' || selfie.magic !== µb.selfieMagic ) {
callback(false);
return;
}
if ( publicSuffixList.fromSelfie(selfie.publicSuffixList) !== true ) {
callback(false);
return;
}
// console.log('µBlock.fromSelfie> selfie looks good');
µb.remoteBlacklists = selfie.filterLists;
µb.netFilteringEngine.fromSelfie(selfie.netFilteringEngine);
µb.cosmeticFilteringEngine.fromSelfie(selfie.cosmeticFilteringEngine);
callback(true);
};
chrome.storage.local.get('selfie', onSelfieReady);
};
/******************************************************************************/
µBlock.destroySelfie = function() {
chrome.storage.local.remove('selfie');
};
/******************************************************************************/
2014-06-24 00:42:43 +02:00
// Load all
µBlock.load = function() {
var µb = this;
2014-09-11 22:00:50 +02:00
var fromSelfie = false;
2014-09-15 17:09:06 +02:00
// Final initialization steps after all needed assets are in memory.
// - Initialize internal state with maybe already existing tabs.
// - Schedule next update operation.
2014-09-11 22:00:50 +02:00
var onAllDone = function() {
2014-09-15 17:09:06 +02:00
// http://code.google.com/p/chromium/issues/detail?id=410868#c11
// Need to be sure to access `chrome.runtime.lastError` to prevent
// spurious warnings in the console.
2014-09-15 17:26:52 +02:00
var scriptDone = function() {
2014-09-15 17:09:06 +02:00
chrome.runtime.lastError;
};
var scriptEnd = function(tabId) {
if ( chrome.runtime.lastError ) {
return;
}
chrome.tabs.executeScript(tabId, {
file: 'js/contentscript-end.js',
allFrames: true,
runAt: 'document_idle'
}, scriptDone);
};
var scriptStart = function(tabId) {
chrome.tabs.executeScript(tabId, {
file: 'js/contentscript-start.js',
allFrames: true,
runAt: 'document_idle'
}, function(){ scriptEnd(tabId); });
};
2014-09-08 23:46:58 +02:00
var bindToTabs = function(tabs) {
var i = tabs.length, tab;
while ( i-- ) {
tab = tabs[i];
µb.bindTabToPageStats(tab.id, tab.url);
// https://github.com/gorhill/uBlock/issues/129
scriptStart(tab.id);
}
};
chrome.tabs.query({ url: 'http://*/*' }, bindToTabs);
chrome.tabs.query({ url: 'https://*/*' }, bindToTabs);
// https://github.com/gorhill/uBlock/issues/184
// If we restored a selfie, check for updates not too far
// in the future.
2014-09-11 22:00:50 +02:00
var nextUpdate = fromSelfie === false && µb.userSettings.autoUpdate ?
µb.nextUpdateAfter :
µb.firstUpdateAfter;
µb.updater.restart(nextUpdate);
2014-09-08 23:46:58 +02:00
};
2014-09-11 22:00:50 +02:00
// https://github.com/gorhill/uBlock/issues/226
// Whitelist in memory.
// Whitelist parser needs PSL to be ready.
var onWhitelistReady = function() {
onAllDone();
};
// Filters are in memory.
// Filter engines need PSL to be ready.
2014-09-08 23:46:58 +02:00
var onFiltersReady = function() {
2014-09-11 22:00:50 +02:00
µb.loadWhitelist(onWhitelistReady);
2014-09-08 23:46:58 +02:00
};
// Load order because dependencies:
// User settings -> PSL -> [filter lists, user whitelist]
var onPSLReady = function() {
2014-09-08 23:46:58 +02:00
µb.loadFilterLists(onFiltersReady);
};
2014-09-08 23:46:58 +02:00
// If no selfie available, take the long way, i.e. load and parse
// raw data.
var onSelfieReady = function(success) {
if ( success === true ) {
2014-09-11 22:00:50 +02:00
fromSelfie = true;
µb.loadWhitelist(onWhitelistReady);
2014-09-08 23:46:58 +02:00
return;
}
µb.assets.autoUpdate = µb.userSettings.autoUpdate;
µb.loadPublicSuffixList(onPSLReady);
};
2014-09-08 23:46:58 +02:00
// User settings are in memory
2014-09-28 18:05:46 +02:00
var onUserSettingsReady = function(settings) {
µb.contextMenu.toggle(settings.contextMenuEnabled);
2014-09-08 23:46:58 +02:00
µb.fromSelfie(onSelfieReady);
};
this.loadUserSettings(onUserSettingsReady);
2014-08-20 07:18:53 +02:00
this.loadLocalSettings();
2014-06-24 00:42:43 +02:00
this.getBytesInUse();
};