mirror of
https://github.com/gorhill/uBlock.git
synced 2024-11-10 09:07:54 +01:00
Add support for diff-patching filter lists
Related discussion: https://github.com/ameshkov/diffupdates The benefits of diff-patching filter lists is much shorter update schedule and significantly less bandwidth consumed. At the moment, only default filter lists are subject to be diff-patched. External filter lists can make their lists diff-patchable by following the specification link above. Only filter lists fetched by the auto-updater are candidate for diff-patching. Forcing a manual update of the filter lists will prevent the diff-patcher from kicking in until one or more lists are auto-updated. Some back-of-the-envelop calculations regarding the load on free CDN solutions used by uBO to distribute its own filter lists: Currently, for each CDN (with lists updating after days): ~560 M req/month, ~78 TB/month With diff-patching lists on a 6-hour schedule: ~390 M req/month, 1 TB/month Those estimates were done according to statistics shown by jsDelivr, which is one of 4 CDNs picked randomly when a list updates: https://www.jsdelivr.com/package/gh/uBlockOrigin/uAssetsCDN?tab=stats
This commit is contained in:
Raymond Hill
parent
032f170dba
commit
d05ff8ffeb
4 changed files with 490 additions and 78 deletions
|
|
@ -163,8 +163,7 @@
|
|||
"cdnURLs": [
|
||||
"https://cdn.jsdelivr.net/gh/uBlockOrigin/uAssetsCDN@latest/thirdparties/easylist.txt",
|
||||
"https://cdn.statically.io/gh/uBlockOrigin/uAssetsCDN/main/thirdparties/easylist.txt",
|
||||
"https://ublockorigin.pages.dev/thirdparties/easylist.txt",
|
||||
"https://easylist.to/easylist/easylist.txt"
|
||||
"https://ublockorigin.pages.dev/thirdparties/easylist.txt"
|
||||
],
|
||||
"supportURL": "https://easylist.to/"
|
||||
},
|
||||
|
|
@ -215,8 +214,7 @@
|
|||
"cdnURLs": [
|
||||
"https://cdn.jsdelivr.net/gh/uBlockOrigin/uAssetsCDN@latest/thirdparties/easyprivacy.txt",
|
||||
"https://cdn.statically.io/gh/uBlockOrigin/uAssetsCDN/main/thirdparties/easyprivacy.txt",
|
||||
"https://ublockorigin.pages.dev/thirdparties/easyprivacy.txt",
|
||||
"https://easylist.to/easylist/easyprivacy.txt"
|
||||
"https://ublockorigin.pages.dev/thirdparties/easyprivacy.txt"
|
||||
],
|
||||
"supportURL": "https://easylist.to/"
|
||||
},
|
||||
|
|
|
|||
280
src/js/assets.js
280
src/js/assets.js
|
|
@ -35,6 +35,9 @@ import { ubolog } from './console.js';
|
|||
const reIsExternalPath = /^(?:[a-z-]+):\/\//;
|
||||
const reIsUserAsset = /^user-/;
|
||||
const errorCantConnectTo = i18n$('errorCantConnectTo');
|
||||
const MS_PER_HOUR = 60 * 60 * 1000;
|
||||
const MS_PER_DAY = 24 * MS_PER_HOUR;
|
||||
const EXPIRES_DEFAULT = 7;
|
||||
|
||||
const assets = {};
|
||||
|
||||
|
|
@ -42,16 +45,57 @@ const assets = {};
|
|||
// bandwidth of remote servers.
|
||||
let remoteServerFriendly = false;
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
const parseExpires = s => {
|
||||
const matches = s.match(/(\d+)\s*([dh])?/i);
|
||||
if ( matches === null ) { return 0; }
|
||||
let updateAfter = parseInt(matches[1], 10);
|
||||
if ( matches[2] === 'h' ) {
|
||||
updateAfter = Math.ceil(updateAfter / 12) / 2;
|
||||
}
|
||||
return updateAfter;
|
||||
};
|
||||
|
||||
const extractMetadataFromList = (content, fields) => {
|
||||
const out = {};
|
||||
const head = content.slice(0, 1024);
|
||||
for ( let field of fields ) {
|
||||
field = field.replace(/\s+/g, '-');
|
||||
const re = new RegExp(`^(?:! *|# +)${field.replace(/-/g, '(?: +|-)')}: *(.+)$`, 'im');
|
||||
const match = re.exec(head);
|
||||
let value = match && match[1].trim() || undefined;
|
||||
if ( value !== undefined && /^%.+%$/.test(value) ) {
|
||||
value = undefined;
|
||||
}
|
||||
field = field.toLowerCase().replace(
|
||||
/-[a-z]/g, s => s.charAt(1).toUpperCase()
|
||||
);
|
||||
out[field] = value;
|
||||
}
|
||||
// Pre-process known fields
|
||||
if ( out.lastModified ) {
|
||||
out.lastModified = (new Date(out.lastModified)).getTime() || 0;
|
||||
}
|
||||
if ( out.expires ) {
|
||||
out.expires = Math.max(parseExpires(out.expires), 0.5);
|
||||
}
|
||||
if ( out.diffExpires ) {
|
||||
out.diffExpires = Math.max(parseExpires(out.diffExpires), 0.25);
|
||||
}
|
||||
return out;
|
||||
};
|
||||
assets.extractMetadataFromList = extractMetadataFromList;
|
||||
|
||||
const resourceTimeFromXhr = xhr => {
|
||||
try {
|
||||
// First lookup timestamp from content
|
||||
let assetTime = 0;
|
||||
if ( typeof xhr.response === 'string' ) {
|
||||
const head = xhr.response.slice(0, 512);
|
||||
const match = /^! Last modified: (.+)$/m.exec(head);
|
||||
if ( match ) {
|
||||
assetTime = (new Date(match[1])).getTime() || 0;
|
||||
}
|
||||
const metadata = extractMetadataFromList(xhr.response, [
|
||||
'Last-Modified'
|
||||
]);
|
||||
assetTime = metadata.lastModified || 0;
|
||||
}
|
||||
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Age
|
||||
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Date
|
||||
|
|
@ -91,6 +135,40 @@ const resourceIsStale = (networkDetails, cacheDetails) => {
|
|||
return false;
|
||||
};
|
||||
|
||||
const getUpdateAfterTime = (assetKey, diff = false) => {
|
||||
const entry = assetCacheRegistry[assetKey];
|
||||
if ( entry ) {
|
||||
if ( diff && typeof entry.diffExpires === 'number' ) {
|
||||
return entry.diffExpires * MS_PER_DAY;
|
||||
}
|
||||
if ( typeof entry.expires === 'number' ) {
|
||||
return entry.expires * MS_PER_DAY;
|
||||
}
|
||||
}
|
||||
if ( assetSourceRegistry ) {
|
||||
const entry = assetSourceRegistry[assetKey];
|
||||
if ( entry && typeof entry.updateAfter === 'number' ) {
|
||||
return entry.updateAfter * MS_PER_DAY;
|
||||
}
|
||||
}
|
||||
return EXPIRES_DEFAULT * MS_PER_DAY; // default to 7-day
|
||||
};
|
||||
|
||||
const getWriteTime = assetKey => {
|
||||
const entry = assetCacheRegistry[assetKey];
|
||||
if ( entry ) { return entry.writeTime || 0; }
|
||||
return 0;
|
||||
};
|
||||
|
||||
const isDiffUpdatableAsset = content => {
|
||||
if ( typeof content !== 'string' ) { return false; }
|
||||
const data = extractMetadataFromList(content, [
|
||||
'Diff-Path',
|
||||
]);
|
||||
return typeof data.diffPath === 'string' &&
|
||||
/^[^%].*[^%]$/.test(data.diffPath);
|
||||
};
|
||||
|
||||
const stringIsNotEmpty = s => typeof s === 'string' && s !== '';
|
||||
|
||||
/******************************************************************************/
|
||||
|
|
@ -252,14 +330,6 @@ assets.fetchText = async function(url) {
|
|||
details.content = '';
|
||||
details.error = 'assets.fetchText(): Not a text file';
|
||||
}
|
||||
|
||||
// Important: Non empty text resource must always end with a newline
|
||||
if (
|
||||
details.content.length !== 0 &&
|
||||
details.content.endsWith('\n') === false
|
||||
) {
|
||||
details.content += '\n';
|
||||
}
|
||||
} catch(ex) {
|
||||
details = ex;
|
||||
}
|
||||
|
|
@ -374,6 +444,13 @@ assets.fetchFilterList = async function(mainlistURL) {
|
|||
return { url: mainlistURL, content: '', error: part.error };
|
||||
}
|
||||
resourceTime = resourceTimeFromParts(allParts, resourceTime);
|
||||
// Skip pre-parser directives for diff-updatable assets
|
||||
if ( allParts.length === 1 && allParts[0] instanceof Object ) {
|
||||
if ( isDiffUpdatableAsset(allParts[0].content) ) {
|
||||
allParts[0] = allParts[0].content;
|
||||
break;
|
||||
}
|
||||
}
|
||||
allParts = processIncludeDirectives(allParts);
|
||||
} while ( allParts.some(part => typeof part !== 'string') );
|
||||
// If we reach this point, this means all fetches were successful.
|
||||
|
|
@ -457,9 +534,6 @@ function registerAssetSource(assetKey, newDict) {
|
|||
} else if ( currentDict.contentURL === undefined ) {
|
||||
currentDict.contentURL = [];
|
||||
}
|
||||
if ( typeof currentDict.updateAfter !== 'number' ) {
|
||||
currentDict.updateAfter = 7;
|
||||
}
|
||||
if ( currentDict.submitter ) {
|
||||
currentDict.submitTime = Date.now(); // To detect stale entries
|
||||
}
|
||||
|
|
@ -607,7 +681,7 @@ async function assetCacheRead(assetKey, updateReadTime = false) {
|
|||
|
||||
const reportBack = function(content) {
|
||||
if ( content instanceof Blob ) { content = ''; }
|
||||
const details = { assetKey: assetKey, content: content };
|
||||
const details = { assetKey, content };
|
||||
if ( content === '' ) { details.error = 'ENOTFOUND'; }
|
||||
return details;
|
||||
};
|
||||
|
|
@ -979,6 +1053,18 @@ async function getRemote(assetKey) {
|
|||
url: contentURL,
|
||||
resourceTime: result.resourceTime || 0,
|
||||
});
|
||||
|
||||
if ( assetDetails.content === 'filters' ) {
|
||||
const metadata = extractMetadataFromList(result.content, [
|
||||
'Last-Modified',
|
||||
'Expires',
|
||||
'Diff-Name',
|
||||
'Diff-Path',
|
||||
'Diff-Expires',
|
||||
]);
|
||||
assetCacheSetDetails(assetKey, metadata);
|
||||
}
|
||||
|
||||
registerAssetSource(assetKey, { birthtime: undefined, error: undefined });
|
||||
return reportBack(result.content);
|
||||
}
|
||||
|
|
@ -1029,8 +1115,7 @@ assets.metadata = async function() {
|
|||
if ( cacheEntry ) {
|
||||
assetEntry.cached = true;
|
||||
assetEntry.writeTime = cacheEntry.writeTime;
|
||||
const obsoleteAfter =
|
||||
cacheEntry.writeTime + assetEntry.updateAfter * 86400000;
|
||||
const obsoleteAfter = cacheEntry.writeTime + getUpdateAfterTime(assetKey);
|
||||
assetEntry.obsolete = obsoleteAfter < now;
|
||||
assetEntry.remoteURL = cacheEntry.remoteURL;
|
||||
} else if (
|
||||
|
|
@ -1074,7 +1159,7 @@ assets.getUpdateAges = async function(conditions = {}) {
|
|||
out.push({
|
||||
assetKey,
|
||||
age,
|
||||
ageNormalized: age / (asset.updateAfter * 86400000),
|
||||
ageNormalized: age / getUpdateAfterTime(assetKey),
|
||||
});
|
||||
}
|
||||
return out;
|
||||
|
|
@ -1091,33 +1176,122 @@ let updaterStatus;
|
|||
let updaterAssetDelay = updaterAssetDelayDefault;
|
||||
let updaterAuto = false;
|
||||
|
||||
const updateFirst = function() {
|
||||
const getAssetDiffDetails = assetKey => {
|
||||
const out = { name: assetKey };
|
||||
const cacheEntry = assetCacheRegistry[assetKey];
|
||||
if ( cacheEntry === undefined ) { return; }
|
||||
if ( cacheEntry.diffPath === undefined ) { return; }
|
||||
if ( cacheEntry.diffName === undefined ) { return; }
|
||||
out.diffName = cacheEntry.diffName;
|
||||
out.patchPath = cacheEntry.diffPath;
|
||||
const assetEntry = assetSourceRegistry[assetKey];
|
||||
if ( assetEntry === undefined ) { return; }
|
||||
if ( Array.isArray(assetEntry.cdnURLs) === false ) { return; }
|
||||
out.cdnURLs = assetEntry.cdnURLs.slice();
|
||||
return out;
|
||||
};
|
||||
|
||||
async function diffUpdater() {
|
||||
const toUpdate = await getUpdateCandidates(true);
|
||||
const now = Date.now();
|
||||
const toHardUpdate = [];
|
||||
const toSoftUpdate = [];
|
||||
while ( toUpdate.length !== 0 ) {
|
||||
const assetKey = toUpdate.shift();
|
||||
const assetDetails = getAssetDiffDetails(assetKey);
|
||||
if ( assetDetails === undefined ) { continue; }
|
||||
if ( assetDetails.patchPath === undefined ) { continue; }
|
||||
if ( assetDetails.diffName === undefined ) { continue; }
|
||||
assetDetails.what = 'update';
|
||||
if ( (getWriteTime(assetKey) + getUpdateAfterTime(assetKey, true)) > now ) {
|
||||
toSoftUpdate.push(assetDetails);
|
||||
} else {
|
||||
toHardUpdate.push(assetDetails);
|
||||
}
|
||||
}
|
||||
if ( toHardUpdate.length === 0 ) { return; }
|
||||
ubolog('Diff updater: cycle start');
|
||||
return new Promise(resolve => {
|
||||
let pendingOps = 0;
|
||||
const bc = new globalThis.BroadcastChannel('diffUpdater');
|
||||
bc.onmessage = ev => {
|
||||
const data = ev.data;
|
||||
if ( data.what === 'ready' ) {
|
||||
ubolog('Diff updater: hard updating', toHardUpdate.map(v => v.name).join());
|
||||
while ( toHardUpdate.length !== 0 ) {
|
||||
const assetDetails = toHardUpdate.shift();
|
||||
assetDetails.fetch = true;
|
||||
bc.postMessage(assetDetails);
|
||||
pendingOps += 1;
|
||||
}
|
||||
return;
|
||||
}
|
||||
if ( data.status === 'needtext' ) {
|
||||
ubolog('Diff updater: need text for', data.name);
|
||||
assetCacheRead(data.name).then(result => {
|
||||
data.text = result.content;
|
||||
data.status = undefined;
|
||||
bc.postMessage(data);
|
||||
});
|
||||
return;
|
||||
}
|
||||
if ( data.status === 'updated' ) {
|
||||
ubolog(`Diff updater: successfully patched ${data.name} using ${data.diffURL}`);
|
||||
const metadata = extractMetadataFromList(data.text, [
|
||||
'Last-Modified',
|
||||
'Expires',
|
||||
'Diff-Name',
|
||||
'Diff-Path',
|
||||
'Diff-Expires',
|
||||
]);
|
||||
assetCacheWrite(data.name, {
|
||||
content: data.text,
|
||||
resourceTime: metadata.lastModified || 0,
|
||||
});
|
||||
assetCacheSetDetails(data.name, metadata);
|
||||
} else if ( data.error ) {
|
||||
ubolog(`Diff updater: failed to diff-update ${data.name}, reason: ${data.error}`);
|
||||
}
|
||||
pendingOps -= 1;
|
||||
if ( pendingOps === 0 && toSoftUpdate.length !== 0 ) {
|
||||
ubolog('Diff updater: soft updating', toSoftUpdate.map(v => v.name).join());
|
||||
while ( toSoftUpdate.length !== 0 ) {
|
||||
bc.postMessage(toSoftUpdate.shift());
|
||||
pendingOps += 1;
|
||||
}
|
||||
}
|
||||
if ( pendingOps !== 0 ) { return; }
|
||||
ubolog('Diff updater: cycle complete');
|
||||
worker.terminate();
|
||||
bc.close();
|
||||
resolve();
|
||||
};
|
||||
const worker = new Worker('js/diff-updater.js');
|
||||
});
|
||||
}
|
||||
|
||||
function updateFirst() {
|
||||
updaterStatus = 'updating';
|
||||
updaterFetched.clear();
|
||||
updaterUpdated.length = 0;
|
||||
fireNotification('before-assets-updated');
|
||||
updateNext();
|
||||
};
|
||||
diffUpdater().catch(reason => {
|
||||
ubolog(reason);
|
||||
}).finally(( ) => {
|
||||
updateNext();
|
||||
});
|
||||
}
|
||||
|
||||
const updateNext = async function() {
|
||||
async function getUpdateCandidates() {
|
||||
const [ assetDict, cacheDict ] = await Promise.all([
|
||||
getAssetSourceRegistry(),
|
||||
getAssetCacheRegistry(),
|
||||
]);
|
||||
|
||||
const now = Date.now();
|
||||
const toUpdate = [];
|
||||
for ( const assetKey in assetDict ) {
|
||||
const assetEntry = assetDict[assetKey];
|
||||
if ( assetEntry.hasRemoteURL !== true ) { continue; }
|
||||
if ( updaterFetched.has(assetKey) ) { continue; }
|
||||
const cacheEntry = cacheDict[assetKey];
|
||||
if (
|
||||
(cacheEntry instanceof Object) &&
|
||||
(cacheEntry.writeTime + assetEntry.updateAfter * 86400000) > now
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
fireNotification('before-asset-updated', {
|
||||
assetKey,
|
||||
|
|
@ -1132,9 +1306,6 @@ const updateNext = async function() {
|
|||
assetCacheRemove(assetKey);
|
||||
}
|
||||
}
|
||||
if ( toUpdate.length === 0 ) {
|
||||
return updateDone();
|
||||
}
|
||||
// https://github.com/uBlockOrigin/uBlock-issues/issues/1165
|
||||
// Update most obsolete asset first.
|
||||
toUpdate.sort((a, b) => {
|
||||
|
|
@ -1142,17 +1313,34 @@ const updateNext = async function() {
|
|||
const tb = cacheDict[b] !== undefined ? cacheDict[b].writeTime : 0;
|
||||
return ta - tb;
|
||||
});
|
||||
updaterFetched.add(toUpdate[0]);
|
||||
return toUpdate;
|
||||
}
|
||||
|
||||
async function updateNext() {
|
||||
const toUpdate = await getUpdateCandidates();
|
||||
const now = Date.now();
|
||||
const toHardUpdate = [];
|
||||
|
||||
while ( toUpdate.length !== 0 ) {
|
||||
const assetKey = toUpdate.shift();
|
||||
const writeTime = getWriteTime(assetKey);
|
||||
const updateDelay = getUpdateAfterTime(assetKey);
|
||||
if ( (writeTime + updateDelay) > now ) { continue; }
|
||||
toHardUpdate.push(assetKey);
|
||||
}
|
||||
if ( toHardUpdate.length === 0 ) {
|
||||
return updateDone();
|
||||
}
|
||||
|
||||
const assetKey = toHardUpdate.pop();
|
||||
updaterFetched.add(assetKey);
|
||||
|
||||
// In auto-update context, be gentle on remote servers.
|
||||
remoteServerFriendly = updaterAuto;
|
||||
|
||||
let result;
|
||||
if (
|
||||
toUpdate[0] !== 'assets.json' ||
|
||||
µb.hiddenSettings.debugAssetsJson !== true
|
||||
) {
|
||||
result = await getRemote(toUpdate[0]);
|
||||
if ( assetKey !== 'assets.json' || µb.hiddenSettings.debugAssetsJson !== true ) {
|
||||
result = await getRemote(assetKey);
|
||||
} else {
|
||||
result = await assets.fetchText(µb.assetsJsonPath);
|
||||
result.assetKey = 'assets.json';
|
||||
|
|
@ -1161,8 +1349,10 @@ const updateNext = async function() {
|
|||
remoteServerFriendly = false;
|
||||
|
||||
if ( result.error ) {
|
||||
ubolog(`Full updater: failed to update ${assetKey}`);
|
||||
fireNotification('asset-update-failed', { assetKey: result.assetKey });
|
||||
} else {
|
||||
ubolog(`Full updater: successfully updated ${assetKey}`);
|
||||
updaterUpdated.push(result.assetKey);
|
||||
if ( result.assetKey === 'assets.json' && result.content !== '' ) {
|
||||
updateAssetSourceRegistry(result.content);
|
||||
|
|
@ -1170,18 +1360,18 @@ const updateNext = async function() {
|
|||
}
|
||||
|
||||
updaterTimer.on(updaterAssetDelay);
|
||||
};
|
||||
}
|
||||
|
||||
const updaterTimer = vAPI.defer.create(updateNext);
|
||||
|
||||
const updateDone = function() {
|
||||
function updateDone() {
|
||||
const assetKeys = updaterUpdated.slice(0);
|
||||
updaterFetched.clear();
|
||||
updaterUpdated.length = 0;
|
||||
updaterStatus = undefined;
|
||||
updaterAssetDelay = updaterAssetDelayDefault;
|
||||
fireNotification('after-assets-updated', { assetKeys });
|
||||
};
|
||||
}
|
||||
|
||||
assets.updateStart = function(details) {
|
||||
const oldUpdateDelay = updaterAssetDelay;
|
||||
|
|
|
|||
243
src/js/diff-updater.js
Normal file
243
src/js/diff-updater.js
Normal file
|
|
@ -0,0 +1,243 @@
|
|||
/*******************************************************************************
|
||||
|
||||
uBlock Origin - a browser extension to block requests.
|
||||
Copyright (C) 2014-present Raymond Hill
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see {http://www.gnu.org/licenses/}.
|
||||
|
||||
Home: https://github.com/gorhill/uBlock
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
// This module can be dynamically loaded or spun off as a worker.
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
const patches = new Map();
|
||||
const encoder = new TextEncoder();
|
||||
const reFileName = /[^\/]+$/;
|
||||
const EMPTYLINE = '';
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
const suffleArray = arr => {
|
||||
const out = arr.slice();
|
||||
for ( let i = 0, n = out.length; i < n; i++ ) {
|
||||
const j = Math.floor(Math.random() * n);
|
||||
if ( j === i ) { continue; }
|
||||
[ out[j], out[i] ] = [ out[i], out[j] ];
|
||||
}
|
||||
return out;
|
||||
};
|
||||
|
||||
const basename = url => {
|
||||
const match = reFileName.exec(url);
|
||||
return match && match[0] || '';
|
||||
};
|
||||
|
||||
const resolveURL = (path, url) => {
|
||||
try {
|
||||
const urlAfter = new URL(path, url);
|
||||
return urlAfter.href;
|
||||
}
|
||||
catch(_) {
|
||||
}
|
||||
};
|
||||
|
||||
function parsePatch(patch) {
|
||||
const patchDetails = new Map();
|
||||
const diffLines = patch.split('\n');
|
||||
let i = 0, n = diffLines.length;
|
||||
while ( i < n ) {
|
||||
const line = diffLines[i++];
|
||||
if ( line.startsWith('diff ') === false ) { continue; }
|
||||
const fields = line.split(/\s+/);
|
||||
const diffBlock = {};
|
||||
for ( let j = 0; j < fields.length; j++ ) {
|
||||
const field = fields[j];
|
||||
const pos = field.indexOf(':');
|
||||
if ( pos === -1 ) { continue; }
|
||||
const name = field.slice(0, pos);
|
||||
if ( name === '' ) { continue; }
|
||||
const value = field.slice(pos+1);
|
||||
switch ( name ) {
|
||||
case 'name':
|
||||
case 'checksum':
|
||||
diffBlock[name] = value;
|
||||
break;
|
||||
case 'lines':
|
||||
diffBlock.lines = parseInt(value, 10);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
if ( diffBlock.name === undefined ) { return; }
|
||||
if ( isNaN(diffBlock.lines) || diffBlock.lines <= 0 ) { return; }
|
||||
if ( diffBlock.checksum === undefined ) { return; }
|
||||
patchDetails.set(diffBlock.name, diffBlock);
|
||||
diffBlock.diff = diffLines.slice(i, i + diffBlock.lines).join('\n');
|
||||
i += diffBlock.lines;
|
||||
}
|
||||
if ( patchDetails.size === 0 ) { return; }
|
||||
return patchDetails;
|
||||
}
|
||||
|
||||
function applyPatch(text, diff) {
|
||||
// Inspired from (Perl) "sub _patch" at:
|
||||
// https://twiki.org/p/pub/Codev/RcsLite/RcsLite.pm
|
||||
// Apparently authored by John Talintyre in Jan. 2002
|
||||
// https://twiki.org/cgi-bin/view/Codev/RcsLite
|
||||
const lines = text.split('\n');
|
||||
const diffLines = diff.split('\n');
|
||||
let iAdjust = 0;
|
||||
let iDiff = 0, nDiff = diffLines.length;
|
||||
while ( iDiff < nDiff ) {
|
||||
const diffParsed = /^([ad])(\d+) (\d+)$/.exec(diffLines[iDiff++]);
|
||||
if ( diffParsed === null ) { return; }
|
||||
const op = diffParsed[1];
|
||||
const iOp = parseInt(diffParsed[2], 10);
|
||||
const nOp = parseInt(diffParsed[3], 10);
|
||||
const iOpAdj = iOp + iAdjust;
|
||||
if ( iOpAdj > lines.length ) { return; }
|
||||
// Delete lines
|
||||
if ( op === 'd' ) {
|
||||
lines.splice(iOpAdj-1, nOp);
|
||||
iAdjust -= nOp;
|
||||
continue;
|
||||
}
|
||||
// Add lines: Don't use splice() to avoid stack limit issues
|
||||
for ( let i = 0; i < nOp; i++ ) {
|
||||
lines.push(EMPTYLINE);
|
||||
}
|
||||
lines.copyWithin(iOpAdj+nOp, iOpAdj);
|
||||
for ( let i = 0; i < nOp; i++ ) {
|
||||
lines[iOpAdj+i] = diffLines[iDiff+i];
|
||||
}
|
||||
iAdjust += nOp;
|
||||
iDiff += nOp;
|
||||
}
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
// Async
|
||||
|
||||
async function applyPatchAndValidate(assetDetails, diffDetails) {
|
||||
const { text } = assetDetails;
|
||||
const { diff, checksum } = diffDetails;
|
||||
const textAfter = applyPatch(text, diff);
|
||||
if ( typeof textAfter !== 'string' ) {
|
||||
assetDetails.error = 'baddiff';
|
||||
return false;
|
||||
}
|
||||
const crypto = globalThis.crypto;
|
||||
if ( typeof crypto !== 'object' ) {
|
||||
assetDetails.error = 'nocrypto';
|
||||
return false;
|
||||
}
|
||||
const arrayin = encoder.encode(textAfter);
|
||||
const arraybuffer = await crypto.subtle.digest('SHA-1', arrayin);
|
||||
const arrayout = new Uint8Array(arraybuffer);
|
||||
const sha1Full = Array.from(arrayout).map(i =>
|
||||
i.toString(16).padStart(2, '0')
|
||||
).join('');
|
||||
if ( sha1Full.startsWith(checksum) === false ) {
|
||||
assetDetails.error = 'badchecksum';
|
||||
return false;
|
||||
}
|
||||
assetDetails.text = textAfter;
|
||||
const head = textAfter.slice(0, 1024);
|
||||
let match = /^! Diff-Path: (\S+)/m.exec(head);
|
||||
assetDetails.patchPath = match ? match[1] : undefined;
|
||||
match = /^! Diff-Name: (\S+)/m.exec(head);
|
||||
assetDetails.diffName = match ? match[1] : undefined;
|
||||
return true;
|
||||
}
|
||||
|
||||
async function fetchPatchDetailsFromCDNs(assetDetails) {
|
||||
const { patchPath, cdnURLs } = assetDetails;
|
||||
if ( Array.isArray(cdnURLs) === false ) { return null; }
|
||||
if ( cdnURLs.length === 0 ) { return null; }
|
||||
for ( const cdnURL of suffleArray(cdnURLs) ) {
|
||||
const diffURL = resolveURL(patchPath, cdnURL);
|
||||
if ( diffURL === undefined ) { continue; }
|
||||
const response = await fetch(diffURL).catch(reason => {
|
||||
console.error(reason);
|
||||
});
|
||||
if ( response === undefined ) { continue; }
|
||||
if ( response.ok !== true ) { continue; }
|
||||
const patchText = await response.text();
|
||||
const patchDetails = parsePatch(patchText);
|
||||
if ( patchDetails === undefined ) { continue; }
|
||||
return { diffURL, patchDetails };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function fetchPatchDetails(assetDetails) {
|
||||
const { patchPath } = assetDetails;
|
||||
const patchFile = basename(patchPath);
|
||||
if ( patchFile === '' ) { return null; }
|
||||
if ( patches.has(patchFile) ) {
|
||||
return patches.get(patchFile);
|
||||
}
|
||||
if ( assetDetails.fetch === false ) { return null; }
|
||||
const patchDetailsPromise = fetchPatchDetailsFromCDNs(assetDetails);
|
||||
patches.set(patchFile, patchDetailsPromise);
|
||||
return patchDetailsPromise;
|
||||
}
|
||||
|
||||
async function fetchAndApplyAllPatches(assetDetails) {
|
||||
const { diffURL, patchDetails } = await fetchPatchDetails(assetDetails);
|
||||
if ( patchDetails === null ) {
|
||||
assetDetails.error = 'nopatch';
|
||||
return assetDetails;
|
||||
}
|
||||
const diffDetails = patchDetails.get(assetDetails.diffName);
|
||||
if ( diffDetails === undefined ) {
|
||||
assetDetails.error = 'nodiff';
|
||||
return assetDetails;
|
||||
}
|
||||
if ( assetDetails.text === undefined ) {
|
||||
assetDetails.status = 'needtext';
|
||||
return assetDetails;
|
||||
}
|
||||
const outcome = await applyPatchAndValidate(assetDetails, diffDetails);
|
||||
if ( outcome !== true ) { return assetDetails; }
|
||||
assetDetails.status = 'updated';
|
||||
assetDetails.diffURL = diffURL;
|
||||
return assetDetails;
|
||||
}
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
const bc = new globalThis.BroadcastChannel('diffUpdater');
|
||||
|
||||
bc.onmessage = ev => {
|
||||
const message = ev.data;
|
||||
switch ( message.what ) {
|
||||
case 'update':
|
||||
fetchAndApplyAllPatches(message).then(response => {
|
||||
bc.postMessage(response);
|
||||
});
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
bc.postMessage({ what: 'ready' });
|
||||
|
||||
/******************************************************************************/
|
||||
|
|
@ -1009,39 +1009,20 @@ import {
|
|||
µb.extractFilterListMetadata = function(assetKey, raw) {
|
||||
const listEntry = this.availableFilterLists[assetKey];
|
||||
if ( listEntry === undefined ) { return; }
|
||||
// Metadata expected to be found at the top of content.
|
||||
const head = raw.slice(0, 1024);
|
||||
// https://github.com/gorhill/uBlock/issues/313
|
||||
// Always try to fetch the name if this is an external filter list.
|
||||
if ( listEntry.group === 'custom' ) {
|
||||
let matches = head.match(/(?:^|\n)(?:!|# )[\t ]*Title[\t ]*:([^\n]+)/i);
|
||||
const title = matches && matches[1].trim() || '';
|
||||
if ( title !== '' && title !== listEntry.title ) {
|
||||
listEntry.title = orphanizeString(title);
|
||||
io.registerAssetSource(assetKey, { title });
|
||||
}
|
||||
matches = head.match(/(?:^|\n)(?:!|# )[\t ]*Homepage[\t ]*:[\t ]*(https?:\/\/\S+)\s/i);
|
||||
const supportURL = matches && matches[1] || '';
|
||||
if ( supportURL !== '' && supportURL !== listEntry.supportURL ) {
|
||||
listEntry.supportURL = orphanizeString(supportURL);
|
||||
io.registerAssetSource(assetKey, { supportURL });
|
||||
}
|
||||
}
|
||||
// Extract update frequency information
|
||||
const matches = head.match(/(?:^|\n)(?:!|# )[\t ]*Expires[\t ]*:[\t ]*(\d+)[\t ]*(h)?/i);
|
||||
if ( matches !== null ) {
|
||||
let updateAfter = parseInt(matches[1], 10);
|
||||
if ( isNaN(updateAfter) === false ) {
|
||||
if ( matches[2] !== undefined ) {
|
||||
updateAfter = Math.ceil(updateAfter / 12) / 2;
|
||||
}
|
||||
updateAfter = Math.max(updateAfter, 0.5);
|
||||
if ( updateAfter !== listEntry.updateAfter ) {
|
||||
listEntry.updateAfter = updateAfter;
|
||||
io.registerAssetSource(assetKey, { updateAfter });
|
||||
}
|
||||
if ( listEntry.group !== 'custom' ) { return; }
|
||||
const data = io.extractMetadataFromList(raw, [ 'Title', 'Homepage' ]);
|
||||
const props = {};
|
||||
if ( data.title && data.title !== listEntry.title ) {
|
||||
props.title = listEntry.title = orphanizeString(data.title);
|
||||
}
|
||||
if ( data.homepage && /^https?:\/\/\S+/.test(data.homepage) ) {
|
||||
if ( data.homepage !== listEntry.supportURL ) {
|
||||
props.supportURL = listEntry.supportURL = orphanizeString(data.homepage);
|
||||
}
|
||||
}
|
||||
io.registerAssetSource(assetKey, props);
|
||||
};
|
||||
|
||||
/******************************************************************************/
|
||||
|
|
|
|||
Loading…
Reference in a new issue