mirror of
https://github.com/gorhill/uBlock.git
synced 2024-11-10 09:07:54 +01:00
Ignore assets older than cached version when fetching from CDNs
As discussed with filter list volunteers. https://github.com/uBlockOrigin/uBlock-discussions/discussions/781#discussioncomment-7283981
This commit is contained in:
Raymond Hill
parent
f122ce7320
commit
7daf31336a
1 changed files with 150 additions and 43 deletions
193
src/js/assets.js
193
src/js/assets.js
|
|
@ -28,6 +28,7 @@ import logger from './logger.js';
|
|||
import µb from './background.js';
|
||||
import { i18n$ } from './i18n.js';
|
||||
import * as sfp from './static-filtering-parser.js';
|
||||
import { ubolog } from './console.js';
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
|
|
@ -41,6 +42,56 @@ const assets = {};
|
|||
// bandwidth of remote servers.
|
||||
let remoteServerFriendly = false;
|
||||
|
||||
const resourceTimeFromXhr = xhr => {
|
||||
try {
|
||||
// First lookup timestamp from content
|
||||
let assetTime = 0;
|
||||
if ( typeof xhr.response === 'string' ) {
|
||||
const head = xhr.response.slice(0, 512);
|
||||
const match = /^! Last modified: (.+)$/m.exec(head);
|
||||
if ( match ) {
|
||||
assetTime = (new Date(match[1])).getTime() || 0;
|
||||
}
|
||||
}
|
||||
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Age
|
||||
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Date
|
||||
let networkTime = 0;
|
||||
const age = parseInt(xhr.getResponseHeader('Age'), 10);
|
||||
if ( isNaN(age) === false ) {
|
||||
const time = (new Date(xhr.getResponseHeader('Date'))).getTime();
|
||||
if ( isNaN(time) === false ) {
|
||||
networkTime = time - age * 1000;
|
||||
}
|
||||
}
|
||||
return Math.max(assetTime, networkTime, 0);
|
||||
} catch(_) {
|
||||
}
|
||||
return 0;
|
||||
};
|
||||
|
||||
const resourceTimeFromParts = (parts, time) => {
|
||||
const goodParts = parts.filter(part => typeof part === 'object');
|
||||
return goodParts.reduce((acc, part) =>
|
||||
((part.resourceTime || 0) > acc ? part.resourceTime : acc),
|
||||
time
|
||||
);
|
||||
};
|
||||
|
||||
const resourceIsStale = (networkDetails, cacheDetails) => {
|
||||
if ( typeof networkDetails.resourceTime !== 'number' ) { return false; }
|
||||
if ( networkDetails.resourceTime === 0 ) { return false; }
|
||||
if ( typeof cacheDetails.resourceTime !== 'number' ) { return false; }
|
||||
if ( cacheDetails.resourceTime === 0 ) { return false; }
|
||||
if ( networkDetails.resourceTime === cacheDetails.resourceTime ) { return true; }
|
||||
if ( networkDetails.resourceTime < cacheDetails.resourceTime ) {
|
||||
ubolog(`Skip ${networkDetails.url}\n\tolder than ${cacheDetails.remoteURL}`);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
const stringIsNotEmpty = s => typeof s === 'string' && s !== '';
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
const observers = [];
|
||||
|
|
@ -109,6 +160,7 @@ assets.fetch = function(url, options = {}) {
|
|||
return fail(details, `${url}: ${details.statusCode} ${details.statusText}`);
|
||||
}
|
||||
details.content = this.response;
|
||||
details.resourceTime = resourceTimeFromXhr(this);
|
||||
resolve(details);
|
||||
};
|
||||
|
||||
|
|
@ -312,19 +364,21 @@ assets.fetchFilterList = async function(mainlistURL) {
|
|||
];
|
||||
// Abort processing `include` directives if at least one included sublist
|
||||
// can't be fetched.
|
||||
let resourceTime = 0;
|
||||
do {
|
||||
allParts = await Promise.all(allParts);
|
||||
const part = allParts.find(part => {
|
||||
return typeof part === 'object' && part.error !== undefined;
|
||||
});
|
||||
const part = allParts
|
||||
.find(part => typeof part === 'object' && part.error !== undefined);
|
||||
if ( part !== undefined ) {
|
||||
return { url: mainlistURL, content: '', error: part.error };
|
||||
}
|
||||
resourceTime = resourceTimeFromParts(allParts, resourceTime);
|
||||
allParts = processIncludeDirectives(allParts);
|
||||
} while ( allParts.some(part => typeof part !== 'string') );
|
||||
// If we reach this point, this means all fetches were successful.
|
||||
return {
|
||||
url: mainlistURL,
|
||||
resourceTime,
|
||||
content: allParts.length === 1
|
||||
? allParts[0]
|
||||
: allParts.join('') + '\n'
|
||||
|
|
@ -347,7 +401,7 @@ assets.fetchFilterList = async function(mainlistURL) {
|
|||
let assetSourceRegistryPromise;
|
||||
let assetSourceRegistry = Object.create(null);
|
||||
|
||||
const getAssetSourceRegistry = function() {
|
||||
function getAssetSourceRegistry() {
|
||||
if ( assetSourceRegistryPromise === undefined ) {
|
||||
assetSourceRegistryPromise = cacheStorage.get(
|
||||
'assetSourceRegistry'
|
||||
|
|
@ -373,9 +427,9 @@ const getAssetSourceRegistry = function() {
|
|||
}
|
||||
|
||||
return assetSourceRegistryPromise;
|
||||
};
|
||||
}
|
||||
|
||||
const registerAssetSource = function(assetKey, newDict) {
|
||||
function registerAssetSource(assetKey, newDict) {
|
||||
const currentDict = assetSourceRegistry[assetKey] || {};
|
||||
for ( const [ k, v ] of Object.entries(newDict) ) {
|
||||
if ( v === undefined || v === null ) {
|
||||
|
|
@ -409,12 +463,12 @@ const registerAssetSource = function(assetKey, newDict) {
|
|||
currentDict.submitTime = Date.now(); // To detect stale entries
|
||||
}
|
||||
assetSourceRegistry[assetKey] = currentDict;
|
||||
};
|
||||
}
|
||||
|
||||
const unregisterAssetSource = function(assetKey) {
|
||||
function unregisterAssetSource(assetKey) {
|
||||
assetCacheRemove(assetKey);
|
||||
delete assetSourceRegistry[assetKey];
|
||||
};
|
||||
}
|
||||
|
||||
const saveAssetSourceRegistry = (( ) => {
|
||||
const save = ( ) => {
|
||||
|
|
@ -431,7 +485,14 @@ const saveAssetSourceRegistry = (( ) => {
|
|||
};
|
||||
})();
|
||||
|
||||
const updateAssetSourceRegistry = function(json, silent = false) {
|
||||
async function assetSourceGetDetails(assetKey) {
|
||||
await getAssetSourceRegistry();
|
||||
const entry = assetSourceRegistry[assetKey];
|
||||
if ( entry === undefined ) { return; }
|
||||
return entry;
|
||||
}
|
||||
|
||||
function updateAssetSourceRegistry(json, silent = false) {
|
||||
let newDict;
|
||||
try {
|
||||
newDict = JSON.parse(json);
|
||||
|
|
@ -467,7 +528,7 @@ const updateAssetSourceRegistry = function(json, silent = false) {
|
|||
registerAssetSource(assetKey, newDict[assetKey]);
|
||||
}
|
||||
saveAssetSourceRegistry();
|
||||
};
|
||||
}
|
||||
|
||||
assets.registerAssetSource = async function(assetKey, details) {
|
||||
await getAssetSourceRegistry();
|
||||
|
|
@ -492,7 +553,7 @@ const assetCacheRegistryStartTime = Date.now();
|
|||
let assetCacheRegistryPromise;
|
||||
let assetCacheRegistry = {};
|
||||
|
||||
const getAssetCacheRegistry = function() {
|
||||
function getAssetCacheRegistry() {
|
||||
if ( assetCacheRegistryPromise === undefined ) {
|
||||
assetCacheRegistryPromise = cacheStorage.get(
|
||||
'assetCacheRegistry'
|
||||
|
|
@ -522,7 +583,7 @@ const getAssetCacheRegistry = function() {
|
|||
}
|
||||
|
||||
return assetCacheRegistryPromise;
|
||||
};
|
||||
}
|
||||
|
||||
const saveAssetCacheRegistry = (( ) => {
|
||||
const save = function() {
|
||||
|
|
@ -539,7 +600,7 @@ const saveAssetCacheRegistry = (( ) => {
|
|||
};
|
||||
})();
|
||||
|
||||
const assetCacheRead = async function(assetKey, updateReadTime = false) {
|
||||
async function assetCacheRead(assetKey, updateReadTime = false) {
|
||||
const t0 = Date.now();
|
||||
const internalKey = `cache/${assetKey}`;
|
||||
|
||||
|
|
@ -580,9 +641,9 @@ const assetCacheRead = async function(assetKey, updateReadTime = false) {
|
|||
}
|
||||
|
||||
return reportBack(bin[internalKey]);
|
||||
};
|
||||
}
|
||||
|
||||
const assetCacheWrite = async function(assetKey, details) {
|
||||
async function assetCacheWrite(assetKey, details) {
|
||||
let content = '';
|
||||
let options = {};
|
||||
if ( typeof details === 'string' ) {
|
||||
|
|
@ -603,6 +664,7 @@ const assetCacheWrite = async function(assetKey, details) {
|
|||
entry = cacheDict[assetKey] = {};
|
||||
}
|
||||
entry.writeTime = entry.readTime = Date.now();
|
||||
entry.resourceTime = options.resourceTime || 0;
|
||||
if ( typeof options.url === 'string' ) {
|
||||
entry.remoteURL = options.url;
|
||||
}
|
||||
|
|
@ -617,9 +679,9 @@ const assetCacheWrite = async function(assetKey, details) {
|
|||
fireNotification('after-asset-updated', result);
|
||||
}
|
||||
return result;
|
||||
};
|
||||
}
|
||||
|
||||
const assetCacheRemove = async function(pattern) {
|
||||
async function assetCacheRemove(pattern) {
|
||||
const cacheDict = await getAssetCacheRegistry();
|
||||
const removedEntries = [];
|
||||
const removedContent = [];
|
||||
|
|
@ -645,9 +707,39 @@ const assetCacheRemove = async function(pattern) {
|
|||
assetKey: removedEntries[i]
|
||||
});
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
const assetCacheMarkAsDirty = async function(pattern, exclude) {
|
||||
async function assetCacheGetDetails(assetKey) {
|
||||
const cacheDict = await getAssetCacheRegistry();
|
||||
const entry = cacheDict[assetKey];
|
||||
if ( entry === undefined ) { return; }
|
||||
return entry;
|
||||
}
|
||||
|
||||
async function assetCacheSetDetails(assetKey, details) {
|
||||
const cacheDict = await getAssetCacheRegistry();
|
||||
const entry = cacheDict[assetKey];
|
||||
if ( entry === undefined ) { return; }
|
||||
let modified = false;
|
||||
for ( const [ k, v ] of Object.entries(details) ) {
|
||||
if ( v === undefined ) {
|
||||
if ( entry[k] !== undefined ) {
|
||||
delete entry[k];
|
||||
modified = true;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if ( v !== entry[k] ) {
|
||||
entry[k] = v;
|
||||
modified = true;
|
||||
}
|
||||
}
|
||||
if ( modified ) {
|
||||
saveAssetCacheRegistry();
|
||||
}
|
||||
}
|
||||
|
||||
async function assetCacheMarkAsDirty(pattern, exclude) {
|
||||
const cacheDict = await getAssetCacheRegistry();
|
||||
let mustSave = false;
|
||||
for ( const assetKey in cacheDict ) {
|
||||
|
|
@ -673,13 +765,7 @@ const assetCacheMarkAsDirty = async function(pattern, exclude) {
|
|||
if ( mustSave ) {
|
||||
cacheStorage.set({ assetCacheRegistry });
|
||||
}
|
||||
};
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
const stringIsNotEmpty = function(s) {
|
||||
return typeof s === 'string' && s !== '';
|
||||
};
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
|
||||
|
|
@ -805,9 +891,17 @@ assets.get = async function(assetKey, options = {}) {
|
|||
|
||||
/******************************************************************************/
|
||||
|
||||
const getRemote = async function(assetKey) {
|
||||
const assetRegistry = await getAssetSourceRegistry();
|
||||
const assetDetails = assetRegistry[assetKey] || {};
|
||||
async function getRemote(assetKey) {
|
||||
const [
|
||||
assetDetails = {},
|
||||
cacheDetails = {},
|
||||
] = await Promise.all([
|
||||
assetSourceGetDetails(assetKey),
|
||||
assetCacheGetDetails(assetKey),
|
||||
]);
|
||||
|
||||
let error;
|
||||
let stale = false;
|
||||
|
||||
const reportBack = function(content, err) {
|
||||
const details = { assetKey, content };
|
||||
|
|
@ -865,27 +959,40 @@ const getRemote = async function(assetKey) {
|
|||
|
||||
// Failure
|
||||
if ( stringIsNotEmpty(result.content) === false ) {
|
||||
let error = result.statusText;
|
||||
error = result.statusText;
|
||||
if ( result.statusCode === 0 ) {
|
||||
error = 'network error';
|
||||
}
|
||||
registerAssetSource(assetKey, {
|
||||
error: { time: Date.now(), error }
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
error = undefined;
|
||||
|
||||
// If fetched resource is same older than cached one, ignore
|
||||
stale = resourceIsStale(result, cacheDetails);
|
||||
if ( stale ) { continue; }
|
||||
|
||||
// Success
|
||||
assetCacheWrite(assetKey, {
|
||||
content: result.content,
|
||||
url: contentURL
|
||||
url: contentURL,
|
||||
resourceTime: result.resourceTime || 0,
|
||||
});
|
||||
registerAssetSource(assetKey, { error: undefined });
|
||||
registerAssetSource(assetKey, { birthtime: undefined, error: undefined });
|
||||
return reportBack(result.content);
|
||||
}
|
||||
|
||||
return reportBack('', 'ENOTFOUND');
|
||||
};
|
||||
if ( error !== undefined ) {
|
||||
registerAssetSource(assetKey, { error: { time: Date.now(), error } });
|
||||
return reportBack('', 'ENOTFOUND');
|
||||
}
|
||||
|
||||
if ( stale ) {
|
||||
assetCacheSetDetails(assetKey, { writeTime: cacheDetails.resourceTime });
|
||||
}
|
||||
|
||||
return reportBack('');
|
||||
}
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
|
|
@ -1052,13 +1159,13 @@ const updateNext = async function() {
|
|||
|
||||
remoteServerFriendly = false;
|
||||
|
||||
if ( result.content !== '' ) {
|
||||
if ( result.error ) {
|
||||
fireNotification('asset-update-failed', { assetKey: result.assetKey });
|
||||
} else {
|
||||
updaterUpdated.push(result.assetKey);
|
||||
if ( result.assetKey === 'assets.json' ) {
|
||||
if ( result.assetKey === 'assets.json' && result.content !== '' ) {
|
||||
updateAssetSourceRegistry(result.content);
|
||||
}
|
||||
} else {
|
||||
fireNotification('asset-update-failed', { assetKey: result.assetKey });
|
||||
}
|
||||
|
||||
updaterTimer.on(updaterAssetDelay);
|
||||
|
|
@ -1072,7 +1179,7 @@ const updateDone = function() {
|
|||
updaterUpdated.length = 0;
|
||||
updaterStatus = undefined;
|
||||
updaterAssetDelay = updaterAssetDelayDefault;
|
||||
fireNotification('after-assets-updated', { assetKeys: assetKeys });
|
||||
fireNotification('after-assets-updated', { assetKeys });
|
||||
};
|
||||
|
||||
assets.updateStart = function(details) {
|
||||
|
|
|
|||
Loading…
Reference in a new issue