external/uBlock
1
0
Fork 0
mirror of https://github.com/gorhill/uBlock.git synced 2024-11-13 02:14:17 +01:00
Code Issues Projects Releases Packages Wiki Activity

More fine tuning of cache storage-related code

Browse source
This commit is contained in:
Raymond Hill 2024-02-27 21:47:06 -05:00
parent a9211cfa2f
commit 79ea85dbc4
No known key found for this signature in database
GPG key ID: 25E1490B761470C2
4 changed files with 138 additions and 80 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
platform/chromium/webext.js
View file

@ -156,6 +156,15 @@ if ( chrome.storage.sync instanceof Object ) {
}; };
} }
// https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/storage/session
webext.storage.session = {
clear: ( ) => Promise.resolve(),
get: ( ) => Promise.resolve(),
getBytesInUse: ( ) => Promise.resolve(),
remove: ( ) => Promise.resolve(),
set: ( ) => Promise.resolve(),
};
// https://bugs.chromium.org/p/chromium/issues/detail?id=608854 // https://bugs.chromium.org/p/chromium/issues/detail?id=608854
if ( chrome.tabs.removeCSS instanceof Function ) { if ( chrome.tabs.removeCSS instanceof Function ) {
webext.tabs.removeCSS = promisifyNoFail(chrome.tabs, 'removeCSS'); webext.tabs.removeCSS = promisifyNoFail(chrome.tabs, 'removeCSS');

2
src/js/assets.js
View file

@ -838,7 +838,7 @@ async function assetCacheSetDetails(assetKey, details) {
} }
} }
if ( modified ) { if ( modified ) {
saveAssetCacheRegistry(); saveAssetCacheRegistry(3);
} }
} }

118
src/js/cachestorage.js
View file

@ -58,17 +58,19 @@ const shouldCache = bin => {
return out; return out;
}; };
const missingKeys = (wanted, inbin, outbin) => { const exGet = (api, wanted, outbin) => {
inbin = inbin || {}; return api.get(wanted).then(inbin => {
const found = Object.keys(inbin); inbin = inbin || {};
Object.assign(outbin, inbin); const found = Object.keys(inbin);
if ( found.length === wanted.length ) { return; } Object.assign(outbin, inbin);
const missing = []; if ( found.length === wanted.length ) { return; }
for ( const key of wanted ) { const missing = [];
if ( outbin.hasOwnProperty(key) ) { continue; } for ( const key of wanted ) {
missing.push(key); if ( outbin.hasOwnProperty(key) ) { continue; }
} missing.push(key);
return missing; }
return missing;
});
}; };
/******************************************************************************* /*******************************************************************************
@ -81,15 +83,15 @@ const missingKeys = (wanted, inbin, outbin) => {
const cacheStorage = (( ) => { const cacheStorage = (( ) => {
const compress = async (key, data) => { const compress = async (bin, key, data) => {
const µbhs = µb.hiddenSettings; const µbhs = µb.hiddenSettings;
const isLarge = typeof data === 'string' && const isLarge = typeof data === 'string' &&
data.length >= µbhs.cacheStorageCompressionThreshold; data.length >= µbhs.cacheStorageCompressionThreshold;
const after = await scuo.serializeAsync(data, { const after = await scuo.serializeAsync(data, {
compress: isLarge && µbhs.cacheStorageCompression, compress: isLarge && µbhs.cacheStorageCompression,
multithreaded: isLarge && µbhs.cacheStorageMultithread || 2, multithreaded: µbhs.cacheStorageMultithread,
}); });
return { key, data: after }; bin[key] = after;
}; };
const decompress = async (bin, key) => { const decompress = async (bin, key) => {
@ -98,27 +100,24 @@ const cacheStorage = (( ) => {
const µbhs = µb.hiddenSettings; const µbhs = µb.hiddenSettings;
const isLarge = data.length >= µbhs.cacheStorageCompressionThreshold; const isLarge = data.length >= µbhs.cacheStorageCompressionThreshold;
bin[key] = await scuo.deserializeAsync(data, { bin[key] = await scuo.deserializeAsync(data, {
multithreaded: isLarge && µbhs.cacheStorageMultithread || 2, multithreaded: isLarge && µbhs.cacheStorageMultithread || 1,
}); });
}; };
return { return {
get(argbin) { get(argbin) {
const outbin = {}; const outbin = {};
const wanted0 = keysFromGetArg(argbin); return exGet(cacheAPI, keysFromGetArg(argbin), outbin).then(wanted => {
return cacheAPI.get(wanted0).then(bin => { if ( wanted === undefined ) { return; }
const wanted1 = missingKeys(wanted0, bin, outbin); return exGet(extensionStorage, wanted, outbin);
if ( wanted1 === undefined ) { return; } }).then(wanted => {
return extensionStorage.get(wanted1).then(bin => { if ( wanted === undefined ) { return; }
const wanted2 = missingKeys(wanted1, bin, outbin); if ( argbin instanceof Object === false ) { return; }
if ( wanted2 === undefined ) { return; } if ( Array.isArray(argbin) ) { return; }
if ( argbin instanceof Object === false ) { return; } for ( const key of wanted ) {
if ( Array.isArray(argbin) ) { return; } if ( argbin.hasOwnProperty(key) === false ) { continue; }
for ( const key of wanted2 ) { outbin[key] = argbin[key];
if ( argbin.hasOwnProperty(key) === false ) { continue; } }
outbin[key] = argbin[key];
}
});
}).then(( ) => { }).then(( ) => {
const promises = []; const promises = [];
for ( const key of Object.keys(outbin) ) { for ( const key of Object.keys(outbin) ) {
@ -147,17 +146,14 @@ const cacheStorage = (( ) => {
async set(keyvalStore) { async set(keyvalStore) {
const keys = Object.keys(keyvalStore); const keys = Object.keys(keyvalStore);
if ( keys.length === 0 ) { return; } if ( keys.length === 0 ) { return; }
const bin = {};
const promises = []; const promises = [];
for ( const key of keys ) { for ( const key of keys ) {
promises.push(compress(key, keyvalStore[key])); promises.push(compress(bin, key, keyvalStore[key]));
} }
const results = await Promise.all(promises); await Promise.all(promises);
const serializedStore = {}; cacheAPI.set(shouldCache(bin));
for ( const { key, data } of results ) { return extensionStorage.set(bin).catch(reason => {
serializedStore[key] = data;
}
cacheAPI.set(shouldCache(serializedStore));
return extensionStorage.set(serializedStore).catch(reason => {
ubolog(reason); ubolog(reason);
}); });
}, },
@ -361,6 +357,54 @@ const cacheAPI = (( ) => {
}; };
})(); })();
/*******************************************************************************
*
* In-memory storage
*
* */
const memoryStorage = (( ) => {
const sessionStorage = webext.storage.session;
return {
get(...args) {
return sessionStorage.get(...args).catch(reason => {
ubolog(reason);
});
},
async keys(regex) {
const results = await sessionStorage.get(null).catch(( ) => {});
const keys = new Set(results[0]);
const bin = results[1] || {};
for ( const key of Object.keys(bin) ) {
if ( regex && regex.test(key) === false ) { continue; }
keys.add(key);
}
return keys;
},
async set(...args) {
return sessionStorage.set(...args).catch(reason => {
ubolog(reason);
});
},
remove(...args) {
return sessionStorage.remove(...args).catch(reason => {
ubolog(reason);
});
},
clear(...args) {
return sessionStorage.clear(...args).catch(reason => {
ubolog(reason);
});
},
};
})();
/******************************************************************************* /*******************************************************************************
* *
* IndexedDB * IndexedDB

89
src/js/scuo-serializer.js
View file

@ -1148,10 +1148,11 @@ const THREAD_DESERIALIZE = 4;
class MainThread { class MainThread {
constructor() { constructor() {
this.name = 'main';
this.jobs = []; this.jobs = [];
this.workload = 0; this.workload = 0;
this.timer = undefined; this.timer = undefined;
this.busy = false; this.busy = 2;
} }
process() { process() {
@ -1164,7 +1165,9 @@ class MainThread {
job.resolve(result); job.resolve(result);
this.processAsync(); this.processAsync();
if ( this.jobs.length === 0 ) { if ( this.jobs.length === 0 ) {
this.busy = false; this.busy = 2;
} else if ( this.busy > 2 ) {
this.busy -= 1;
} }
} }
@ -1174,11 +1177,12 @@ class MainThread {
this.timer = globalThis.requestIdleCallback(deadline => { this.timer = globalThis.requestIdleCallback(deadline => {
this.timer = undefined; this.timer = undefined;
globalThis.queueMicrotask(( ) => { globalThis.queueMicrotask(( ) => {
this.timer = undefined;
this.process(); this.process();
}); });
this.busy = deadline.timeRemaining() === 0; if ( deadline.timeRemaining() === 0 ) {
}, { timeout: 7 }); this.busy += 1;
}
}, { timeout: 5 });
} }
serialize(data, options) { serialize(data, options) {
@ -1199,16 +1203,17 @@ class MainThread {
} }
get queueSize() { get queueSize() {
return this.jobs.length + 1; return this.jobs.length;
} }
get workSize() { get workSize() {
return this.busy ? Number.MAX_SAFE_INTEGER : this.workload * 2; return this.workload * this.busy;
} }
} }
class Thread { class Thread {
constructor(gcer) { constructor(gcer) {
this.name = 'worker';
this.jobs = new Map(); this.jobs = new Map();
this.jobIdGenerator = 1; this.jobIdGenerator = 1;
this.workload = 0; this.workload = 0;
@ -1263,7 +1268,10 @@ class Thread {
} }
onmessage(ev) { onmessage(ev) {
const job = ev.data; this.ondone(ev.data);
}
ondone(job) {
const resolve = this.jobs.get(job.id); const resolve = this.jobs.get(job.id);
if ( resolve === undefined ) { return; } if ( resolve === undefined ) { return; }
this.jobs.delete(job.id); this.jobs.delete(job.id);
@ -1274,37 +1282,35 @@ class Thread {
} }
async serialize(data, options) { async serialize(data, options) {
this.workerAccessTime = Date.now();
const worker = await this.workerPromise;
if ( worker === null ) {
const result = serialize(data, options);
this.countdownWorker();
return result;
}
const id = this.jobIdGenerator++;
return new Promise(resolve => { return new Promise(resolve => {
const id = this.jobIdGenerator++;
this.workload += 1; this.workload += 1;
const job = { what: THREAD_SERIALIZE, id, data, options, size: 1 }; this.jobs.set(id, resolve);
this.jobs.set(job.id, resolve); return this.workerPromise.then(worker => {
worker.postMessage(job); this.workerAccessTime = Date.now();
if ( worker === null ) {
this.ondone({ id, result: serialize(data, options), size: 1 });
} else {
worker.postMessage({ what: THREAD_SERIALIZE, id, data, options, size: 1 });
}
});
}); });
} }
async deserialize(data, options) { async deserialize(data, options) {
this.workerAccessTime = Date.now();
const worker = await this.workerPromise;
if ( worker === null ) {
const result = deserialize(data, options);
this.countdownWorker();
return result;
}
const id = this.jobIdGenerator++;
return new Promise(resolve => { return new Promise(resolve => {
const id = this.jobIdGenerator++;
const size = data.length; const size = data.length;
this.workload += size; this.workload += size;
const job = { what: THREAD_DESERIALIZE, id, data, options, size }; this.jobs.set(id, resolve);
this.jobs.set(job.id, resolve); return this.workerPromise.then(worker => {
worker.postMessage(job); this.workerAccessTime = Date.now();
if ( worker === null ) {
this.ondone({ id, result: deserialize(data, options), size });
} else {
worker.postMessage({ what: THREAD_DESERIALIZE, id, data, options, size });
}
});
}); });
} }
@ -1323,12 +1329,11 @@ const threads = {
const poolSize = this.pool.length; const poolSize = this.pool.length;
if ( poolSize !== 0 && poolSize >= maxPoolSize ) { if ( poolSize !== 0 && poolSize >= maxPoolSize ) {
if ( poolSize === 1 ) { return this.pool[0]; } if ( poolSize === 1 ) { return this.pool[0]; }
return this.pool.reduce((best, candidate) => { return this.pool.reduce((a, b) => {
if ( candidate.queueSize === 0 ) { return candidate; } //console.log(`${a.name}: q=${a.queueSize} w=${a.workSize} ${b.name}: q=${b.queueSize} w=${b.workSize}`);
if ( best.queueSize === 0 ) { return best; } if ( b.queueSize === 0 ) { return b; }
return candidate.workSize < best.workSize if ( a.queueSize === 0 ) { return a; }
? candidate return b.workSize < a.workSize ? b : a;
: best;
}); });
} }
const thread = new Thread(thread => { const thread = new Thread(thread => {
@ -1346,9 +1351,9 @@ export async function serializeAsync(data, options = {}) {
if ( maxThreadCount === 0 ) { if ( maxThreadCount === 0 ) {
return serialize(data, options); return serialize(data, options);
} }
const result = await threads const thread = threads.thread(maxThreadCount);
.thread(maxThreadCount) //console.log(`serializeAsync: thread=${thread.name} workload=${thread.workSize}`);
.serialize(data, options); const result = await thread.serialize(data, options);
if ( result !== undefined ) { return result; } if ( result !== undefined ) { return result; }
return serialize(data, options); return serialize(data, options);
} }
@ -1359,9 +1364,9 @@ export async function deserializeAsync(data, options = {}) {
if ( maxThreadCount === 0 ) { if ( maxThreadCount === 0 ) {
return deserialize(data, options); return deserialize(data, options);
} }
const result = await threads const thread = threads.thread(maxThreadCount);
.thread(maxThreadCount) //console.log(`deserializeAsync: thread=${thread.name} data=${data.length} workload=${thread.workSize}`);
.deserialize(data, options); const result = await thread.deserialize(data, options);
if ( result !== undefined ) { return result; } if ( result !== undefined ) { return result; }
return deserialize(data, options); return deserialize(data, options);
} }