mirror of
https://github.com/gorhill/uBlock.git
synced 2024-11-10 09:07:54 +01:00
reorganize cache storage compression; workaround fix for #2812
This commit is contained in:
Raymond Hill
parent
8f1b4b52fd
commit
38aabc937a
13 changed files with 1002 additions and 313 deletions
|
|
@ -58,7 +58,7 @@
|
|||
},
|
||||
"incognito": "split",
|
||||
"manifest_version": 2,
|
||||
"minimum_chrome_version": "45.0",
|
||||
"minimum_chrome_version": "47.0",
|
||||
"name": "uBlock Origin",
|
||||
"optional_permissions": [
|
||||
"file:///*"
|
||||
|
|
|
|||
|
|
@ -5,17 +5,19 @@
|
|||
<title>uBlock Origin</title>
|
||||
</head>
|
||||
<body>
|
||||
<script src="lib/lz4/lz4-block-codec-any.js"></script>
|
||||
<script src="lib/punycode.js"></script>
|
||||
<script src="lib/publicsuffixlist.js"></script>
|
||||
<script src="js/vapi.js"></script>
|
||||
<script src="js/vapi-common.js"></script>
|
||||
<script src="js/vapi-background.js"></script>
|
||||
<script src="js/vapi-webrequest.js"></script><!-- Forks can pick the webext, chromium, or their own implementation -->
|
||||
<script src="js/vapi-cachestorage.js"></script><!-- Optional -->
|
||||
<script src="js/background.js"></script>
|
||||
<script src="js/hntrie.js"></script>
|
||||
<script src="js/utils.js"></script>
|
||||
<script src="js/uritools.js"></script>
|
||||
<script src="js/lz4.js"></script>
|
||||
<script src="js/cachestorage.js"></script>
|
||||
<script src="js/assets.js"></script>
|
||||
<script src="js/redirect-engine.js"></script>
|
||||
<script src="js/dynamic-net-filtering.js"></script>
|
||||
|
|
|
|||
282
src/js/assets.js
282
src/js/assets.js
|
|
@ -19,8 +19,6 @@
|
|||
Home: https://github.com/gorhill/uBlock
|
||||
*/
|
||||
|
||||
/* global WebAssembly */
|
||||
|
||||
'use strict';
|
||||
|
||||
/******************************************************************************/
|
||||
|
|
@ -304,7 +302,7 @@ var saveAssetSourceRegistry = (function() {
|
|||
var timer;
|
||||
var save = function() {
|
||||
timer = undefined;
|
||||
vAPI.cacheStorage.set({ assetSourceRegistry: assetSourceRegistry });
|
||||
µBlock.cacheStorage.set({ assetSourceRegistry: assetSourceRegistry });
|
||||
};
|
||||
return function(lazily) {
|
||||
if ( timer !== undefined ) {
|
||||
|
|
@ -387,7 +385,7 @@ var getAssetSourceRegistry = function(callback) {
|
|||
);
|
||||
};
|
||||
|
||||
vAPI.cacheStorage.get('assetSourceRegistry', function(bin) {
|
||||
µBlock.cacheStorage.get('assetSourceRegistry', function(bin) {
|
||||
if ( !bin || !bin.assetSourceRegistry ) {
|
||||
createRegistry();
|
||||
return;
|
||||
|
|
@ -411,247 +409,6 @@ api.unregisterAssetSource = function(assetKey) {
|
|||
});
|
||||
};
|
||||
|
||||
/*******************************************************************************
|
||||
|
||||
Experimental support for cache storage compression.
|
||||
|
||||
For background information on the topic, see:
|
||||
https://github.com/uBlockOrigin/uBlock-issues/issues/141#issuecomment-407737186
|
||||
|
||||
**/
|
||||
|
||||
let lz4Codec = (function() {
|
||||
let lz4wasmInstance;
|
||||
let pendingInitialization;
|
||||
let textEncoder, textDecoder;
|
||||
let ttlCount = 0;
|
||||
let ttlTimer;
|
||||
|
||||
const ttlDelay = 60 * 1000;
|
||||
|
||||
let init = function() {
|
||||
if (
|
||||
lz4wasmInstance === null ||
|
||||
WebAssembly instanceof Object === false ||
|
||||
typeof WebAssembly.instantiateStreaming !== 'function'
|
||||
) {
|
||||
lz4wasmInstance = null;
|
||||
return Promise.resolve(null);
|
||||
}
|
||||
if ( lz4wasmInstance instanceof WebAssembly.Instance ) {
|
||||
return Promise.resolve(lz4wasmInstance);
|
||||
}
|
||||
if ( pendingInitialization === undefined ) {
|
||||
pendingInitialization = WebAssembly.instantiateStreaming(
|
||||
fetch('lib/lz4-block-codec.wasm', { mode: 'same-origin' })
|
||||
).then(result => {
|
||||
pendingInitialization = undefined;
|
||||
lz4wasmInstance = result && result.instance || null;
|
||||
});
|
||||
pendingInitialization.catch(( ) => {
|
||||
lz4wasmInstance = null;
|
||||
});
|
||||
}
|
||||
return pendingInitialization;
|
||||
};
|
||||
|
||||
// We can't shrink memory usage of wasm instances, and in the current
|
||||
// case memory usage can grow to a significant amount given that
|
||||
// a single contiguous memory buffer is required to accommodate both
|
||||
// input and output data. Thus a time-to-live implementation which
|
||||
// will cause the wasm instance to be forgotten after enough time
|
||||
// elapse without the instance being used.
|
||||
|
||||
let destroy = function() {
|
||||
console.info(
|
||||
'uBO: freeing lz4-block-codec.wasm instance (memory.buffer = %d kB)',
|
||||
lz4wasmInstance.exports.memory.buffer.byteLength >>> 10
|
||||
);
|
||||
lz4wasmInstance = undefined;
|
||||
textEncoder = textDecoder = undefined;
|
||||
ttlCount = 0;
|
||||
ttlTimer = undefined;
|
||||
};
|
||||
|
||||
let ttlManage = function(count) {
|
||||
if ( ttlTimer !== undefined ) {
|
||||
clearTimeout(ttlTimer);
|
||||
ttlTimer = undefined;
|
||||
}
|
||||
ttlCount += count;
|
||||
if ( ttlCount > 0 ) { return; }
|
||||
if ( lz4wasmInstance === null ) { return; }
|
||||
ttlTimer = vAPI.setTimeout(destroy, ttlDelay);
|
||||
};
|
||||
|
||||
let growMemoryTo = function(byteLength) {
|
||||
let lz4api = lz4wasmInstance.exports;
|
||||
let neededByteLength = lz4api.getLinearMemoryOffset() + byteLength;
|
||||
let pageCountBefore = lz4api.memory.buffer.byteLength >>> 16;
|
||||
let pageCountAfter = (neededByteLength + 65535) >>> 16;
|
||||
if ( pageCountAfter > pageCountBefore ) {
|
||||
lz4api.memory.grow(pageCountAfter - pageCountBefore);
|
||||
}
|
||||
return lz4api.memory;
|
||||
};
|
||||
|
||||
let resolveEncodedValue = function(resolve, key, value) {
|
||||
let t0 = window.performance.now();
|
||||
let lz4api = lz4wasmInstance.exports;
|
||||
let mem0 = lz4api.getLinearMemoryOffset();
|
||||
let memory = growMemoryTo(mem0 + 65536 * 4);
|
||||
let hashTable = new Int32Array(memory.buffer, mem0, 65536);
|
||||
hashTable.fill(-65536, 0, 65536);
|
||||
let hashTableSize = hashTable.byteLength;
|
||||
if ( textEncoder === undefined ) {
|
||||
textEncoder = new TextEncoder();
|
||||
}
|
||||
let inputArray = textEncoder.encode(value);
|
||||
let inputSize = inputArray.byteLength;
|
||||
let memSize =
|
||||
hashTableSize +
|
||||
inputSize +
|
||||
8 + lz4api.lz4BlockEncodeBound(inputSize);
|
||||
memory = growMemoryTo(memSize);
|
||||
let inputMem = new Uint8Array(
|
||||
memory.buffer,
|
||||
mem0 + hashTableSize,
|
||||
inputSize
|
||||
);
|
||||
inputMem.set(inputArray);
|
||||
let outputSize = lz4api.lz4BlockEncode(
|
||||
mem0 + hashTableSize,
|
||||
inputSize,
|
||||
mem0 + hashTableSize + inputSize + 8
|
||||
);
|
||||
if ( outputSize === 0 ) { resolve(value); }
|
||||
let outputMem = new Uint8Array(
|
||||
memory.buffer,
|
||||
mem0 + hashTableSize + inputSize,
|
||||
8 + outputSize
|
||||
);
|
||||
outputMem[0] = 0x18;
|
||||
outputMem[1] = 0x4D;
|
||||
outputMem[2] = 0x22;
|
||||
outputMem[3] = 0x04;
|
||||
outputMem[4] = (inputSize >>> 0) & 0xFF;
|
||||
outputMem[5] = (inputSize >>> 8) & 0xFF;
|
||||
outputMem[6] = (inputSize >>> 16) & 0xFF;
|
||||
outputMem[7] = (inputSize >>> 24) & 0xFF;
|
||||
console.info(
|
||||
'uBO: [%s] compressed %d bytes into %d bytes in %s ms',
|
||||
key,
|
||||
inputSize,
|
||||
outputSize,
|
||||
(window.performance.now() - t0).toFixed(2)
|
||||
);
|
||||
resolve(new Blob([ outputMem ]));
|
||||
};
|
||||
|
||||
let resolveDecodedValue = function(resolve, ev, key, value) {
|
||||
let inputBuffer = ev.target.result;
|
||||
if ( inputBuffer instanceof ArrayBuffer === false ) {
|
||||
return resolve(value);
|
||||
}
|
||||
let t0 = window.performance.now();
|
||||
let metadata = new Uint8Array(inputBuffer, 0, 8);
|
||||
if (
|
||||
metadata[0] !== 0x18 ||
|
||||
metadata[1] !== 0x4D ||
|
||||
metadata[2] !== 0x22 ||
|
||||
metadata[3] !== 0x04
|
||||
) {
|
||||
return resolve(value);
|
||||
}
|
||||
let inputSize = inputBuffer.byteLength - 8;
|
||||
let outputSize =
|
||||
(metadata[4] << 0) |
|
||||
(metadata[5] << 8) |
|
||||
(metadata[6] << 16) |
|
||||
(metadata[7] << 24);
|
||||
let lz4api = lz4wasmInstance.exports;
|
||||
let mem0 = lz4api.getLinearMemoryOffset();
|
||||
let memSize = inputSize + outputSize;
|
||||
let memory = growMemoryTo(memSize);
|
||||
let inputArea = new Uint8Array(
|
||||
memory.buffer,
|
||||
mem0,
|
||||
inputSize
|
||||
);
|
||||
inputArea.set(new Uint8Array(inputBuffer, 8, inputSize));
|
||||
outputSize = lz4api.lz4BlockDecode(inputSize);
|
||||
if ( outputSize === 0 ) {
|
||||
return resolve(value);
|
||||
}
|
||||
let outputArea = new Uint8Array(
|
||||
memory.buffer,
|
||||
mem0 + inputSize,
|
||||
outputSize
|
||||
);
|
||||
if ( textDecoder === undefined ) {
|
||||
textDecoder = new TextDecoder();
|
||||
}
|
||||
value = textDecoder.decode(outputArea);
|
||||
console.info(
|
||||
'uBO: [%s] decompressed %d bytes into %d bytes in %s ms',
|
||||
key,
|
||||
inputSize,
|
||||
outputSize,
|
||||
(window.performance.now() - t0).toFixed(2)
|
||||
);
|
||||
resolve(value);
|
||||
};
|
||||
|
||||
let encodeValue = function(key, value) {
|
||||
if ( !lz4wasmInstance ) {
|
||||
return Promise.resolve(value);
|
||||
}
|
||||
return new Promise(resolve => {
|
||||
resolveEncodedValue(resolve, key, value);
|
||||
});
|
||||
};
|
||||
|
||||
let decodeValue = function(key, value) {
|
||||
if ( !lz4wasmInstance ) {
|
||||
return Promise.resolve(value);
|
||||
}
|
||||
return new Promise(resolve => {
|
||||
let blobReader = new FileReader();
|
||||
blobReader.onloadend = ev => {
|
||||
resolveDecodedValue(resolve, ev, key, value);
|
||||
};
|
||||
blobReader.readAsArrayBuffer(value);
|
||||
});
|
||||
};
|
||||
|
||||
return {
|
||||
encode: function(key, value) {
|
||||
if ( typeof value !== 'string' || value.length < 4096 ) {
|
||||
return Promise.resolve(value);
|
||||
}
|
||||
ttlManage(1);
|
||||
return init().then(( ) => {
|
||||
return encodeValue(key, value);
|
||||
}).then(result => {
|
||||
ttlManage(-1);
|
||||
return result;
|
||||
});
|
||||
},
|
||||
decode: function(key, value) {
|
||||
if ( value instanceof Blob === false ) {
|
||||
return Promise.resolve(value);
|
||||
}
|
||||
ttlManage(1);
|
||||
return init().then(( ) => {
|
||||
return decodeValue(key, value);
|
||||
}).then(result => {
|
||||
ttlManage(-1);
|
||||
return result;
|
||||
});
|
||||
}
|
||||
};
|
||||
})();
|
||||
|
||||
/*******************************************************************************
|
||||
|
||||
The purpose of the asset cache registry is to keep track of all assets
|
||||
|
|
@ -688,7 +445,7 @@ var getAssetCacheRegistry = function(callback) {
|
|||
}
|
||||
};
|
||||
|
||||
vAPI.cacheStorage.get('assetCacheRegistry', function(bin) {
|
||||
µBlock.cacheStorage.get('assetCacheRegistry', function(bin) {
|
||||
if ( bin && bin.assetCacheRegistry ) {
|
||||
assetCacheRegistry = bin.assetCacheRegistry;
|
||||
}
|
||||
|
|
@ -700,7 +457,7 @@ var saveAssetCacheRegistry = (function() {
|
|||
var timer;
|
||||
var save = function() {
|
||||
timer = undefined;
|
||||
vAPI.cacheStorage.set({ assetCacheRegistry: assetCacheRegistry });
|
||||
µBlock.cacheStorage.set({ assetCacheRegistry: assetCacheRegistry });
|
||||
};
|
||||
return function(lazily) {
|
||||
if ( timer !== undefined ) { clearTimeout(timer); }
|
||||
|
|
@ -735,16 +492,11 @@ var assetCacheRead = function(assetKey, callback) {
|
|||
}
|
||||
entry.readTime = Date.now();
|
||||
saveAssetCacheRegistry(true);
|
||||
if ( µBlock.hiddenSettings.cacheStorageCompression !== true ) {
|
||||
return reportBack(bin[internalKey]);
|
||||
}
|
||||
lz4Codec.decode(internalKey, bin[internalKey]).then(result => {
|
||||
reportBack(result);
|
||||
});
|
||||
reportBack(bin[internalKey]);
|
||||
};
|
||||
|
||||
let onReady = function() {
|
||||
vAPI.cacheStorage.get(internalKey, onAssetRead);
|
||||
µBlock.cacheStorage.get(internalKey, onAssetRead);
|
||||
};
|
||||
|
||||
getAssetCacheRegistry(onReady);
|
||||
|
|
@ -763,10 +515,7 @@ var assetCacheWrite = function(assetKey, details, callback) {
|
|||
return assetCacheRemove(assetKey, callback);
|
||||
}
|
||||
|
||||
let reportBack = function(content) {
|
||||
let bin = { assetCacheRegistry: assetCacheRegistry };
|
||||
bin[internalKey] = content;
|
||||
vAPI.cacheStorage.set(bin);
|
||||
let reportBack = function() {
|
||||
let details = { assetKey: assetKey, content: content };
|
||||
if ( typeof callback === 'function' ) {
|
||||
callback(details);
|
||||
|
|
@ -783,12 +532,9 @@ var assetCacheWrite = function(assetKey, details, callback) {
|
|||
if ( details instanceof Object && typeof details.url === 'string' ) {
|
||||
entry.remoteURL = details.url;
|
||||
}
|
||||
if ( µBlock.hiddenSettings.cacheStorageCompression !== true ) {
|
||||
return reportBack(content);
|
||||
}
|
||||
lz4Codec.encode(internalKey, content).then(result => {
|
||||
reportBack(result);
|
||||
});
|
||||
let bin = { assetCacheRegistry: assetCacheRegistry };
|
||||
bin[internalKey] = content;
|
||||
µBlock.cacheStorage.set(bin, reportBack);
|
||||
};
|
||||
getAssetCacheRegistry(onReady);
|
||||
};
|
||||
|
|
@ -810,9 +556,9 @@ var assetCacheRemove = function(pattern, callback) {
|
|||
delete cacheDict[assetKey];
|
||||
}
|
||||
if ( removedContent.length !== 0 ) {
|
||||
vAPI.cacheStorage.remove(removedContent);
|
||||
µBlock.cacheStorage.remove(removedContent);
|
||||
var bin = { assetCacheRegistry: assetCacheRegistry };
|
||||
vAPI.cacheStorage.set(bin);
|
||||
µBlock.cacheStorage.set(bin);
|
||||
}
|
||||
if ( typeof callback === 'function' ) {
|
||||
callback();
|
||||
|
|
@ -852,7 +598,7 @@ var assetCacheMarkAsDirty = function(pattern, exclude, callback) {
|
|||
}
|
||||
if ( mustSave ) {
|
||||
var bin = { assetCacheRegistry: assetCacheRegistry };
|
||||
vAPI.cacheStorage.set(bin);
|
||||
µBlock.cacheStorage.set(bin);
|
||||
}
|
||||
if ( typeof callback === 'function' ) {
|
||||
callback();
|
||||
|
|
@ -892,7 +638,7 @@ var readUserAsset = function(assetKey, callback) {
|
|||
var content = '';
|
||||
if ( typeof bin['cached_asset_content://assets/user/filters.txt'] === 'string' ) {
|
||||
content = bin['cached_asset_content://assets/user/filters.txt'];
|
||||
vAPI.cacheStorage.remove('cached_asset_content://assets/user/filters.txt');
|
||||
µBlock.cacheStorage.remove('cached_asset_content://assets/user/filters.txt');
|
||||
}
|
||||
if ( typeof bin['assets/user/filters.txt'] === 'string' ) {
|
||||
content = bin['assets/user/filters.txt'];
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@
|
|||
// has been added, for seamless migration of cache-related entries into
|
||||
// indexedDB.
|
||||
|
||||
vAPI.cacheStorage = (function() {
|
||||
µBlock.cacheStorage = (function() {
|
||||
|
||||
// Firefox-specific: we use indexedDB because chrome.storage.local() has
|
||||
// poor performance in Firefox. See:
|
||||
|
|
@ -50,6 +50,7 @@ vAPI.cacheStorage = (function() {
|
|||
const STORAGE_NAME = 'uBlock0CacheStorage';
|
||||
let db;
|
||||
let pendingInitialization;
|
||||
let dbByteLength;
|
||||
|
||||
let get = function get(input, callback) {
|
||||
if ( typeof callback !== 'function' ) { return; }
|
||||
|
|
@ -81,11 +82,17 @@ vAPI.cacheStorage = (function() {
|
|||
};
|
||||
|
||||
let getBytesInUse = function getBytesInUse(keys, callback) {
|
||||
// TODO: implement this
|
||||
callback(0);
|
||||
getDbSize(callback);
|
||||
};
|
||||
|
||||
let api = { get, set, remove, clear, getBytesInUse, error: undefined };
|
||||
let api = {
|
||||
get,
|
||||
set,
|
||||
remove,
|
||||
clear,
|
||||
getBytesInUse,
|
||||
error: undefined
|
||||
};
|
||||
|
||||
let genericErrorHandler = function(ev) {
|
||||
let error = ev.target && ev.target.error;
|
||||
|
|
@ -158,20 +165,33 @@ vAPI.cacheStorage = (function() {
|
|||
return pendingInitialization;
|
||||
};
|
||||
|
||||
let getFromDb = function(keys, keystore, callback) {
|
||||
let getFromDb = function(keys, keyvalStore, callback) {
|
||||
if ( typeof callback !== 'function' ) { return; }
|
||||
if ( keys.length === 0 ) { return callback(keystore); }
|
||||
if ( keys.length === 0 ) { return callback(keyvalStore); }
|
||||
let promises = [];
|
||||
let gotOne = function() {
|
||||
if ( typeof this.result === 'object' ) {
|
||||
keystore[this.result.key] = this.result.value;
|
||||
}
|
||||
if ( typeof this.result !== 'object' ) { return; }
|
||||
keyvalStore[this.result.key] = this.result.value;
|
||||
if ( this.result.value instanceof Blob === false ) { return; }
|
||||
promises.push(
|
||||
µBlock.lz4Codec.decode(
|
||||
this.result.key,
|
||||
this.result.value
|
||||
).then(result => {
|
||||
keyvalStore[result.key] = result.data;
|
||||
})
|
||||
);
|
||||
};
|
||||
getDb().then(( ) => {
|
||||
if ( !db ) { return callback(); }
|
||||
let transaction = db.transaction(STORAGE_NAME);
|
||||
transaction.oncomplete =
|
||||
transaction.onerror =
|
||||
transaction.onabort = ( ) => callback(keystore);
|
||||
transaction.onabort = ( ) => {
|
||||
Promise.all(promises).then(( ) => {
|
||||
callback(keyvalStore);
|
||||
});
|
||||
};
|
||||
let table = transaction.objectStore(STORAGE_NAME);
|
||||
for ( let key of keys ) {
|
||||
let req = table.get(key);
|
||||
|
|
@ -182,43 +202,110 @@ vAPI.cacheStorage = (function() {
|
|||
});
|
||||
};
|
||||
|
||||
let getAllFromDb = function(callback) {
|
||||
if ( typeof callback !== 'function' ) {
|
||||
callback = noopfn;
|
||||
}
|
||||
let visitAllFromDb = function(visitFn) {
|
||||
getDb().then(( ) => {
|
||||
if ( !db ) { return callback(); }
|
||||
let keystore = {};
|
||||
if ( !db ) { return visitFn(); }
|
||||
let transaction = db.transaction(STORAGE_NAME);
|
||||
transaction.oncomplete =
|
||||
transaction.onerror =
|
||||
transaction.onabort = ( ) => callback(keystore);
|
||||
let table = transaction.objectStore(STORAGE_NAME),
|
||||
req = table.openCursor();
|
||||
transaction.onabort = ( ) => visitFn();
|
||||
let table = transaction.objectStore(STORAGE_NAME);
|
||||
let req = table.openCursor();
|
||||
req.onsuccess = function(ev) {
|
||||
let cursor = ev.target.result;
|
||||
let cursor = ev.target && ev.target.result;
|
||||
if ( !cursor ) { return; }
|
||||
keystore[cursor.key] = cursor.value;
|
||||
let entry = cursor.value;
|
||||
visitFn(entry);
|
||||
cursor.continue();
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
let getAllFromDb = function(callback) {
|
||||
if ( typeof callback !== 'function' ) { return; }
|
||||
let promises = [];
|
||||
let keyvalStore = {};
|
||||
visitAllFromDb(entry => {
|
||||
if ( entry === undefined ) {
|
||||
Promise.all(promises).then(( ) => {
|
||||
callback(keyvalStore);
|
||||
});
|
||||
return;
|
||||
}
|
||||
keyvalStore[entry.key] = entry.value;
|
||||
if ( entry.value instanceof Blob === false ) { return; }
|
||||
promises.push(
|
||||
µBlock.lz4Codec.decode(
|
||||
entry.key,
|
||||
entry.value
|
||||
).then(result => {
|
||||
keyvalStore[result.key] = result.value;
|
||||
})
|
||||
);
|
||||
});
|
||||
};
|
||||
|
||||
let getDbSize = function(callback) {
|
||||
if ( typeof callback !== 'function' ) { return; }
|
||||
if ( typeof dbByteLength === 'number' ) {
|
||||
return Promise.resolve().then(( ) => {
|
||||
callback(dbByteLength);
|
||||
});
|
||||
}
|
||||
let textEncoder = new TextEncoder();
|
||||
let totalByteLength = 0;
|
||||
visitAllFromDb(entry => {
|
||||
if ( entry === undefined ) {
|
||||
dbByteLength = totalByteLength;
|
||||
return callback(totalByteLength);
|
||||
}
|
||||
let value = entry.value;
|
||||
if ( typeof value === 'string' ) {
|
||||
totalByteLength += textEncoder.encode(value).byteLength;
|
||||
} else if ( value instanceof Blob ) {
|
||||
totalByteLength += value.size;
|
||||
} else {
|
||||
totalByteLength += textEncoder.encode(JSON.stringify(value)).byteLength;
|
||||
}
|
||||
if ( typeof entry.key === 'string' ) {
|
||||
totalByteLength += textEncoder.encode(entry.key).byteLength;
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
|
||||
// https://github.com/uBlockOrigin/uBlock-issues/issues/141
|
||||
// Mind that IDBDatabase.transaction() and IDBObjectStore.put()
|
||||
// can throw:
|
||||
// https://developer.mozilla.org/en-US/docs/Web/API/IDBDatabase/transaction
|
||||
// https://developer.mozilla.org/en-US/docs/Web/API/IDBObjectStore/put
|
||||
|
||||
let putToDb = function(keystore, callback) {
|
||||
let putToDb = function(keyvalStore, callback) {
|
||||
if ( typeof callback !== 'function' ) {
|
||||
callback = noopfn;
|
||||
}
|
||||
let keys = Object.keys(keystore);
|
||||
let keys = Object.keys(keyvalStore);
|
||||
if ( keys.length === 0 ) { return callback(); }
|
||||
getDb().then(( ) => {
|
||||
let promises = [ getDb() ];
|
||||
let entries = [];
|
||||
let dontCompress = µBlock.hiddenSettings.cacheStorageCompression !== true;
|
||||
let handleEncodingResult = result => {
|
||||
entries.push({ key: result.key, value: result.data });
|
||||
};
|
||||
for ( let key of keys ) {
|
||||
let data = keyvalStore[key];
|
||||
if ( typeof data !== 'string' || dontCompress ) {
|
||||
entries.push({ key, value: data });
|
||||
continue;
|
||||
}
|
||||
promises.push(
|
||||
µBlock.lz4Codec.encode(key, data).then(handleEncodingResult)
|
||||
);
|
||||
}
|
||||
Promise.all(promises).then(( ) => {
|
||||
if ( !db ) { return callback(); }
|
||||
let finish = ( ) => {
|
||||
dbByteLength = undefined;
|
||||
if ( callback === undefined ) { return; }
|
||||
let cb = callback;
|
||||
callback = undefined;
|
||||
|
|
@ -230,12 +317,8 @@ vAPI.cacheStorage = (function() {
|
|||
transaction.onerror =
|
||||
transaction.onabort = finish;
|
||||
let table = transaction.objectStore(STORAGE_NAME);
|
||||
for ( let key of keys ) {
|
||||
let entry = {};
|
||||
entry.key = key;
|
||||
entry.value = keystore[key];
|
||||
for ( let entry of entries ) {
|
||||
table.put(entry);
|
||||
entry = undefined;
|
||||
}
|
||||
} catch (ex) {
|
||||
finish();
|
||||
|
|
@ -252,6 +335,7 @@ vAPI.cacheStorage = (function() {
|
|||
getDb().then(db => {
|
||||
if ( !db ) { return callback(); }
|
||||
let finish = ( ) => {
|
||||
dbByteLength = undefined;
|
||||
if ( callback === undefined ) { return; }
|
||||
let cb = callback;
|
||||
callback = undefined;
|
||||
|
|
@ -279,6 +363,7 @@ vAPI.cacheStorage = (function() {
|
|||
getDb().then(db => {
|
||||
if ( !db ) { return callback(); }
|
||||
let finish = ( ) => {
|
||||
dbByteLength = undefined;
|
||||
if ( callback === undefined ) { return; }
|
||||
let cb = callback;
|
||||
callback = undefined;
|
||||
193
src/js/lz4.js
Normal file
193
src/js/lz4.js
Normal file
|
|
@ -0,0 +1,193 @@
|
|||
/*******************************************************************************
|
||||
|
||||
uBlock Origin - a browser extension to block requests.
|
||||
Copyright (C) 2018-present Raymond Hill
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see {http://www.gnu.org/licenses/}.
|
||||
|
||||
Home: https://github.com/gorhill/uBlock
|
||||
*/
|
||||
|
||||
/* global lz4BlockCodec */
|
||||
|
||||
'use strict';
|
||||
|
||||
/*******************************************************************************
|
||||
|
||||
Experimental support for storage compression.
|
||||
|
||||
For background information on the topic, see:
|
||||
https://github.com/uBlockOrigin/uBlock-issues/issues/141#issuecomment-407737186
|
||||
|
||||
**/
|
||||
|
||||
µBlock.lz4Codec = (function() { // >>>> Start of private namespace
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
let lz4CodecInstance;
|
||||
let pendingInitialization;
|
||||
let textEncoder, textDecoder;
|
||||
let ttlCount = 0;
|
||||
let ttlTimer;
|
||||
|
||||
const ttlDelay = 60 * 1000;
|
||||
|
||||
let init = function() {
|
||||
if ( lz4CodecInstance === null ) {
|
||||
return Promise.resolve(null);
|
||||
}
|
||||
if ( lz4CodecInstance !== undefined ) {
|
||||
return Promise.resolve(lz4CodecInstance);
|
||||
}
|
||||
if ( pendingInitialization === undefined ) {
|
||||
pendingInitialization = lz4BlockCodec.createInstance()
|
||||
.then(instance => {
|
||||
lz4CodecInstance = instance;
|
||||
pendingInitialization = undefined;
|
||||
});
|
||||
}
|
||||
return pendingInitialization;
|
||||
};
|
||||
|
||||
// We can't shrink memory usage of lz4 codec instances, and in the
|
||||
// current case memory usage can grow to a significant amount given
|
||||
// that a single contiguous memory buffer is required to accommodate
|
||||
// both input and output data. Thus a time-to-live implementation
|
||||
// which will cause the wasm instance to be forgotten after enough
|
||||
// time elapse without the instance being used.
|
||||
|
||||
let destroy = function() {
|
||||
console.info('uBO: freeing lz4-block-codec instance');
|
||||
lz4CodecInstance = undefined;
|
||||
textEncoder = textDecoder = undefined;
|
||||
ttlCount = 0;
|
||||
ttlTimer = undefined;
|
||||
};
|
||||
|
||||
let ttlManage = function(count) {
|
||||
if ( ttlTimer !== undefined ) {
|
||||
clearTimeout(ttlTimer);
|
||||
ttlTimer = undefined;
|
||||
}
|
||||
ttlCount += count;
|
||||
if ( ttlCount > 0 ) { return; }
|
||||
if ( lz4CodecInstance === null ) { return; }
|
||||
ttlTimer = vAPI.setTimeout(destroy, ttlDelay);
|
||||
};
|
||||
|
||||
let uint8ArrayFromBlob = function(key, data) {
|
||||
if ( data instanceof Blob === false ) {
|
||||
return Promise.resolve({ key, data });
|
||||
}
|
||||
return new Promise(resolve => {
|
||||
let blobReader = new FileReader();
|
||||
blobReader.onloadend = ev => {
|
||||
resolve({ key, data: new Uint8Array(ev.target.result) });
|
||||
};
|
||||
blobReader.readAsArrayBuffer(data);
|
||||
});
|
||||
};
|
||||
|
||||
let encodeValue = function(key, value) {
|
||||
if ( !lz4CodecInstance ) { return; }
|
||||
let t0 = window.performance.now();
|
||||
if ( textEncoder === undefined ) {
|
||||
textEncoder = new TextEncoder();
|
||||
}
|
||||
let inputArray = textEncoder.encode(value);
|
||||
let inputSize = inputArray.byteLength;
|
||||
let outputArray = lz4CodecInstance.encodeBlock(inputArray, 8);
|
||||
outputArray[0] = 0x18;
|
||||
outputArray[1] = 0x4D;
|
||||
outputArray[2] = 0x22;
|
||||
outputArray[3] = 0x04;
|
||||
outputArray[4] = (inputSize >>> 0) & 0xFF;
|
||||
outputArray[5] = (inputSize >>> 8) & 0xFF;
|
||||
outputArray[6] = (inputSize >>> 16) & 0xFF;
|
||||
outputArray[7] = (inputSize >>> 24) & 0xFF;
|
||||
console.info(
|
||||
'uBO: [%s] compressed %d bytes into %d bytes in %s ms',
|
||||
key,
|
||||
inputArray.byteLength,
|
||||
outputArray.byteLength,
|
||||
(window.performance.now() - t0).toFixed(2)
|
||||
);
|
||||
return outputArray;
|
||||
};
|
||||
|
||||
let decodeValue = function(key, inputArray) {
|
||||
if ( !lz4CodecInstance ) { return; }
|
||||
let t0 = window.performance.now();
|
||||
if (
|
||||
inputArray[0] !== 0x18 || inputArray[1] !== 0x4D ||
|
||||
inputArray[2] !== 0x22 || inputArray[3] !== 0x04
|
||||
) {
|
||||
return;
|
||||
}
|
||||
let outputSize =
|
||||
(inputArray[4] << 0) | (inputArray[5] << 8) |
|
||||
(inputArray[6] << 16) | (inputArray[7] << 24);
|
||||
let outputArray = lz4CodecInstance.decodeBlock(inputArray, 8, outputSize);
|
||||
if ( textDecoder === undefined ) {
|
||||
textDecoder = new TextDecoder();
|
||||
}
|
||||
let value = textDecoder.decode(outputArray);
|
||||
console.info(
|
||||
'uBO: [%s] decompressed %d bytes into %d bytes in %s ms',
|
||||
key,
|
||||
inputArray.byteLength,
|
||||
outputSize,
|
||||
(window.performance.now() - t0).toFixed(2)
|
||||
);
|
||||
return value;
|
||||
};
|
||||
|
||||
return {
|
||||
encode: function(key, data) {
|
||||
if ( typeof data !== 'string' || data.length < 4096 ) {
|
||||
return Promise.resolve({ key, data });
|
||||
}
|
||||
ttlManage(1);
|
||||
return init().then(( ) => {
|
||||
ttlManage(-1);
|
||||
let encoded = encodeValue(key, data) || data;
|
||||
if ( encoded instanceof Uint8Array ) {
|
||||
encoded = new Blob([ encoded ]);
|
||||
}
|
||||
return { key, data: encoded };
|
||||
});
|
||||
},
|
||||
decode: function(key, data) {
|
||||
if ( data instanceof Blob === false ) {
|
||||
return Promise.resolve({ key, data });
|
||||
}
|
||||
ttlManage(1);
|
||||
return Promise.all([
|
||||
init(),
|
||||
uint8ArrayFromBlob(key, data)
|
||||
]).then(results => {
|
||||
ttlManage(-1);
|
||||
let result = results[1];
|
||||
return {
|
||||
key: result.key,
|
||||
data: decodeValue(result.key, result.data) || result.data
|
||||
};
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
})(); // <<<< End of private namespace
|
||||
|
|
@ -787,7 +787,7 @@ var restoreUserData = function(request) {
|
|||
|
||||
// If we are going to restore all, might as well wipe out clean local
|
||||
// storage
|
||||
vAPI.cacheStorage.clear();
|
||||
µb.cacheStorage.clear();
|
||||
vAPI.storage.clear(onAllRemoved);
|
||||
vAPI.localStorage.removeItem('immediateHiddenSettings');
|
||||
};
|
||||
|
|
@ -803,9 +803,9 @@ var resetUserData = function() {
|
|||
vAPI.app.restart();
|
||||
}
|
||||
};
|
||||
vAPI.cacheStorage.clear(countdown); // 1
|
||||
vAPI.storage.clear(countdown); // 2
|
||||
µb.saveLocalSettings(countdown); // 3
|
||||
µb.cacheStorage.clear(countdown); // 1
|
||||
vAPI.storage.clear(countdown); // 2
|
||||
µb.saveLocalSettings(countdown); // 3
|
||||
vAPI.localStorage.removeItem('immediateHiddenSettings');
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
/*******************************************************************************
|
||||
|
||||
uBlock Origin - a browser extension to block requests.
|
||||
Copyright (C) 2015-2018 Raymond Hill
|
||||
Copyright (C) 2015-present Raymond Hill
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
|
|
@ -504,7 +504,7 @@ RedirectEngine.prototype.resourcesFromString = function(text) {
|
|||
var resourcesSelfieVersion = 3;
|
||||
|
||||
RedirectEngine.prototype.selfieFromResources = function() {
|
||||
vAPI.cacheStorage.set({
|
||||
µBlock.cacheStorage.set({
|
||||
resourcesSelfie: {
|
||||
version: resourcesSelfieVersion,
|
||||
resources: Array.from(this.resources)
|
||||
|
|
@ -534,11 +534,11 @@ RedirectEngine.prototype.resourcesFromSelfie = function(callback) {
|
|||
callback(true);
|
||||
};
|
||||
|
||||
vAPI.cacheStorage.get('resourcesSelfie', onSelfieReady);
|
||||
µBlock.cacheStorage.get('resourcesSelfie', onSelfieReady);
|
||||
};
|
||||
|
||||
RedirectEngine.prototype.invalidateResourcesSelfie = function() {
|
||||
vAPI.cacheStorage.remove('resourcesSelfie');
|
||||
µBlock.cacheStorage.remove('resourcesSelfie');
|
||||
};
|
||||
|
||||
/******************************************************************************/
|
||||
|
|
|
|||
|
|
@ -29,14 +29,35 @@
|
|||
if ( typeof callback !== 'function' ) {
|
||||
callback = this.noopFunc;
|
||||
}
|
||||
var getBytesInUseHandler = function(bytesInUse) {
|
||||
let bytesInUse;
|
||||
let countdown = 0;
|
||||
|
||||
let process = count => {
|
||||
if ( typeof count === 'number' ) {
|
||||
if ( bytesInUse === undefined ) {
|
||||
bytesInUse = 0;
|
||||
}
|
||||
bytesInUse += count;
|
||||
}
|
||||
countdown -= 1;
|
||||
if ( countdown > 0 ) { return; }
|
||||
µBlock.storageUsed = bytesInUse;
|
||||
callback(bytesInUse);
|
||||
};
|
||||
|
||||
// Not all platforms implement this method.
|
||||
if ( vAPI.storage.getBytesInUse instanceof Function ) {
|
||||
vAPI.storage.getBytesInUse(null, getBytesInUseHandler);
|
||||
} else {
|
||||
countdown += 1;
|
||||
vAPI.storage.getBytesInUse(null, process);
|
||||
}
|
||||
if (
|
||||
this.cacheStorage !== vAPI.storage &&
|
||||
this.cacheStorage.getBytesInUse instanceof Function
|
||||
) {
|
||||
countdown += 1;
|
||||
this.cacheStorage.getBytesInUse(null, process);
|
||||
}
|
||||
if ( countdown === 0 ) {
|
||||
callback();
|
||||
}
|
||||
};
|
||||
|
|
@ -1032,11 +1053,11 @@
|
|||
redirectEngine: µb.redirectEngine.toSelfie(),
|
||||
staticExtFilteringEngine: µb.staticExtFilteringEngine.toSelfie()
|
||||
});
|
||||
vAPI.cacheStorage.set({ selfie: selfie });
|
||||
µb.cacheStorage.set({ selfie: selfie });
|
||||
};
|
||||
|
||||
let load = function(callback) {
|
||||
vAPI.cacheStorage.get('selfie', function(bin) {
|
||||
µb.cacheStorage.get('selfie', function(bin) {
|
||||
if (
|
||||
bin instanceof Object === false ||
|
||||
typeof bin.selfie !== 'string'
|
||||
|
|
@ -1067,7 +1088,7 @@
|
|||
clearTimeout(timer);
|
||||
timer = null;
|
||||
}
|
||||
vAPI.cacheStorage.remove('selfie');
|
||||
µb.cacheStorage.remove('selfie');
|
||||
timer = vAPI.setTimeout(create, µb.selfieAfter);
|
||||
};
|
||||
|
||||
|
|
|
|||
Binary file not shown.
171
src/lib/lz4/lz4-block-codec-any.js
Normal file
171
src/lib/lz4/lz4-block-codec-any.js
Normal file
|
|
@ -0,0 +1,171 @@
|
|||
/*******************************************************************************
|
||||
|
||||
lz4-block-codec-any.js
|
||||
A wrapper to instanciate a wasm- and/or js-based LZ4 block
|
||||
encoder/decoder.
|
||||
Copyright (C) 2018 Raymond Hill
|
||||
|
||||
BSD-2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
Home: https://github.com/gorhill/lz4-wasm
|
||||
|
||||
I used the same license as the one picked by creator of LZ4 out of respect
|
||||
for his creation, see https://lz4.github.io/lz4/
|
||||
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
(function(context) { // >>>> Start of private namespace
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
let wd = (function() {
|
||||
let url = document.currentScript.src;
|
||||
let match = /[^\/]+$/.exec(url);
|
||||
return match !== null ?
|
||||
url.slice(0, match.index) :
|
||||
'';
|
||||
})();
|
||||
|
||||
let removeScript = function(script) {
|
||||
if ( !script ) { return; }
|
||||
if ( script.parentNode === null ) { return; }
|
||||
script.parentNode.removeChild(script);
|
||||
};
|
||||
|
||||
let createInstanceWASM = function() {
|
||||
if ( context.LZ4BlockWASM instanceof Function ) {
|
||||
let instance = new context.LZ4BlockWASM();
|
||||
return instance.init().then(( ) => { return instance; });
|
||||
}
|
||||
if ( context.LZ4BlockWASM === null ) {
|
||||
return Promise.resolve(null);
|
||||
}
|
||||
return new Promise((resolve, reject) => {
|
||||
let script = document.createElement('script');
|
||||
script.src = wd + 'lz4-block-codec-wasm.js';
|
||||
script.addEventListener('load', ( ) => {
|
||||
if ( context.LZ4BlockWASM instanceof Function === false ) {
|
||||
context.LZ4BlockWASM = null;
|
||||
context.LZ4BlockWASM = undefined;
|
||||
resolve(null);
|
||||
} else {
|
||||
let instance = new context.LZ4BlockWASM();
|
||||
instance.init()
|
||||
.then(( ) => {
|
||||
resolve(instance);
|
||||
})
|
||||
.catch(error => {
|
||||
reject(error);
|
||||
});
|
||||
}
|
||||
});
|
||||
script.addEventListener('error', ( ) => {
|
||||
context.LZ4BlockWASM = null;
|
||||
resolve(null);
|
||||
});
|
||||
document.head.appendChild(script);
|
||||
removeScript(script);
|
||||
});
|
||||
};
|
||||
|
||||
let createInstanceJS = function() {
|
||||
if ( context.LZ4BlockJS instanceof Function ) {
|
||||
let instance = new context.LZ4BlockJS();
|
||||
return instance.init().then(( ) => { return instance; });
|
||||
}
|
||||
if ( context.LZ4BlockJS === null ) {
|
||||
return Promise.resolve(null);
|
||||
}
|
||||
return new Promise((resolve, reject) => {
|
||||
let script = document.createElement('script');
|
||||
script.src = wd + 'lz4-block-codec-js.js';
|
||||
script.addEventListener('load', ( ) => {
|
||||
if ( context.LZ4BlockJS instanceof Function === false ) {
|
||||
context.LZ4BlockJS = null;
|
||||
resolve(null);
|
||||
} else {
|
||||
let instance = new context.LZ4BlockJS();
|
||||
instance.init()
|
||||
.then(( ) => {
|
||||
resolve(instance);
|
||||
})
|
||||
.catch(error => {
|
||||
reject(error);
|
||||
});
|
||||
}
|
||||
});
|
||||
script.addEventListener('error', ( ) => {
|
||||
context.LZ4BlockJS = null;
|
||||
resolve(null);
|
||||
});
|
||||
document.head.appendChild(script);
|
||||
removeScript(script);
|
||||
});
|
||||
};
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
context.lz4BlockCodec = {
|
||||
createInstance: function(flavor) {
|
||||
let instantiator;
|
||||
if ( flavor === 'wasm' ) {
|
||||
instantiator = createInstanceWASM;
|
||||
} else if ( flavor === 'js' ) {
|
||||
instantiator = createInstanceJS;
|
||||
} else {
|
||||
instantiator = createInstanceWASM || createInstanceJS;
|
||||
}
|
||||
return (instantiator)()
|
||||
.then(instance => {
|
||||
if ( instance ) { return instance; }
|
||||
if ( flavor === undefined ) {
|
||||
return createInstanceJS();
|
||||
}
|
||||
return null;
|
||||
})
|
||||
.catch(( ) => {
|
||||
if ( flavor === undefined ) {
|
||||
return createInstanceJS();
|
||||
}
|
||||
return null;
|
||||
});
|
||||
},
|
||||
reset: function() {
|
||||
context.LZ4BlockWASM = undefined;
|
||||
context.LZ4BlockJS = undefined;
|
||||
}
|
||||
};
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
})(this || self); // <<<< End of private namespace
|
||||
|
||||
/******************************************************************************/
|
||||
281
src/lib/lz4/lz4-block-codec-js.js
Normal file
281
src/lib/lz4/lz4-block-codec-js.js
Normal file
|
|
@ -0,0 +1,281 @@
|
|||
/*******************************************************************************
|
||||
|
||||
lz4-block-codec-js.js
|
||||
A javascript wrapper around a pure javascript implementation of
|
||||
LZ4 block format codec.
|
||||
Copyright (C) 2018 Raymond Hill
|
||||
|
||||
BSD-2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
Home: https://github.com/gorhill/lz4-wasm
|
||||
|
||||
I used the same license as the one picked by creator of LZ4 out of respect
|
||||
for his creation, see https://lz4.github.io/lz4/
|
||||
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
(function(context) { // >>>> Start of private namespace
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
let encodeBound = function(size) {
|
||||
return size > 0x7E000000 ?
|
||||
0 :
|
||||
size + size / 255 + 16;
|
||||
};
|
||||
|
||||
let encodeBlock = function(instance, iBuf, outOffset) {
|
||||
let iLen = iBuf.byteLength;
|
||||
if ( iLen >= 0x7E000000 ) { throw new TypeError(); }
|
||||
|
||||
// "The last match must start at least 12 bytes before end of block"
|
||||
let lastMatchPos = iLen - 12;
|
||||
|
||||
// "The last 5 bytes are always literals"
|
||||
let lastLiteralPos = iLen - 5;
|
||||
|
||||
if ( instance.hashTable === undefined ) {
|
||||
instance.hashTable = new Int32Array(65536);
|
||||
}
|
||||
instance.hashTable.fill(-65536);
|
||||
|
||||
if ( iBuf instanceof ArrayBuffer ) {
|
||||
iBuf = new Uint8Array(iBuf);
|
||||
}
|
||||
|
||||
let oBuf = new Uint8Array(outOffset + encodeBound(iLen));
|
||||
let iPos = 0;
|
||||
let oPos = outOffset;
|
||||
let anchorPos = 0;
|
||||
|
||||
// sequence-finding loop
|
||||
for (;;) {
|
||||
let refPos;
|
||||
let mOffset;
|
||||
let sequence = iBuf[iPos] << 8 | iBuf[iPos+1] << 16 | iBuf[iPos+2] << 24;
|
||||
|
||||
// match-finding loop
|
||||
while ( iPos <= lastMatchPos ) {
|
||||
sequence = sequence >>> 8 | iBuf[iPos+3] << 24;
|
||||
let hash = (sequence * 0x9E37 & 0xFFFF) + (sequence * 0x79B1 >>> 16) & 0xFFFF;
|
||||
refPos = instance.hashTable[hash];
|
||||
instance.hashTable[hash] = iPos;
|
||||
mOffset = iPos - refPos;
|
||||
if (
|
||||
mOffset < 65536 &&
|
||||
iBuf[refPos+0] === ((sequence ) & 0xFF) &&
|
||||
iBuf[refPos+1] === ((sequence >>> 8) & 0xFF) &&
|
||||
iBuf[refPos+2] === ((sequence >>> 16) & 0xFF) &&
|
||||
iBuf[refPos+3] === ((sequence >>> 24) & 0xFF)
|
||||
) {
|
||||
break;
|
||||
}
|
||||
iPos += 1;
|
||||
}
|
||||
|
||||
// no match found
|
||||
if ( iPos > lastMatchPos ) { break; }
|
||||
|
||||
// match found
|
||||
let lLen = iPos - anchorPos;
|
||||
let mLen = iPos;
|
||||
iPos += 4; refPos += 4;
|
||||
while ( iPos < lastLiteralPos && iBuf[iPos] === iBuf[refPos] ) {
|
||||
iPos += 1; refPos += 1;
|
||||
}
|
||||
mLen = iPos - mLen;
|
||||
let token = mLen < 19 ? mLen - 4 : 15;
|
||||
|
||||
// write token, length of literals if needed
|
||||
if ( lLen >= 15 ) {
|
||||
oBuf[oPos++] = 0xF0 | token;
|
||||
let l = lLen - 15;
|
||||
while ( l >= 255 ) {
|
||||
oBuf[oPos++] = 255;
|
||||
l -= 255;
|
||||
}
|
||||
oBuf[oPos++] = l;
|
||||
} else {
|
||||
oBuf[oPos++] = (lLen << 4) | token;
|
||||
}
|
||||
|
||||
// write literals
|
||||
while ( lLen-- ) {
|
||||
oBuf[oPos++] = iBuf[anchorPos++];
|
||||
}
|
||||
|
||||
if ( mLen === 0 ) { break; }
|
||||
|
||||
// write offset of match
|
||||
oBuf[oPos+0] = mOffset;
|
||||
oBuf[oPos+1] = mOffset >>> 8;
|
||||
oPos += 2;
|
||||
|
||||
// write length of match if needed
|
||||
if ( mLen >= 19 ) {
|
||||
let l = mLen - 19;
|
||||
while ( l >= 255 ) {
|
||||
oBuf[oPos++] = 255;
|
||||
l -= 255;
|
||||
}
|
||||
oBuf[oPos++] = l;
|
||||
}
|
||||
|
||||
anchorPos = iPos;
|
||||
}
|
||||
|
||||
// last sequence is literals only
|
||||
let lLen = iLen - anchorPos;
|
||||
if ( lLen >= 15 ) {
|
||||
oBuf[oPos++] = 0xF0;
|
||||
let l = lLen - 15;
|
||||
while ( l >= 255 ) {
|
||||
oBuf[oPos++] = 255;
|
||||
l -= 255;
|
||||
}
|
||||
oBuf[oPos++] = l;
|
||||
} else {
|
||||
oBuf[oPos++] = lLen << 4;
|
||||
}
|
||||
while ( lLen-- ) {
|
||||
oBuf[oPos++] = iBuf[anchorPos++];
|
||||
}
|
||||
|
||||
return new Uint8Array(oBuf.buffer, 0, oPos);
|
||||
};
|
||||
|
||||
let decodeBlock = function(instance, iBuf, iOffset, oLen) {
|
||||
let iLen = iBuf.byteLength;
|
||||
if (
|
||||
instance.outputBuffer === undefined ||
|
||||
instance.outputBuffer.byteLength < oLen
|
||||
) {
|
||||
instance.outputBuffer = new ArrayBuffer(oLen + 0xFFFF & 0x7FFF0000);
|
||||
}
|
||||
let oBuf = new Uint8Array(instance.outputBuffer, 0, oLen);
|
||||
let iPos = iOffset, oPos = 0;
|
||||
|
||||
while ( iPos < iLen ) {
|
||||
let token = iBuf[iPos++];
|
||||
|
||||
// literals
|
||||
let clen = token >>> 4;
|
||||
|
||||
// length of literals
|
||||
if ( clen !== 0 ) {
|
||||
if ( clen === 15 ) {
|
||||
let l;
|
||||
for (;;) {
|
||||
l = iBuf[iPos++];
|
||||
if ( l !== 255 ) { break; }
|
||||
clen += 255;
|
||||
}
|
||||
clen += l;
|
||||
}
|
||||
|
||||
// copy literals
|
||||
let end = iPos + clen;
|
||||
while ( iPos < end ) {
|
||||
oBuf[oPos++] = iBuf[iPos++];
|
||||
}
|
||||
if ( iPos === iLen ) { break; }
|
||||
}
|
||||
|
||||
// match
|
||||
let mOffset = iBuf[iPos+0] | (iBuf[iPos+1] << 8);
|
||||
if ( mOffset === 0 || mOffset > oPos ) { return 0; }
|
||||
iPos += 2;
|
||||
|
||||
// length of match
|
||||
clen = (token & 0x0F) + 4;
|
||||
if ( clen === 19 ) {
|
||||
let l;
|
||||
for (;;) {
|
||||
l = iBuf[iPos++];
|
||||
if ( l !== 255 ) { break; }
|
||||
clen += 255;
|
||||
}
|
||||
clen += l;
|
||||
}
|
||||
|
||||
// copy match
|
||||
let mPos = oPos - mOffset;
|
||||
let end = oPos + clen;
|
||||
while ( oPos < end ) {
|
||||
oBuf[oPos++] = oBuf[mPos++];
|
||||
}
|
||||
}
|
||||
|
||||
return oBuf;
|
||||
};
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
context.LZ4BlockJS = function() {
|
||||
this.hashTable = undefined;
|
||||
this.outputBuffer = undefined;
|
||||
};
|
||||
|
||||
context.LZ4BlockJS.prototype = {
|
||||
flavor: 'js',
|
||||
init: function() {
|
||||
return Promise.resolve();
|
||||
},
|
||||
|
||||
reset: function() {
|
||||
this.hashTable = undefined;
|
||||
this.outputBuffer = undefined;
|
||||
},
|
||||
|
||||
encodeBlock: function(input, outputOffset) {
|
||||
if ( input instanceof ArrayBuffer ) {
|
||||
input = new Uint8Array(input);
|
||||
} else if ( input instanceof Uint8Array === false ) {
|
||||
throw new TypeError();
|
||||
}
|
||||
return encodeBlock(this, input, outputOffset);
|
||||
},
|
||||
|
||||
decodeBlock: function(input, inputOffset, outputSize) {
|
||||
if ( input instanceof ArrayBuffer ) {
|
||||
input = new Uint8Array(input);
|
||||
} else if ( input instanceof Uint8Array === false ) {
|
||||
throw new TypeError();
|
||||
}
|
||||
return decodeBlock(this, input, inputOffset, outputSize);
|
||||
}
|
||||
};
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
})(this || self); // <<<< End of private namespace
|
||||
|
||||
/******************************************************************************/
|
||||
190
src/lib/lz4/lz4-block-codec-wasm.js
Normal file
190
src/lib/lz4/lz4-block-codec-wasm.js
Normal file
|
|
@ -0,0 +1,190 @@
|
|||
/*******************************************************************************
|
||||
|
||||
lz4-block-codec-wasm.js
|
||||
A javascript wrapper around a WebAssembly implementation of
|
||||
LZ4 block format codec.
|
||||
Copyright (C) 2018 Raymond Hill
|
||||
|
||||
BSD-2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
Home: https://github.com/gorhill/lz4-wasm
|
||||
|
||||
I used the same license as the one picked by creator of LZ4 out of respect
|
||||
for his creation, see https://lz4.github.io/lz4/
|
||||
|
||||
*/
|
||||
|
||||
/* global WebAssembly */
|
||||
|
||||
'use strict';
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
(function(context) { // >>>> Start of private namespace
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
let wd = (function() {
|
||||
let url = document.currentScript.src;
|
||||
let match = /[^\/]+$/.exec(url);
|
||||
return match !== null ?
|
||||
url.slice(0, match.index) :
|
||||
'';
|
||||
})();
|
||||
|
||||
let growMemoryTo = function(wasmInstance, byteLength) {
|
||||
let lz4api = wasmInstance.exports;
|
||||
let neededByteLength = lz4api.getLinearMemoryOffset() + byteLength;
|
||||
let pageCountBefore = lz4api.memory.buffer.byteLength >>> 16;
|
||||
let pageCountAfter = (neededByteLength + 65535) >>> 16;
|
||||
if ( pageCountAfter > pageCountBefore ) {
|
||||
lz4api.memory.grow(pageCountAfter - pageCountBefore);
|
||||
}
|
||||
return lz4api.memory.buffer;
|
||||
};
|
||||
|
||||
let encodeBlock = function(wasmInstance, inputArray, outputOffset) {
|
||||
let lz4api = wasmInstance.exports;
|
||||
let mem0 = lz4api.getLinearMemoryOffset();
|
||||
let hashTableSize = 65536 * 4;
|
||||
let inputSize = inputArray.byteLength;
|
||||
if ( inputSize >= 0x7E000000 ) { throw new TypeError(); }
|
||||
let memSize =
|
||||
hashTableSize +
|
||||
inputSize +
|
||||
outputOffset + lz4api.lz4BlockEncodeBound(inputSize);
|
||||
let memBuffer = growMemoryTo(wasmInstance, memSize);
|
||||
let hashTable = new Int32Array(memBuffer, mem0, 65536);
|
||||
hashTable.fill(-65536, 0, 65536);
|
||||
let inputMem = new Uint8Array(memBuffer, mem0 + hashTableSize, inputSize);
|
||||
inputMem.set(inputArray);
|
||||
let outputSize = lz4api.lz4BlockEncode(
|
||||
mem0 + hashTableSize,
|
||||
inputSize,
|
||||
mem0 + hashTableSize + inputSize + outputOffset
|
||||
);
|
||||
if ( outputSize === 0 ) {
|
||||
inputSize = 0 - inputSize;
|
||||
}
|
||||
let outputArray = new Uint8Array(
|
||||
memBuffer,
|
||||
mem0 + hashTableSize + inputSize,
|
||||
outputOffset + outputSize
|
||||
);
|
||||
return outputArray;
|
||||
};
|
||||
|
||||
let decodeBlock = function(wasmInstance, inputArray, inputOffset, outputSize) {
|
||||
let inputSize = inputArray.byteLength;
|
||||
let lz4api = wasmInstance.exports;
|
||||
let mem0 = lz4api.getLinearMemoryOffset();
|
||||
let memSize = inputSize + outputSize;
|
||||
let memBuffer = growMemoryTo(wasmInstance, memSize);
|
||||
let inputArea = new Uint8Array(memBuffer, mem0, inputSize);
|
||||
inputArea.set(inputArray);
|
||||
outputSize = lz4api.lz4BlockDecode(
|
||||
mem0 + inputOffset,
|
||||
inputSize - inputOffset,
|
||||
mem0 + inputSize
|
||||
);
|
||||
if ( outputSize === 0 ) {
|
||||
throw new Error('LZ4BlockWASM: block-level compression failed');
|
||||
}
|
||||
return new Uint8Array(memBuffer, mem0 + inputSize, outputSize);
|
||||
};
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
context.LZ4BlockWASM = function() {
|
||||
this.lz4wasmInstance = undefined;
|
||||
};
|
||||
|
||||
context.LZ4BlockWASM.prototype = {
|
||||
flavor: 'wasm',
|
||||
init: function() {
|
||||
if ( this.lz4wasmInstance instanceof WebAssembly.Instance ) {
|
||||
return Promise.resolve(this.lz4wasmInstance);
|
||||
}
|
||||
if (
|
||||
this.lz4wasmInstance === null ||
|
||||
WebAssembly instanceof Object === false ||
|
||||
typeof WebAssembly.instantiateStreaming !== 'function'
|
||||
) {
|
||||
this.lz4wasmInstance = null;
|
||||
return Promise.resolve(null);
|
||||
}
|
||||
if ( this.lz4wasmInstance === undefined ) {
|
||||
this.lz4wasmInstance = WebAssembly.instantiateStreaming(
|
||||
fetch(wd + 'lz4-block-codec.wasm', { mode: 'same-origin' })
|
||||
).then(result => {
|
||||
this.lz4wasmInstance = undefined;
|
||||
this.lz4wasmInstance = result && result.instance || null;
|
||||
if ( this.lz4wasmInstance !== null ) { return this; }
|
||||
return null;
|
||||
});
|
||||
this.lz4wasmInstance.catch(( ) => {
|
||||
this.lz4wasmInstance = null;
|
||||
return null;
|
||||
});
|
||||
}
|
||||
return this.lz4wasmInstance;
|
||||
},
|
||||
|
||||
reset: function() {
|
||||
this.lz4wasmInstance = undefined;
|
||||
},
|
||||
|
||||
encodeBlock: function(input, outputOffset) {
|
||||
if ( this.lz4wasmInstance instanceof WebAssembly.Instance === false ) {
|
||||
throw new Error('LZ4BlockWASM: not initialized');
|
||||
}
|
||||
if ( input instanceof ArrayBuffer ) {
|
||||
input = new Uint8Array(input);
|
||||
} else if ( input instanceof Uint8Array === false ) {
|
||||
throw new TypeError();
|
||||
}
|
||||
return encodeBlock(this.lz4wasmInstance, input, outputOffset);
|
||||
},
|
||||
|
||||
decodeBlock: function(input, inputOffset, outputSize) {
|
||||
if ( this.lz4wasmInstance instanceof WebAssembly.Instance === false ) {
|
||||
throw new Error('LZ4BlockWASM: not initialized');
|
||||
}
|
||||
if ( input instanceof ArrayBuffer ) {
|
||||
input = new Uint8Array(input);
|
||||
} else if ( input instanceof Uint8Array === false ) {
|
||||
throw new TypeError();
|
||||
}
|
||||
return decodeBlock(this.lz4wasmInstance, input, inputOffset, outputSize);
|
||||
}
|
||||
};
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
})(this || self); // <<<< End of private namespace
|
||||
|
||||
/******************************************************************************/
|
||||
BIN
src/lib/lz4/lz4-block-codec.wasm
Normal file
BIN
src/lib/lz4/lz4-block-codec.wasm
Normal file
Binary file not shown.
Loading…
Reference in a new issue