external/uBlock
1
0
Fork 0
mirror of https://github.com/gorhill/uBlock.git synced 2024-11-10 01:02:08 +01:00
Code Issues Projects Releases Packages Wiki Activity

Add new static network filter option: urltransform

Browse source 
The `urltransform` option allows to redirect a non-blocked network
request to another URL. There are restrictions on its usage:

- require a trusted source -- thus uBO-maintained lists or user
  filters
- the `urltransform` value must start with a `/`

If at least one of these conditions is not fulfilled, the filter
will be invalid and rejected.

The requirement to start with `/` is to enforce that only the path
part of a URL can be modified, thus ensuring the network request
is redirected to the same scheme and authority (as defined at
https://en.wikipedia.org/wiki/Uniform_Resource_Identifier#Syntax).

Usage example (redirect requests for CSS resources to a non-existing
resource, for demonstration purpose):

    ||iana.org^$css,urltransform=/notfound.css

Name of this option is inspired from DNR API:
https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/declarativeNetRequest/URLTransform

This commit required to bring the concept of "trusted source" to
the static network filtering engine.
This commit is contained in:
Raymond Hill 2023-10-16 09:47:29 -04:00
parent bee64ebd90
commit 2e4525fe3c
No known key found for this signature in database
GPG key ID: 25E1490B761470C2
14 changed files with 110 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
platform/mv3/make-scriptlets.js
View file

@ -88,7 +88,7 @@ export function compile(details) {
const scriptletToken = details.args[0];
const resourceEntry = resourceDetails.get(scriptletToken);
if ( resourceEntry === undefined ) { return; }
if ( resourceEntry.requiresTrust && details.isTrusted !== true ) {
if ( resourceEntry.requiresTrust && details.trustedSource !== true ) {
console.log(`Rejecting ${scriptletToken}: source is not trusted`);
return;
}

1
src/js/1p-filters.js
View file

@ -48,6 +48,7 @@ const cmEditor = new CodeMirror(qs$('#userFilters'), {
styleActiveLine: {
nonEmpty: true,
},
trustedSource: true,
});
uBlockDashboard.patchCodeMirrorEditor(cmEditor);

1
src/js/asset-viewer.js
View file

@ -82,6 +82,7 @@ import './codemirror/ubo-static-filtering.js';
what : 'getAssetContent',
url: assetKey,
});
cmEditor.setOption('trustedSource', details.trustedSource === true);
cmEditor.setValue(details && details.content || '');
if ( subscribeElem !== null ) {

3
src/js/benchmarks.js
View file

@ -181,7 +181,8 @@ const loadBenchmarkDataset = (( ) => {
if ( r === 1 ) { blockCount += 1; }
else if ( r === 2 ) { allowCount += 1; }
if ( r !== 1 ) {
if ( staticNetFilteringEngine.hasQuery(fctxt) ) {
staticNetFilteringEngine.transformRequest(fctxt);
if ( fctxt.redirectURL !== undefined && staticNetFilteringEngine.hasQuery(fctxt) ) {
staticNetFilteringEngine.filterQuery(fctxt, 'removeparam');
}
if ( fctxt.type === 'main_frame' || fctxt.type === 'sub_frame' ) {

17
src/js/codemirror/ubo-static-filtering.js
View file

@ -37,12 +37,21 @@ const preparseDirectiveHints = [];
const originHints = [];
let hintHelperRegistered = false;
/******************************************************************************/
let trustedSource = false;
CodeMirror.defineOption('trustedSource', false, (cm, state) => {
trustedSource = state;
self.dispatchEvent(new Event('trustedSource'));
});
/******************************************************************************/
CodeMirror.defineMode('ubo-static-filtering', function() {
const astParser = new sfp.AstFilterParser({
interactive: true,
trustedSource,
nativeCssHas: vAPI.webextFlavor.env.includes('native_css_has'),
});
const astWalker = astParser.getWalker();
@ -205,6 +214,10 @@ CodeMirror.defineMode('ubo-static-filtering', function() {
return '+';
};
self.addEventListener('trustedSource', ( ) => {
astParser.options.trustedSource = trustedSource;
});
return {
lineComment: '!',
token: function(stream) {
@ -977,6 +990,10 @@ CodeMirror.registerHelper('fold', 'ubo-static-filtering', (( ) => {
}
};
self.addEventListener('trustedSource', ( ) => {
astParser.options.trustedSource = trustedSource;
});
CodeMirror.defineInitHook(cm => {
cm.on('changes', onChanges);
cm.on('beforeChange', onBeforeChanges);

1
src/js/messaging.js
View file

@ -108,6 +108,7 @@ const onMessage = function(request, sender, callback) {
dontCache: true,
needSourceURL: true,
}).then(result => {
result.trustedSource = µb.isTrustedList(result.assetKey);
callback(result);
});
return;

24
src/js/pagestore.js
View file

@ -860,7 +860,7 @@ const PageStore = class {
if ( (fctxt.itype & fctxt.INLINE_ANY) === 0 ) {
if ( result === 1 ) {
this.redirectBlockedRequest(fctxt);
} else if ( snfe.hasQuery(fctxt) ) {
} else {
this.redirectNonBlockedRequest(fctxt);
}
}
@ -922,25 +922,31 @@ const PageStore = class {
}
redirectBlockedRequest(fctxt) {
const directives = staticNetFilteringEngine.redirectRequest(
redirectEngine,
fctxt
);
const directives = staticNetFilteringEngine.redirectRequest(redirectEngine, fctxt);
if ( directives === undefined ) { return; }
if ( logger.enabled !== true ) { return; }
fctxt.pushFilters(directives.map(a => a.logData()));
if ( fctxt.redirectURL === undefined ) { return; }
fctxt.pushFilter({
source: 'redirect',
raw: redirectEngine.resourceNameRegister
raw: directives[directives.length-1].value
});
}
redirectNonBlockedRequest(fctxt) {
const directives = staticNetFilteringEngine.filterQuery(fctxt);
if ( directives === undefined ) { return; }
const transformDirectives = staticNetFilteringEngine.transformRequest(fctxt);
const pruneDirectives = fctxt.redirectURL === undefined &&
staticNetFilteringEngine.hasQuery(fctxt) &&
staticNetFilteringEngine.filterQuery(fctxt) ||
undefined;
if ( transformDirectives === undefined && pruneDirectives === undefined ) { return; }
if ( logger.enabled !== true ) { return; }
fctxt.pushFilters(directives.map(a => a.logData()));
if ( transformDirectives !== undefined ) {
fctxt.pushFilters(transformDirectives.map(a => a.logData()));
}
if ( pruneDirectives !== undefined ) {
fctxt.pushFilters(pruneDirectives.map(a => a.logData()));
}
if ( fctxt.redirectURL === undefined ) { return; }
fctxt.pushFilter({
source: 'redirect',

2
src/js/redirect-engine.js
View file

@ -167,7 +167,6 @@ class RedirectEngine {
this.resources = new Map();
this.reset();
this.modifyTime = Date.now();
this.resourceNameRegister = '';
}
reset() {
@ -183,7 +182,6 @@ class RedirectEngine {
) {
const entry = this.resources.get(this.aliases.get(token) || token);
if ( entry === undefined ) { return; }
this.resourceNameRegister = token;
return entry.toURL(fctxt, asDataURI);
}

2
src/js/reverselookup.js
View file

@ -131,6 +131,7 @@ const fromNetFilter = async function(rawFilter) {
const writer = new CompiledListWriter();
const parser = new sfp.AstFilterParser({
expertMode: true,
trustedSource: true,
maxTokenLength: staticNetFilteringEngine.MAX_TOKEN_LENGTH,
nativeCssHas: vAPI.webextFlavor.env.includes('native_css_has'),
});
@ -169,6 +170,7 @@ const fromExtendedFilter = async function(details) {
const parser = new sfp.AstFilterParser({
expertMode: true,
trustedSource: true,
nativeCssHas: vAPI.webextFlavor.env.includes('native_css_has'),
});
parser.parse(details.rawFilter);

2
src/js/scriptlet-filtering.js
View file

@ -306,7 +306,7 @@ scriptletFilteringEngine.compile = function(parser, writer) {
// Only exception filters are allowed to be global.
const isException = parser.isException();
const normalized = normalizeRawFilter(parser, writer.properties.get('isTrusted'));
const normalized = normalizeRawFilter(parser, writer.properties.get('trustedSource'));
// Can fail if there is a mismatch with trust requirement
if ( normalized === undefined ) { return; }

10
src/js/static-dnr-filtering.js
View file

@ -109,8 +109,8 @@ function addExtendedToDNR(context, parser) {
let details = context.scriptletFilters.get(argsToken);
if ( details === undefined ) {
context.scriptletFilters.set(argsToken, details = { args });
if ( context.isTrusted ) {
details.isTrusted = true;
if ( context.trustedSource ) {
details.trustedSource = true;
}
}
if ( not ) {
@ -299,9 +299,11 @@ function addToDNR(context, list) {
if ( parser.isComment() ) {
if ( line === `!#trusted on ${context.secret}` ) {
context.isTrusted = true;
parser.trustedSource = true;
context.trustedSource = true;
} else if ( line === `!#trusted off ${context.secret}` ) {
context.isTrusted = false;
parser.trustedSource = false;
context.trustedSource = false;
}
continue;
}

12
src/js/static-filtering-parser.js
View file

@ -188,6 +188,7 @@ export const NODE_TYPE_NET_OPTION_NAME_REMOVEPARAM = iota++;
export const NODE_TYPE_NET_OPTION_NAME_SCRIPT = iota++;
export const NODE_TYPE_NET_OPTION_NAME_SHIDE = iota++;
export const NODE_TYPE_NET_OPTION_NAME_TO = iota++;
export const NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM = iota++;
export const NODE_TYPE_NET_OPTION_NAME_XHR = iota++;
export const NODE_TYPE_NET_OPTION_NAME_WEBRTC = iota++;
export const NODE_TYPE_NET_OPTION_NAME_WEBSOCKET = iota++;
@ -267,6 +268,7 @@ export const nodeTypeFromOptionName = new Map([
[ 'shide', NODE_TYPE_NET_OPTION_NAME_SHIDE ],
/* synonym */ [ 'specifichide', NODE_TYPE_NET_OPTION_NAME_SHIDE ],
[ 'to', NODE_TYPE_NET_OPTION_NAME_TO ],
[ 'urltransform', NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM ],
[ 'xhr', NODE_TYPE_NET_OPTION_NAME_XHR ],
/* synonym */ [ 'xmlhttprequest', NODE_TYPE_NET_OPTION_NAME_XHR ],
[ 'webrtc', NODE_TYPE_NET_OPTION_NAME_WEBRTC ],
@ -1315,6 +1317,7 @@ export class AstFilterParser {
break;
case NODE_TYPE_NET_OPTION_NAME_REDIRECT:
case NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE:
case NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM:
realBad = isNegated || (isException || hasValue) === false ||
modifierType !== 0;
if ( realBad ) { break; }
@ -1374,6 +1377,14 @@ export class AstFilterParser {
realBad = abstractTypeCount || behaviorTypeCount || unredirectableTypeCount;
break;
}
case NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM:
realBad = abstractTypeCount || behaviorTypeCount || unredirectableTypeCount ||
this.options.trustedSource !== true;
if ( realBad !== true ) {
const path = this.getNetOptionValue(NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM);
realBad = path.charCodeAt(0) !== 0x2F /* / */;
}
break;
case NODE_TYPE_NET_OPTION_NAME_REMOVEPARAM:
realBad = abstractTypeCount || behaviorTypeCount;
break;
@ -2973,6 +2984,7 @@ export const netOptionTokenDescriptors = new Map([
[ 'shide', { } ],
/* synonym */ [ 'specifichide', { } ],
[ 'to', { mustAssign: true } ],
[ 'urltransform', { mustAssign: true } ],
[ 'xhr', { canNegate: true } ],
/* synonym */ [ 'xmlhttprequest', { canNegate: true } ],
[ 'webrtc', { } ],

44
src/js/static-net-filtering.js
View file

@ -178,11 +178,14 @@ const typeValueToDNRTypeName = [
'other',
];
// Do not change order. Compiled filter lists rely on this order being
// consistent across sessions.
const MODIFIER_TYPE_REDIRECT = 1;
const MODIFIER_TYPE_REDIRECTRULE = 2;
const MODIFIER_TYPE_REMOVEPARAM = 3;
const MODIFIER_TYPE_CSP = 4;
const MODIFIER_TYPE_PERMISSIONS = 5;
const MODIFIER_TYPE_URLTRANSFORM = 6;
const modifierTypeFromName = new Map([
[ 'redirect', MODIFIER_TYPE_REDIRECT ],
@ -190,6 +193,7 @@ const modifierTypeFromName = new Map([
[ 'removeparam', MODIFIER_TYPE_REMOVEPARAM ],
[ 'csp', MODIFIER_TYPE_CSP ],
[ 'permissions', MODIFIER_TYPE_PERMISSIONS ],
[ 'urltransform', MODIFIER_TYPE_URLTRANSFORM ],
]);
const modifierNameFromType = new Map([
@ -198,6 +202,7 @@ const modifierNameFromType = new Map([
[ MODIFIER_TYPE_REMOVEPARAM, 'removeparam' ],
[ MODIFIER_TYPE_CSP, 'csp' ],
[ MODIFIER_TYPE_PERMISSIONS, 'permissions' ],
[ MODIFIER_TYPE_URLTRANSFORM, 'urltransform' ],
]);
//const typeValueFromCatBits = catBits => (catBits >>> TypeBitsOffset) & 0b11111;
@ -3182,6 +3187,7 @@ class FilterCompiler {
[ sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECT, MODIFIER_TYPE_REDIRECT ],
[ sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE, MODIFIER_TYPE_REDIRECTRULE ],
[ sfp.NODE_TYPE_NET_OPTION_NAME_REMOVEPARAM, MODIFIER_TYPE_REMOVEPARAM ],
[ sfp.NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM, MODIFIER_TYPE_URLTRANSFORM ],
]);
// These top 100 "bad tokens" are collated using the "miss" histogram
// from tokenHistograms(). The "score" is their occurrence among the
@ -3484,6 +3490,12 @@ class FilterCompiler {
if ( this.toDomainOpt === '' ) { return false; }
this.optionUnitBits |= this.TO_BIT;
break;
case sfp.NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM:
if ( this.processModifierOption(id, parser.getNetOptionValue(id)) === false ) {
return false;
}
this.optionUnitBits |= this.REDIRECT_BIT;
break;
default:
break;
}
@ -3575,6 +3587,7 @@ class FilterCompiler {
case sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE:
case sfp.NODE_TYPE_NET_OPTION_NAME_REMOVEPARAM:
case sfp.NODE_TYPE_NET_OPTION_NAME_TO:
case sfp.NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM:
if ( this.processOptionWithValue(parser, type) === false ) {
return this.FILTER_INVALID;
}
@ -4521,6 +4534,20 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) {
dnrAddRuleError(rule, 'Unsupported modifier exception');
}
break;
case 'urltransform': {
const path = rule.__modifierValue;
let priority = rule.priority || 1;
if ( rule.__modifierAction !== AllowAction ) {
const transform = { path };
rule.action.type = 'redirect';
rule.action.redirect = { transform };
rule.priority = priority + 1;
} else {
rule.action.type = 'block';
rule.priority = priority + 2;
}
break;
}
default:
dnrAddRuleError(rule, `Unsupported modifier ${rule.__modifierType}`);
break;
@ -5230,18 +5257,27 @@ FilterContainer.prototype.redirectRequest = function(redirectEngine, fctxt) {
}
// Redirect to highest-ranked directive
const directive = directives[highest];
if ( (directive.bits & AllowAction) === 0 ) {
if ( (directive.bits & AllowAction) !== 0 ) { return directives; }
const { token } = parseRedirectRequestValue(directive);
fctxt.redirectURL = redirectEngine.tokenToURL(fctxt, token);
if ( fctxt.redirectURL === undefined ) { return; }
}
return directives;
};
FilterContainer.prototype.transformRequest = function(fctxt) {
const directives = this.matchAndFetchModifiers(fctxt, 'urltransform');
if ( directives === undefined ) { return; }
const directive = directives[directives.length-1];
if ( (directive.bits & AllowAction) !== 0 ) { return directives; }
const redirectURL = new URL(fctxt.url);
redirectURL.pathname = directive.value;
fctxt.redirectURL = redirectURL.href;
return directives;
};
function parseRedirectRequestValue(directive) {
if ( directive.cache === null ) {
directive.cache =
sfp.parseRedirectValue(directive.value);
directive.cache = sfp.parseRedirectValue(directive.value);
}
return directive.cache;
}

12
src/js/storage.js
View file

@ -569,7 +569,7 @@ self.addEventListener('hiddenSettingsChanged', ( ) => {
const compiledFilters = this.compileFilters(filters, {
assetKey: this.userFiltersPath,
isTrusted: true,
trustedSource: true,
});
const snfe = staticNetFilteringEngine;
const cfe = cosmeticFilteringEngine;
@ -908,7 +908,7 @@ self.addEventListener('hiddenSettingsChanged', ( ) => {
µb.inMemoryFiltersCompiled =
µb.compileFilters(µb.inMemoryFilters.join('\n'), {
assetKey: 'in-memory',
isTrusted: true,
trustedSource: true,
});
}
if ( µb.inMemoryFiltersCompiled !== '' ) {
@ -976,7 +976,7 @@ self.addEventListener('hiddenSettingsChanged', ( ) => {
const compiledContent = this.compileFilters(rawDetails.content, {
assetKey,
isTrusted: this.isTrustedList(assetKey),
trustedSource: this.isTrustedList(assetKey),
});
io.put(compiledPath, compiledContent);
@ -1041,9 +1041,10 @@ self.addEventListener('hiddenSettingsChanged', ( ) => {
// Populate the writer with information potentially useful to the
// client compilers.
const trustedSource = details.trustedSource === true;
if ( details.assetKey ) {
writer.properties.set('name', details.assetKey);
writer.properties.set('isTrusted', details.isTrusted === true);
writer.properties.set('trustedSource', trustedSource);
}
const assetName = details.assetKey ? details.assetKey : '?';
const expertMode =
@ -1051,6 +1052,7 @@ self.addEventListener('hiddenSettingsChanged', ( ) => {
this.hiddenSettings.filterAuthorMode !== false;
const parser = new sfp.AstFilterParser({
expertMode,
trustedSource,
maxTokenLength: staticNetFilteringEngine.MAX_TOKEN_LENGTH,
nativeCssHas: vAPI.webextFlavor.env.includes('native_css_has'),
});
@ -1564,7 +1566,7 @@ self.addEventListener('hiddenSettingsChanged', ( ) => {
'compiled/' + details.assetKey,
this.compileFilters(details.content, {
assetKey: details.assetKey,
isTrusted: this.isTrustedList(details.assetKey),
trustedSource: this.isTrustedList(details.assetKey),
})
);
}