From 834ca15520bfc832a0d47ff75e864a095c3c5e29 Mon Sep 17 00:00:00 2001 From: magnolia1234 <7676006-magnolia1234@users.noreply.gitlab.com> Date: Mon, 28 Oct 2019 19:26:52 +0100 Subject: [PATCH 1/3] Cookie-fix for The Diplomat, Chicago Business, Chemical & Engineering News, The Age Cookie-fix for The Diplomat, Chicago Business, Chemical & Engineering News, The Age. Collatoral damage of my maintenance to background.js. Until a new release you can install the Cookie Remover Add-on: https://addons.mozilla.org/en-US/firefox/addon/cookie-remover for these sites. Btw theharald.com.au has changed to newcastleherald.com.au (not changed), centralwesterndaily.com.au has no cookie-fix --- background.js | 866 +++++++++++++++++++++++++------------------------- 1 file changed, 437 insertions(+), 429 deletions(-) diff --git a/background.js b/background.js index bcb58a8..e5e6c15 100644 --- a/background.js +++ b/background.js @@ -1,429 +1,437 @@ -'use strict'; - -var defaultSites = { - 'Algemeen Dagblad': 'ad.nl', - 'Baltimore Sun': 'baltimoresun.com', - 'Barron\'s': 'barrons.com', - 'Bloomberg': 'bloomberg.com', - 'Business Insider': 'businessinsider.com', - 'Caixin': 'caixinglobal.com', - 'Chemical & Engineering News': 'cen.acs.org', - 'Chicago Tribune': 'chicagotribune.com', - 'Central Western Daily': 'centralwesterndaily.com.au', - 'Crain\'s Chicago Business': 'chicagobusiness.com', - 'Corriere Della Sera': 'corriere.it', - 'Daily Press': 'dailypress.com', - 'Denver Post': 'denverpost.com', - 'De Tijd': 'tijd.be', - 'De Groene Amsterdammer': 'groene.nl', - 'de Volkskrant': 'volkskrant.nl', - 'The Economist': 'economist.com', - 'Eindhovens Dagblad': 'ed.nl', - 'Examiner': 'examiner.com.au', - 'Financial Times': 'ft.com', - 'Foreign Policy': 'foreignpolicy.com', - 'Glassdoor': 'glassdoor.com', - 'Haaretz': 'haaretz.co.il', - 'Haaretz English': 'haaretz.com', - 'Handelsblatt': 'handelsblatt.com', - 'Hartford Courant': 'courant.com', - 'Harvard Business Review': 'hbr.org', - 'Inc.com': 'inc.com', - 'Investors Chronicle': 'investorschronicle.co.uk', - 'La Repubblica': 'repubblica.it', - 'Le Monde': 'lemonde.fr', - 'Le Temps': 'letemps.ch', - 'Los Angeles Times': 'latimes.com', - 'Medium': 'medium.com', - 'Medscape': 'medscape.com', - 'MIT Technology Review': 'technologyreview.com', - 'Mountain View Voice': 'mv-voice.com', - 'National Post': 'nationalpost.com', - 'New Statesman': 'newstatesman.com', - 'New York Magazine': 'nymag.com', - 'Nikkei Asian Review': 'asia.nikkei.com', - 'NRC': 'nrc.nl', - 'New Zealand Herald': 'nzherald.co.nz', - 'Orange County Register': 'ocregister.com', - 'Orlando Sentinel': 'orlandosentinel.com', - 'Palo Alto Online': 'paloaltoonline.com', - 'Parool': 'parool.nl', - 'Quartz': 'qz.com', - 'Quora': 'quora.com', - 'Statista': 'statista.com', - 'Telegraaf': 'telegraaf.nl', - 'SunSentinel': 'sun-sentinel.com', - 'The Advocate': 'theadvocate.com.au', - 'The Age': 'theage.com.au', - 'The Atlantic': 'theatlantic.com', - 'The Australian': 'theaustralian.com.au', - 'The Australian Financial Review': 'afr.com', - 'The Boston Globe': 'bostonglobe.com', - 'The Business Journals': 'bizjournals.com', - 'The Diplomat': 'thediplomat.com', - 'The Globe and Mail': 'theglobeandmail.com', - 'The Herald': 'theherald.com.au', - 'The Japan Times': 'japantimes.co.jp', - 'TheMarker': 'themarker.com', - 'The Mercury News': 'mercurynews.com', - 'The Morning Call': 'mcall.com', - 'The Nation': 'thenation.com', - 'The New York Times': 'nytimes.com', - 'The New Yorker': 'newyorker.com', - 'The News-Gazette': 'news-gazette.com', - 'The Saturday Paper': 'thesaturdaypaper.com.au', - 'The Spectator': 'spectator.co.uk', - 'The Seattle Times': 'seattletimes.com', - 'The Sydney Morning Herald': 'smh.com.au', - 'The Telegraph': 'telegraph.co.uk', - 'The Times': 'thetimes.co.uk', - 'The Toronto Star': 'thestar.com', - 'The Washington Post': 'washingtonpost.com', - 'The Wall Street Journal': 'wsj.com', - 'Towards Data Science': 'towardsdatascience.com', - 'Trouw': 'trouw.nl', - 'Vanity Fair': 'vanityfair.com', - 'Vrij Nederland': 'vn.nl', - 'Wired': 'wired.com' -}; - -const restrictions = { - 'barrons.com': 'barrons.com/articles' -} - -// Don't remove cookies before page load -const allow_cookies = [ -'ad.nl', -'asia.nikkei.com', -'bostonglobe.com', -'denverpost.com', -'economist.com', -'ed.nl', -'examiner.com.au', -'ft.com', -'hacked.com', -'hbr.org', -'lemonde.fr', -'letemps.ch', -'medium.com', -'mercurynews.com', -'newstatesman.com', -'nymag.com', -'nytimes.com', -'ocregister.com', -'parool.nl', -'qz.com', -'spectator.co.uk', -'telegraaf.nl', -'theadvocate.com.au', -'theaustralian.com.au', -'thestar.com', -'towardsdatascience.com', -'trouw.nl', -'vn.nl', -'volkskrant.nl', -'washingtonpost.com', -'wsj.com', -] - -// Removes cookies after page load -const remove_cookies = [ -'ad.nl', -'asia.nikkei.com', -'bostonglobe.com', -'denverpost.com', -'economist.com', -'ed.nl', -'examiner.com.au', -'ft.com', -'hacked.com', -'hbr.org', -'letemps.ch', -'medium.com', -'mercurynews.com', -'newstatesman.com', -'nymag.com', -'nytimes.com', -'ocregister.com', -'qz.com', -'spectator.co.uk', -'telegraaf.nl', -'theadvocate.com.au', -'thestar.com', -'towardsdatascience.com', -'vn.nl', -'washingtonpost.com', -'wsj.com', -] - -// select specific cookie(s) to hold from remove_cookies domains -const remove_cookies_select_hold = { - '.nrc.nl': ['nmt_closed_cookiebar'], - '.washingtonpost.com': ['wp_gdpr'], - '.wsj.com': ['wsjregion'] -} - -// select only specific cookie(s) to drop from remove_cookies domains -const remove_cookies_select_drop = { - 'www.nrc.nl': ['counter'] -} - -// Override User-Agent with Googlebot -const use_google_bot = [ -'barrons.com', -'nytimes.com', -'telegraph.co.uk', -'theaustralian.com.au', -'thetimes.co.uk', -'wsj.com', -] - -function setDefaultOptions() { - browser.storage.sync.set({ - sites: defaultSites - }, function() { - browser.runtime.openOptionsPage(); - }); -} - -const blockedRegexes = [ -/.+:\/\/.+\.tribdss\.com\//, -/thenation\.com\/.+\/paywall-script\.php/, -/haaretz\.co\.il\/htz\/js\/inter\.js/, -/nzherald\.co\.nz\/.+\/headjs\/.+\.js/ -]; - -const userAgentDesktop = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" -const userAgentMobile = "Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible ; Googlebot/2.1 ; +http://www.google.com/bot.html)" - -var enabledSites = []; - -// Get the enabled sites -browser.storage.sync.get({ - sites: {} -}, function(items) { - var sites = items.sites; - enabledSites = Object.keys(items.sites).map(function(key) { - return items.sites[key]; - }); -}); - -// Listen for changes to options -browser.storage.onChanged.addListener(function(changes, namespace) { - var key; - for (key in changes) { - var storageChange = changes[key]; - if (key === 'sites') { - var sites = storageChange.newValue; - enabledSites = Object.keys(sites).map(function(key) { - return sites[key]; - }); - } - } -}); - -// Set and show default options on install -browser.runtime.onInstalled.addListener(function(details) { - if (details.reason == "install") { - setDefaultOptions(); - } else if (details.reason == "update") { - // User updated extension - } -}); - -/** -// WSJ bypass -browser.webRequest.onBeforeSendHeaders.addListener(function(details) { - if (!isSiteEnabled(details) || details.url.indexOf("mod=rsswn") !== -1 || details.url.indexOf("/print-edition/") !== -1) { - return; - } - - var param; - var updatedUrl; - - param = getParameterByName("mod", details.url); - - if (param === null) { - updatedUrl = stripQueryStringAndHashFromPath(details.url); - updatedUrl += "?mod=rsswn"; - } else { - updatedUrl = details.url.replace(param, "rsswn"); - } - return { redirectUrl: updatedUrl}; -}, -{urls:["*://*.wsj.com/*"], types:["main_frame"]}, -["blocking"] -); -**/ - -// Disable javascript for these sites -chrome.webRequest.onBeforeRequest.addListener(function(details) { - if (!isSiteEnabled(details) || details.url.indexOf("mod=rsswn") !== -1) { - return; - } - return {cancel: true}; - }, - { - urls: ["*://*.theglobeandmail.com/*", "*://*.economist.com/*", "*://*.thestar.com/*", "*://*.newstatesman.com/*", "*://*.bostonglobe.com/*", "*://*.afr.com/*"], - types: ["script"] - }, - ["blocking"] -); - -browser.webRequest.onBeforeSendHeaders.addListener(function(details) { - if (!isSiteEnabled(details)) { - return; - } - - if (blockedRegexes.some(function(regex) { return regex.test(details.url); })) { - return { cancel: true }; - } - - var requestHeaders = details.requestHeaders; - var tabId = details.tabId; - - var useUserAgentMobile = false; - var setReferer = false; - - // if referer exists, set it to google - requestHeaders = requestHeaders.map(function(requestHeader) { - if (requestHeader.name === 'Referer') { - if (details.url.indexOf("cooking.nytimes.com/api/v1/users/bootstrap") !== -1) { - // this fixes images not being loaded on cooking.nytimes.com main page - // referrer has to be *nytimes.com otherwise returns 403 - requestHeader.value = 'https://cooking.nytimes.com'; - } else if (details.url.indexOf("wsj.com") !== -1 || details.url.indexOf("ft.com") !== -1) { - requestHeader.value = 'https://www.facebook.com/'; - } else { - requestHeader.value = 'https://www.google.com/'; - } - setReferer = true; - } - if (requestHeader.name === 'User-Agent') { - useUserAgentMobile = requestHeader.value.toLowerCase().includes("mobile"); - } - - return requestHeader; - }); - - // otherwise add it - if (!setReferer) { - if (details.url.indexOf("wsj.com") !== -1) { - requestHeaders.push({ - name: 'Referer', - value: 'https://www.facebook.com/' - }); - } else { - requestHeaders.push({ - name: 'Referer', - value: 'https://www.google.com/' - }); - } - } - - // override User-Agent to use Googlebot - var useGoogleBot = use_google_bot.filter(function(item) { - return typeof item == 'string' && details.url.indexOf(item) > -1; - }).length > 0; - - if (useGoogleBot) { - requestHeaders.push({ - "name": "User-Agent", - "value": useUserAgentMobile ? userAgentMobile : userAgentDesktop - }) - requestHeaders.push({ - "name": "X-Forwarded-For", - "value": "66.249.66.1" - }) - } - - // remove cookies before page load - requestHeaders = requestHeaders.map(function(requestHeader) { - for (var siteIndex in allow_cookies) { - if (details.url.indexOf(allow_cookies[siteIndex]) !== -1) { - return requestHeader; - } - } - if (requestHeader.name === 'Cookie') { - requestHeader.value = ''; - } - return requestHeader; - }); - - if (tabId !== -1) { - // run contentScript inside tab - browser.tabs.executeScript(tabId, { - file: 'contentScript.js', - runAt: 'document_start' - }, function(res) { - if (browser.runtime.lastError || res[0]) { - return; - } - }); - } - - return { requestHeaders: requestHeaders }; -}, { - urls: [''] -}, ['blocking', 'requestHeaders']); - -// remove cookies after page load -browser.webRequest.onCompleted.addListener(function(details) { - for (var domainIndex in remove_cookies) { - var domainVar = remove_cookies[domainIndex]; - if (!enabledSites.includes(domainVar) || details.url.indexOf(domainVar) === -1) { - continue; // don't remove cookies - } - browser.cookies.getAll({domain: domainVar}, function(cookies) { - for (var i=0; i"] -}); - -function isSiteEnabled(details) { - var isEnabled = enabledSites.some(function(enabledSite) { - var useSite = details.url.indexOf("." + enabledSite) !== -1; - if (enabledSite in restrictions) { - return useSite && details.url.indexOf(restrictions[enabledSite]) !== -1; - } - return useSite; - }); - return isEnabled; -} - -function getParameterByName(name, url) { - if (!url) url = window.location.href; - name = name.replace(/[\[\]]/g, '\\$&'); - var regex = new RegExp('[?&]' + name + '(=([^&#]*)|&|#|$)'), - results = regex.exec(url); - if (!results) return null; - if (!results[2]) return ''; - return decodeURIComponent(results[2].replace(/\+/g, ' ')); -} - -function stripQueryStringAndHashFromPath(url) { - return url.split("?")[0].split("#")[0]; -} +'use strict'; + +var defaultSites = { + 'Algemeen Dagblad': 'ad.nl', + 'Baltimore Sun': 'baltimoresun.com', + 'Barron\'s': 'barrons.com', + 'Bloomberg': 'bloomberg.com', + 'Business Insider': 'businessinsider.com', + 'Caixin': 'caixinglobal.com', + 'Chemical & Engineering News': 'cen.acs.org', + 'Chicago Tribune': 'chicagotribune.com', + 'Central Western Daily': 'centralwesterndaily.com.au', + 'Crain\'s Chicago Business': 'chicagobusiness.com', + 'Corriere Della Sera': 'corriere.it', + 'Daily Press': 'dailypress.com', + 'Denver Post': 'denverpost.com', + 'De Tijd': 'tijd.be', + 'De Groene Amsterdammer': 'groene.nl', + 'de Volkskrant': 'volkskrant.nl', + 'The Economist': 'economist.com', + 'Eindhovens Dagblad': 'ed.nl', + 'Examiner': 'examiner.com.au', + 'Financial Times': 'ft.com', + 'Foreign Policy': 'foreignpolicy.com', + 'Glassdoor': 'glassdoor.com', + 'Haaretz': 'haaretz.co.il', + 'Haaretz English': 'haaretz.com', + 'Handelsblatt': 'handelsblatt.com', + 'Hartford Courant': 'courant.com', + 'Harvard Business Review': 'hbr.org', + 'Inc.com': 'inc.com', + 'Investors Chronicle': 'investorschronicle.co.uk', + 'La Repubblica': 'repubblica.it', + 'Le Monde': 'lemonde.fr', + 'Le Temps': 'letemps.ch', + 'Los Angeles Times': 'latimes.com', + 'Medium': 'medium.com', + 'Medscape': 'medscape.com', + 'MIT Technology Review': 'technologyreview.com', + 'Mountain View Voice': 'mv-voice.com', + 'National Post': 'nationalpost.com', + 'New Statesman': 'newstatesman.com', + 'New York Magazine': 'nymag.com', + 'Nikkei Asian Review': 'asia.nikkei.com', + 'NRC': 'nrc.nl', + 'New Zealand Herald': 'nzherald.co.nz', + 'Orange County Register': 'ocregister.com', + 'Orlando Sentinel': 'orlandosentinel.com', + 'Palo Alto Online': 'paloaltoonline.com', + 'Parool': 'parool.nl', + 'Quartz': 'qz.com', + 'Quora': 'quora.com', + 'Statista': 'statista.com', + 'Telegraaf': 'telegraaf.nl', + 'SunSentinel': 'sun-sentinel.com', + 'The Advocate': 'theadvocate.com.au', + 'The Age': 'theage.com.au', + 'The Atlantic': 'theatlantic.com', + 'The Australian': 'theaustralian.com.au', + 'The Australian Financial Review': 'afr.com', + 'The Boston Globe': 'bostonglobe.com', + 'The Business Journals': 'bizjournals.com', + 'The Diplomat': 'thediplomat.com', + 'The Globe and Mail': 'theglobeandmail.com', + 'The Herald': 'theherald.com.au', + 'The Japan Times': 'japantimes.co.jp', + 'TheMarker': 'themarker.com', + 'The Mercury News': 'mercurynews.com', + 'The Morning Call': 'mcall.com', + 'The Nation': 'thenation.com', + 'The New York Times': 'nytimes.com', + 'The New Yorker': 'newyorker.com', + 'The News-Gazette': 'news-gazette.com', + 'The Saturday Paper': 'thesaturdaypaper.com.au', + 'The Spectator': 'spectator.co.uk', + 'The Seattle Times': 'seattletimes.com', + 'The Sydney Morning Herald': 'smh.com.au', + 'The Telegraph': 'telegraph.co.uk', + 'The Times': 'thetimes.co.uk', + 'The Toronto Star': 'thestar.com', + 'The Washington Post': 'washingtonpost.com', + 'The Wall Street Journal': 'wsj.com', + 'Towards Data Science': 'towardsdatascience.com', + 'Trouw': 'trouw.nl', + 'Vanity Fair': 'vanityfair.com', + 'Vrij Nederland': 'vn.nl', + 'Wired': 'wired.com' +}; + +const restrictions = { + 'barrons.com': 'barrons.com/articles' +} + +// Don't remove cookies before page load +const allow_cookies = [ +'ad.nl', +'asia.nikkei.com', +'bostonglobe.com', +'cen.acs.org', +'chicagobusiness.com', +'denverpost.com', +'economist.com', +'ed.nl', +'examiner.com.au', +'ft.com', +'hacked.com', +'hbr.org', +'lemonde.fr', +'letemps.ch', +'medium.com', +'mercurynews.com', +'newstatesman.com', +'nymag.com', +'nytimes.com', +'ocregister.com', +'parool.nl', +'qz.com', +'spectator.co.uk', +'telegraaf.nl', +'theadvocate.com.au', +'theage.com.au', +'theaustralian.com.au', +'thediplomat.com', +'thestar.com', +'towardsdatascience.com', +'trouw.nl', +'vn.nl', +'volkskrant.nl', +'washingtonpost.com', +'wsj.com', +] + +// Removes cookies after page load +const remove_cookies = [ +'ad.nl', +'asia.nikkei.com', +'bostonglobe.com', +'cen.acs.org', +'chicagobusiness.com', +'denverpost.com', +'economist.com', +'ed.nl', +'examiner.com.au', +'ft.com', +'hacked.com', +'hbr.org', +'letemps.ch', +'medium.com', +'mercurynews.com', +'newstatesman.com', +'nymag.com', +'nytimes.com', +'ocregister.com', +'qz.com', +'spectator.co.uk', +'telegraaf.nl', +'theadvocate.com.au', +'theage.com.au', +'thediplomat.com', +'thestar.com', +'towardsdatascience.com', +'vn.nl', +'washingtonpost.com', +'wsj.com', +] + +// select specific cookie(s) to hold from remove_cookies domains +const remove_cookies_select_hold = { + '.nrc.nl': ['nmt_closed_cookiebar'], + '.washingtonpost.com': ['wp_gdpr'], + '.wsj.com': ['wsjregion'] +} + +// select only specific cookie(s) to drop from remove_cookies domains +const remove_cookies_select_drop = { + 'www.nrc.nl': ['counter'] +} + +// Override User-Agent with Googlebot +const use_google_bot = [ +'barrons.com', +'nytimes.com', +'telegraph.co.uk', +'theaustralian.com.au', +'thetimes.co.uk', +'wsj.com', +] + +function setDefaultOptions() { + browser.storage.sync.set({ + sites: defaultSites + }, function() { + browser.runtime.openOptionsPage(); + }); +} + +const blockedRegexes = [ +/.+:\/\/.+\.tribdss\.com\//, +/thenation\.com\/.+\/paywall-script\.php/, +/haaretz\.co\.il\/htz\/js\/inter\.js/, +/nzherald\.co\.nz\/.+\/headjs\/.+\.js/ +]; + +const userAgentDesktop = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" +const userAgentMobile = "Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible ; Googlebot/2.1 ; +http://www.google.com/bot.html)" + +var enabledSites = []; + +// Get the enabled sites +browser.storage.sync.get({ + sites: {} +}, function(items) { + var sites = items.sites; + enabledSites = Object.keys(items.sites).map(function(key) { + return items.sites[key]; + }); +}); + +// Listen for changes to options +browser.storage.onChanged.addListener(function(changes, namespace) { + var key; + for (key in changes) { + var storageChange = changes[key]; + if (key === 'sites') { + var sites = storageChange.newValue; + enabledSites = Object.keys(sites).map(function(key) { + return sites[key]; + }); + } + } +}); + +// Set and show default options on install +browser.runtime.onInstalled.addListener(function(details) { + if (details.reason == "install") { + setDefaultOptions(); + } else if (details.reason == "update") { + // User updated extension + } +}); + +/** +// WSJ bypass +browser.webRequest.onBeforeSendHeaders.addListener(function(details) { + if (!isSiteEnabled(details) || details.url.indexOf("mod=rsswn") !== -1 || details.url.indexOf("/print-edition/") !== -1) { + return; + } + + var param; + var updatedUrl; + + param = getParameterByName("mod", details.url); + + if (param === null) { + updatedUrl = stripQueryStringAndHashFromPath(details.url); + updatedUrl += "?mod=rsswn"; + } else { + updatedUrl = details.url.replace(param, "rsswn"); + } + return { redirectUrl: updatedUrl}; +}, +{urls:["*://*.wsj.com/*"], types:["main_frame"]}, +["blocking"] +); +**/ + +// Disable javascript for these sites +chrome.webRequest.onBeforeRequest.addListener(function(details) { + if (!isSiteEnabled(details) || details.url.indexOf("mod=rsswn") !== -1) { + return; + } + return {cancel: true}; + }, + { + urls: ["*://*.theglobeandmail.com/*", "*://*.economist.com/*", "*://*.thestar.com/*", "*://*.newstatesman.com/*", "*://*.bostonglobe.com/*", "*://*.afr.com/*"], + types: ["script"] + }, + ["blocking"] +); + +browser.webRequest.onBeforeSendHeaders.addListener(function(details) { + if (!isSiteEnabled(details)) { + return; + } + + if (blockedRegexes.some(function(regex) { return regex.test(details.url); })) { + return { cancel: true }; + } + + var requestHeaders = details.requestHeaders; + var tabId = details.tabId; + + var useUserAgentMobile = false; + var setReferer = false; + + // if referer exists, set it to google + requestHeaders = requestHeaders.map(function(requestHeader) { + if (requestHeader.name === 'Referer') { + if (details.url.indexOf("cooking.nytimes.com/api/v1/users/bootstrap") !== -1) { + // this fixes images not being loaded on cooking.nytimes.com main page + // referrer has to be *nytimes.com otherwise returns 403 + requestHeader.value = 'https://cooking.nytimes.com'; + } else if (details.url.indexOf("wsj.com") !== -1 || details.url.indexOf("ft.com") !== -1) { + requestHeader.value = 'https://www.facebook.com/'; + } else { + requestHeader.value = 'https://www.google.com/'; + } + setReferer = true; + } + if (requestHeader.name === 'User-Agent') { + useUserAgentMobile = requestHeader.value.toLowerCase().includes("mobile"); + } + + return requestHeader; + }); + + // otherwise add it + if (!setReferer) { + if (details.url.indexOf("wsj.com") !== -1) { + requestHeaders.push({ + name: 'Referer', + value: 'https://www.facebook.com/' + }); + } else { + requestHeaders.push({ + name: 'Referer', + value: 'https://www.google.com/' + }); + } + } + + // override User-Agent to use Googlebot + var useGoogleBot = use_google_bot.filter(function(item) { + return typeof item == 'string' && details.url.indexOf(item) > -1; + }).length > 0; + + if (useGoogleBot) { + requestHeaders.push({ + "name": "User-Agent", + "value": useUserAgentMobile ? userAgentMobile : userAgentDesktop + }) + requestHeaders.push({ + "name": "X-Forwarded-For", + "value": "66.249.66.1" + }) + } + + // remove cookies before page load + requestHeaders = requestHeaders.map(function(requestHeader) { + for (var siteIndex in allow_cookies) { + if (details.url.indexOf(allow_cookies[siteIndex]) !== -1) { + return requestHeader; + } + } + if (requestHeader.name === 'Cookie') { + requestHeader.value = ''; + } + return requestHeader; + }); + + if (tabId !== -1) { + // run contentScript inside tab + browser.tabs.executeScript(tabId, { + file: 'contentScript.js', + runAt: 'document_start' + }, function(res) { + if (browser.runtime.lastError || res[0]) { + return; + } + }); + } + + return { requestHeaders: requestHeaders }; +}, { + urls: [''] +}, ['blocking', 'requestHeaders']); + +// remove cookies after page load +browser.webRequest.onCompleted.addListener(function(details) { + for (var domainIndex in remove_cookies) { + var domainVar = remove_cookies[domainIndex]; + if (!enabledSites.includes(domainVar) || details.url.indexOf(domainVar) === -1) { + continue; // don't remove cookies + } + browser.cookies.getAll({domain: domainVar}, function(cookies) { + for (var i=0; i"] +}); + +function isSiteEnabled(details) { + var isEnabled = enabledSites.some(function(enabledSite) { + var useSite = details.url.indexOf("." + enabledSite) !== -1; + if (enabledSite in restrictions) { + return useSite && details.url.indexOf(restrictions[enabledSite]) !== -1; + } + return useSite; + }); + return isEnabled; +} + +function getParameterByName(name, url) { + if (!url) url = window.location.href; + name = name.replace(/[\[\]]/g, '\\$&'); + var regex = new RegExp('[?&]' + name + '(=([^&#]*)|&|#|$)'), + results = regex.exec(url); + if (!results) return null; + if (!results[2]) return ''; + return decodeURIComponent(results[2].replace(/\+/g, ' ')); +} + +function stripQueryStringAndHashFromPath(url) { + return url.split("?")[0].split("#")[0]; +} From d56cf10714335e8c9640e7a29897ea986e4b5f95 Mon Sep 17 00:00:00 2001 From: magnolia1234 <7676006-magnolia1234@users.noreply.gitlab.com> Date: Tue, 29 Oct 2019 14:05:40 +0100 Subject: [PATCH 2/3] Fix styling --- background.js | 874 +++++++++++++++++++++++++------------------------- 1 file changed, 437 insertions(+), 437 deletions(-) diff --git a/background.js b/background.js index e5e6c15..882f056 100644 --- a/background.js +++ b/background.js @@ -1,437 +1,437 @@ -'use strict'; - -var defaultSites = { - 'Algemeen Dagblad': 'ad.nl', - 'Baltimore Sun': 'baltimoresun.com', - 'Barron\'s': 'barrons.com', - 'Bloomberg': 'bloomberg.com', - 'Business Insider': 'businessinsider.com', - 'Caixin': 'caixinglobal.com', - 'Chemical & Engineering News': 'cen.acs.org', - 'Chicago Tribune': 'chicagotribune.com', - 'Central Western Daily': 'centralwesterndaily.com.au', - 'Crain\'s Chicago Business': 'chicagobusiness.com', - 'Corriere Della Sera': 'corriere.it', - 'Daily Press': 'dailypress.com', - 'Denver Post': 'denverpost.com', - 'De Tijd': 'tijd.be', - 'De Groene Amsterdammer': 'groene.nl', - 'de Volkskrant': 'volkskrant.nl', - 'The Economist': 'economist.com', - 'Eindhovens Dagblad': 'ed.nl', - 'Examiner': 'examiner.com.au', - 'Financial Times': 'ft.com', - 'Foreign Policy': 'foreignpolicy.com', - 'Glassdoor': 'glassdoor.com', - 'Haaretz': 'haaretz.co.il', - 'Haaretz English': 'haaretz.com', - 'Handelsblatt': 'handelsblatt.com', - 'Hartford Courant': 'courant.com', - 'Harvard Business Review': 'hbr.org', - 'Inc.com': 'inc.com', - 'Investors Chronicle': 'investorschronicle.co.uk', - 'La Repubblica': 'repubblica.it', - 'Le Monde': 'lemonde.fr', - 'Le Temps': 'letemps.ch', - 'Los Angeles Times': 'latimes.com', - 'Medium': 'medium.com', - 'Medscape': 'medscape.com', - 'MIT Technology Review': 'technologyreview.com', - 'Mountain View Voice': 'mv-voice.com', - 'National Post': 'nationalpost.com', - 'New Statesman': 'newstatesman.com', - 'New York Magazine': 'nymag.com', - 'Nikkei Asian Review': 'asia.nikkei.com', - 'NRC': 'nrc.nl', - 'New Zealand Herald': 'nzherald.co.nz', - 'Orange County Register': 'ocregister.com', - 'Orlando Sentinel': 'orlandosentinel.com', - 'Palo Alto Online': 'paloaltoonline.com', - 'Parool': 'parool.nl', - 'Quartz': 'qz.com', - 'Quora': 'quora.com', - 'Statista': 'statista.com', - 'Telegraaf': 'telegraaf.nl', - 'SunSentinel': 'sun-sentinel.com', - 'The Advocate': 'theadvocate.com.au', - 'The Age': 'theage.com.au', - 'The Atlantic': 'theatlantic.com', - 'The Australian': 'theaustralian.com.au', - 'The Australian Financial Review': 'afr.com', - 'The Boston Globe': 'bostonglobe.com', - 'The Business Journals': 'bizjournals.com', - 'The Diplomat': 'thediplomat.com', - 'The Globe and Mail': 'theglobeandmail.com', - 'The Herald': 'theherald.com.au', - 'The Japan Times': 'japantimes.co.jp', - 'TheMarker': 'themarker.com', - 'The Mercury News': 'mercurynews.com', - 'The Morning Call': 'mcall.com', - 'The Nation': 'thenation.com', - 'The New York Times': 'nytimes.com', - 'The New Yorker': 'newyorker.com', - 'The News-Gazette': 'news-gazette.com', - 'The Saturday Paper': 'thesaturdaypaper.com.au', - 'The Spectator': 'spectator.co.uk', - 'The Seattle Times': 'seattletimes.com', - 'The Sydney Morning Herald': 'smh.com.au', - 'The Telegraph': 'telegraph.co.uk', - 'The Times': 'thetimes.co.uk', - 'The Toronto Star': 'thestar.com', - 'The Washington Post': 'washingtonpost.com', - 'The Wall Street Journal': 'wsj.com', - 'Towards Data Science': 'towardsdatascience.com', - 'Trouw': 'trouw.nl', - 'Vanity Fair': 'vanityfair.com', - 'Vrij Nederland': 'vn.nl', - 'Wired': 'wired.com' -}; - -const restrictions = { - 'barrons.com': 'barrons.com/articles' -} - -// Don't remove cookies before page load -const allow_cookies = [ -'ad.nl', -'asia.nikkei.com', -'bostonglobe.com', -'cen.acs.org', -'chicagobusiness.com', -'denverpost.com', -'economist.com', -'ed.nl', -'examiner.com.au', -'ft.com', -'hacked.com', -'hbr.org', -'lemonde.fr', -'letemps.ch', -'medium.com', -'mercurynews.com', -'newstatesman.com', -'nymag.com', -'nytimes.com', -'ocregister.com', -'parool.nl', -'qz.com', -'spectator.co.uk', -'telegraaf.nl', -'theadvocate.com.au', -'theage.com.au', -'theaustralian.com.au', -'thediplomat.com', -'thestar.com', -'towardsdatascience.com', -'trouw.nl', -'vn.nl', -'volkskrant.nl', -'washingtonpost.com', -'wsj.com', -] - -// Removes cookies after page load -const remove_cookies = [ -'ad.nl', -'asia.nikkei.com', -'bostonglobe.com', -'cen.acs.org', -'chicagobusiness.com', -'denverpost.com', -'economist.com', -'ed.nl', -'examiner.com.au', -'ft.com', -'hacked.com', -'hbr.org', -'letemps.ch', -'medium.com', -'mercurynews.com', -'newstatesman.com', -'nymag.com', -'nytimes.com', -'ocregister.com', -'qz.com', -'spectator.co.uk', -'telegraaf.nl', -'theadvocate.com.au', -'theage.com.au', -'thediplomat.com', -'thestar.com', -'towardsdatascience.com', -'vn.nl', -'washingtonpost.com', -'wsj.com', -] - -// select specific cookie(s) to hold from remove_cookies domains -const remove_cookies_select_hold = { - '.nrc.nl': ['nmt_closed_cookiebar'], - '.washingtonpost.com': ['wp_gdpr'], - '.wsj.com': ['wsjregion'] -} - -// select only specific cookie(s) to drop from remove_cookies domains -const remove_cookies_select_drop = { - 'www.nrc.nl': ['counter'] -} - -// Override User-Agent with Googlebot -const use_google_bot = [ -'barrons.com', -'nytimes.com', -'telegraph.co.uk', -'theaustralian.com.au', -'thetimes.co.uk', -'wsj.com', -] - -function setDefaultOptions() { - browser.storage.sync.set({ - sites: defaultSites - }, function() { - browser.runtime.openOptionsPage(); - }); -} - -const blockedRegexes = [ -/.+:\/\/.+\.tribdss\.com\//, -/thenation\.com\/.+\/paywall-script\.php/, -/haaretz\.co\.il\/htz\/js\/inter\.js/, -/nzherald\.co\.nz\/.+\/headjs\/.+\.js/ -]; - -const userAgentDesktop = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" -const userAgentMobile = "Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible ; Googlebot/2.1 ; +http://www.google.com/bot.html)" - -var enabledSites = []; - -// Get the enabled sites -browser.storage.sync.get({ - sites: {} -}, function(items) { - var sites = items.sites; - enabledSites = Object.keys(items.sites).map(function(key) { - return items.sites[key]; - }); -}); - -// Listen for changes to options -browser.storage.onChanged.addListener(function(changes, namespace) { - var key; - for (key in changes) { - var storageChange = changes[key]; - if (key === 'sites') { - var sites = storageChange.newValue; - enabledSites = Object.keys(sites).map(function(key) { - return sites[key]; - }); - } - } -}); - -// Set and show default options on install -browser.runtime.onInstalled.addListener(function(details) { - if (details.reason == "install") { - setDefaultOptions(); - } else if (details.reason == "update") { - // User updated extension - } -}); - -/** -// WSJ bypass -browser.webRequest.onBeforeSendHeaders.addListener(function(details) { - if (!isSiteEnabled(details) || details.url.indexOf("mod=rsswn") !== -1 || details.url.indexOf("/print-edition/") !== -1) { - return; - } - - var param; - var updatedUrl; - - param = getParameterByName("mod", details.url); - - if (param === null) { - updatedUrl = stripQueryStringAndHashFromPath(details.url); - updatedUrl += "?mod=rsswn"; - } else { - updatedUrl = details.url.replace(param, "rsswn"); - } - return { redirectUrl: updatedUrl}; -}, -{urls:["*://*.wsj.com/*"], types:["main_frame"]}, -["blocking"] -); -**/ - -// Disable javascript for these sites -chrome.webRequest.onBeforeRequest.addListener(function(details) { - if (!isSiteEnabled(details) || details.url.indexOf("mod=rsswn") !== -1) { - return; - } - return {cancel: true}; - }, - { - urls: ["*://*.theglobeandmail.com/*", "*://*.economist.com/*", "*://*.thestar.com/*", "*://*.newstatesman.com/*", "*://*.bostonglobe.com/*", "*://*.afr.com/*"], - types: ["script"] - }, - ["blocking"] -); - -browser.webRequest.onBeforeSendHeaders.addListener(function(details) { - if (!isSiteEnabled(details)) { - return; - } - - if (blockedRegexes.some(function(regex) { return regex.test(details.url); })) { - return { cancel: true }; - } - - var requestHeaders = details.requestHeaders; - var tabId = details.tabId; - - var useUserAgentMobile = false; - var setReferer = false; - - // if referer exists, set it to google - requestHeaders = requestHeaders.map(function(requestHeader) { - if (requestHeader.name === 'Referer') { - if (details.url.indexOf("cooking.nytimes.com/api/v1/users/bootstrap") !== -1) { - // this fixes images not being loaded on cooking.nytimes.com main page - // referrer has to be *nytimes.com otherwise returns 403 - requestHeader.value = 'https://cooking.nytimes.com'; - } else if (details.url.indexOf("wsj.com") !== -1 || details.url.indexOf("ft.com") !== -1) { - requestHeader.value = 'https://www.facebook.com/'; - } else { - requestHeader.value = 'https://www.google.com/'; - } - setReferer = true; - } - if (requestHeader.name === 'User-Agent') { - useUserAgentMobile = requestHeader.value.toLowerCase().includes("mobile"); - } - - return requestHeader; - }); - - // otherwise add it - if (!setReferer) { - if (details.url.indexOf("wsj.com") !== -1) { - requestHeaders.push({ - name: 'Referer', - value: 'https://www.facebook.com/' - }); - } else { - requestHeaders.push({ - name: 'Referer', - value: 'https://www.google.com/' - }); - } - } - - // override User-Agent to use Googlebot - var useGoogleBot = use_google_bot.filter(function(item) { - return typeof item == 'string' && details.url.indexOf(item) > -1; - }).length > 0; - - if (useGoogleBot) { - requestHeaders.push({ - "name": "User-Agent", - "value": useUserAgentMobile ? userAgentMobile : userAgentDesktop - }) - requestHeaders.push({ - "name": "X-Forwarded-For", - "value": "66.249.66.1" - }) - } - - // remove cookies before page load - requestHeaders = requestHeaders.map(function(requestHeader) { - for (var siteIndex in allow_cookies) { - if (details.url.indexOf(allow_cookies[siteIndex]) !== -1) { - return requestHeader; - } - } - if (requestHeader.name === 'Cookie') { - requestHeader.value = ''; - } - return requestHeader; - }); - - if (tabId !== -1) { - // run contentScript inside tab - browser.tabs.executeScript(tabId, { - file: 'contentScript.js', - runAt: 'document_start' - }, function(res) { - if (browser.runtime.lastError || res[0]) { - return; - } - }); - } - - return { requestHeaders: requestHeaders }; -}, { - urls: [''] -}, ['blocking', 'requestHeaders']); - -// remove cookies after page load -browser.webRequest.onCompleted.addListener(function(details) { - for (var domainIndex in remove_cookies) { - var domainVar = remove_cookies[domainIndex]; - if (!enabledSites.includes(domainVar) || details.url.indexOf(domainVar) === -1) { - continue; // don't remove cookies - } - browser.cookies.getAll({domain: domainVar}, function(cookies) { - for (var i=0; i"] -}); - -function isSiteEnabled(details) { - var isEnabled = enabledSites.some(function(enabledSite) { - var useSite = details.url.indexOf("." + enabledSite) !== -1; - if (enabledSite in restrictions) { - return useSite && details.url.indexOf(restrictions[enabledSite]) !== -1; - } - return useSite; - }); - return isEnabled; -} - -function getParameterByName(name, url) { - if (!url) url = window.location.href; - name = name.replace(/[\[\]]/g, '\\$&'); - var regex = new RegExp('[?&]' + name + '(=([^&#]*)|&|#|$)'), - results = regex.exec(url); - if (!results) return null; - if (!results[2]) return ''; - return decodeURIComponent(results[2].replace(/\+/g, ' ')); -} - -function stripQueryStringAndHashFromPath(url) { - return url.split("?")[0].split("#")[0]; -} +'use strict'; + +var defaultSites = { + 'Algemeen Dagblad': 'ad.nl', + 'Baltimore Sun': 'baltimoresun.com', + 'Barron\'s': 'barrons.com', + 'Bloomberg': 'bloomberg.com', + 'Business Insider': 'businessinsider.com', + 'Caixin': 'caixinglobal.com', + 'Chemical & Engineering News': 'cen.acs.org', + 'Chicago Tribune': 'chicagotribune.com', + 'Central Western Daily': 'centralwesterndaily.com.au', + 'Crain\'s Chicago Business': 'chicagobusiness.com', + 'Corriere Della Sera': 'corriere.it', + 'Daily Press': 'dailypress.com', + 'Denver Post': 'denverpost.com', + 'De Tijd': 'tijd.be', + 'De Groene Amsterdammer': 'groene.nl', + 'de Volkskrant': 'volkskrant.nl', + 'The Economist': 'economist.com', + 'Eindhovens Dagblad': 'ed.nl', + 'Examiner': 'examiner.com.au', + 'Financial Times': 'ft.com', + 'Foreign Policy': 'foreignpolicy.com', + 'Glassdoor': 'glassdoor.com', + 'Haaretz': 'haaretz.co.il', + 'Haaretz English': 'haaretz.com', + 'Handelsblatt': 'handelsblatt.com', + 'Hartford Courant': 'courant.com', + 'Harvard Business Review': 'hbr.org', + 'Inc.com': 'inc.com', + 'Investors Chronicle': 'investorschronicle.co.uk', + 'La Repubblica': 'repubblica.it', + 'Le Monde': 'lemonde.fr', + 'Le Temps': 'letemps.ch', + 'Los Angeles Times': 'latimes.com', + 'Medium': 'medium.com', + 'Medscape': 'medscape.com', + 'MIT Technology Review': 'technologyreview.com', + 'Mountain View Voice': 'mv-voice.com', + 'National Post': 'nationalpost.com', + 'New Statesman': 'newstatesman.com', + 'New York Magazine': 'nymag.com', + 'Nikkei Asian Review': 'asia.nikkei.com', + 'NRC': 'nrc.nl', + 'New Zealand Herald': 'nzherald.co.nz', + 'Orange County Register': 'ocregister.com', + 'Orlando Sentinel': 'orlandosentinel.com', + 'Palo Alto Online': 'paloaltoonline.com', + 'Parool': 'parool.nl', + 'Quartz': 'qz.com', + 'Quora': 'quora.com', + 'Statista': 'statista.com', + 'Telegraaf': 'telegraaf.nl', + 'SunSentinel': 'sun-sentinel.com', + 'The Advocate': 'theadvocate.com.au', + 'The Age': 'theage.com.au', + 'The Atlantic': 'theatlantic.com', + 'The Australian': 'theaustralian.com.au', + 'The Australian Financial Review': 'afr.com', + 'The Boston Globe': 'bostonglobe.com', + 'The Business Journals': 'bizjournals.com', + 'The Diplomat': 'thediplomat.com', + 'The Globe and Mail': 'theglobeandmail.com', + 'The Herald': 'theherald.com.au', + 'The Japan Times': 'japantimes.co.jp', + 'TheMarker': 'themarker.com', + 'The Mercury News': 'mercurynews.com', + 'The Morning Call': 'mcall.com', + 'The Nation': 'thenation.com', + 'The New York Times': 'nytimes.com', + 'The New Yorker': 'newyorker.com', + 'The News-Gazette': 'news-gazette.com', + 'The Saturday Paper': 'thesaturdaypaper.com.au', + 'The Spectator': 'spectator.co.uk', + 'The Seattle Times': 'seattletimes.com', + 'The Sydney Morning Herald': 'smh.com.au', + 'The Telegraph': 'telegraph.co.uk', + 'The Times': 'thetimes.co.uk', + 'The Toronto Star': 'thestar.com', + 'The Washington Post': 'washingtonpost.com', + 'The Wall Street Journal': 'wsj.com', + 'Towards Data Science': 'towardsdatascience.com', + 'Trouw': 'trouw.nl', + 'Vanity Fair': 'vanityfair.com', + 'Vrij Nederland': 'vn.nl', + 'Wired': 'wired.com' +}; + +const restrictions = { + 'barrons.com': 'barrons.com/articles' +} + +// Don't remove cookies before page load +const allow_cookies = [ +'ad.nl', +'asia.nikkei.com', +'bostonglobe.com', +'cen.acs.org', +'chicagobusiness.com', +'denverpost.com', +'economist.com', +'ed.nl', +'examiner.com.au', +'ft.com', +'hacked.com', +'hbr.org', +'lemonde.fr', +'letemps.ch', +'medium.com', +'mercurynews.com', +'newstatesman.com', +'nymag.com', +'nytimes.com', +'ocregister.com', +'parool.nl', +'qz.com', +'spectator.co.uk', +'telegraaf.nl', +'theadvocate.com.au', +'theage.com.au', +'theaustralian.com.au', +'thediplomat.com', +'thestar.com', +'towardsdatascience.com', +'trouw.nl', +'vn.nl', +'volkskrant.nl', +'washingtonpost.com', +'wsj.com', +] + +// Removes cookies after page load +const remove_cookies = [ +'ad.nl', +'asia.nikkei.com', +'bostonglobe.com', +'cen.acs.org', +'chicagobusiness.com', +'denverpost.com', +'economist.com', +'ed.nl', +'examiner.com.au', +'ft.com', +'hacked.com', +'hbr.org', +'letemps.ch', +'medium.com', +'mercurynews.com', +'newstatesman.com', +'nymag.com', +'nytimes.com', +'ocregister.com', +'qz.com', +'spectator.co.uk', +'telegraaf.nl', +'theadvocate.com.au', +'theage.com.au', +'thediplomat.com', +'thestar.com', +'towardsdatascience.com', +'vn.nl', +'washingtonpost.com', +'wsj.com', +] + +// select specific cookie(s) to hold from remove_cookies domains +const remove_cookies_select_hold = { + '.nrc.nl': ['nmt_closed_cookiebar'], + '.washingtonpost.com': ['wp_gdpr'], + '.wsj.com': ['wsjregion'] +} + +// select only specific cookie(s) to drop from remove_cookies domains +const remove_cookies_select_drop = { + 'www.nrc.nl': ['counter'] +} + +// Override User-Agent with Googlebot +const use_google_bot = [ +'barrons.com', +'nytimes.com', +'telegraph.co.uk', +'theaustralian.com.au', +'thetimes.co.uk', +'wsj.com', +] + +function setDefaultOptions() { + browser.storage.sync.set({ + sites: defaultSites + }, function() { + browser.runtime.openOptionsPage(); + }); +} + +const blockedRegexes = [ +/.+:\/\/.+\.tribdss\.com\//, +/thenation\.com\/.+\/paywall-script\.php/, +/haaretz\.co\.il\/htz\/js\/inter\.js/, +/nzherald\.co\.nz\/.+\/headjs\/.+\.js/ +]; + +const userAgentDesktop = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" +const userAgentMobile = "Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible ; Googlebot/2.1 ; +http://www.google.com/bot.html)" + +var enabledSites = []; + +// Get the enabled sites +browser.storage.sync.get({ + sites: {} +}, function(items) { + var sites = items.sites; + enabledSites = Object.keys(items.sites).map(function(key) { + return items.sites[key]; + }); +}); + +// Listen for changes to options +browser.storage.onChanged.addListener(function(changes, namespace) { + var key; + for (key in changes) { + var storageChange = changes[key]; + if (key === 'sites') { + var sites = storageChange.newValue; + enabledSites = Object.keys(sites).map(function(key) { + return sites[key]; + }); + } + } +}); + +// Set and show default options on install +browser.runtime.onInstalled.addListener(function(details) { + if (details.reason == "install") { + setDefaultOptions(); + } else if (details.reason == "update") { + // User updated extension + } +}); + +/** +// WSJ bypass +browser.webRequest.onBeforeSendHeaders.addListener(function(details) { + if (!isSiteEnabled(details) || details.url.indexOf("mod=rsswn") !== -1 || details.url.indexOf("/print-edition/") !== -1) { + return; + } + + var param; + var updatedUrl; + + param = getParameterByName("mod", details.url); + + if (param === null) { + updatedUrl = stripQueryStringAndHashFromPath(details.url); + updatedUrl += "?mod=rsswn"; + } else { + updatedUrl = details.url.replace(param, "rsswn"); + } + return { redirectUrl: updatedUrl}; +}, +{urls:["*://*.wsj.com/*"], types:["main_frame"]}, +["blocking"] +); +**/ + +// Disable javascript for these sites +chrome.webRequest.onBeforeRequest.addListener(function(details) { + if (!isSiteEnabled(details) || details.url.indexOf("mod=rsswn") !== -1) { + return; + } + return {cancel: true}; + }, + { + urls: ["*://*.theglobeandmail.com/*", "*://*.economist.com/*", "*://*.thestar.com/*", "*://*.newstatesman.com/*", "*://*.bostonglobe.com/*", "*://*.afr.com/*"], + types: ["script"] + }, + ["blocking"] +); + +browser.webRequest.onBeforeSendHeaders.addListener(function(details) { + if (!isSiteEnabled(details)) { + return; + } + + if (blockedRegexes.some(function(regex) { return regex.test(details.url); })) { + return { cancel: true }; + } + + var requestHeaders = details.requestHeaders; + var tabId = details.tabId; + + var useUserAgentMobile = false; + var setReferer = false; + + // if referer exists, set it to google + requestHeaders = requestHeaders.map(function(requestHeader) { + if (requestHeader.name === 'Referer') { + if (details.url.indexOf("cooking.nytimes.com/api/v1/users/bootstrap") !== -1) { + // this fixes images not being loaded on cooking.nytimes.com main page + // referrer has to be *nytimes.com otherwise returns 403 + requestHeader.value = 'https://cooking.nytimes.com'; + } else if (details.url.indexOf("wsj.com") !== -1 || details.url.indexOf("ft.com") !== -1) { + requestHeader.value = 'https://www.facebook.com/'; + } else { + requestHeader.value = 'https://www.google.com/'; + } + setReferer = true; + } + if (requestHeader.name === 'User-Agent') { + useUserAgentMobile = requestHeader.value.toLowerCase().includes("mobile"); + } + + return requestHeader; + }); + + // otherwise add it + if (!setReferer) { + if (details.url.indexOf("wsj.com") !== -1) { + requestHeaders.push({ + name: 'Referer', + value: 'https://www.facebook.com/' + }); + } else { + requestHeaders.push({ + name: 'Referer', + value: 'https://www.google.com/' + }); + } + } + + // override User-Agent to use Googlebot + var useGoogleBot = use_google_bot.filter(function(item) { + return typeof item == 'string' && details.url.indexOf(item) > -1; + }).length > 0; + + if (useGoogleBot) { + requestHeaders.push({ + "name": "User-Agent", + "value": useUserAgentMobile ? userAgentMobile : userAgentDesktop + }) + requestHeaders.push({ + "name": "X-Forwarded-For", + "value": "66.249.66.1" + }) + } + + // remove cookies before page load + requestHeaders = requestHeaders.map(function(requestHeader) { + for (var siteIndex in allow_cookies) { + if (details.url.indexOf(allow_cookies[siteIndex]) !== -1) { + return requestHeader; + } + } + if (requestHeader.name === 'Cookie') { + requestHeader.value = ''; + } + return requestHeader; + }); + + if (tabId !== -1) { + // run contentScript inside tab + browser.tabs.executeScript(tabId, { + file: 'contentScript.js', + runAt: 'document_start' + }, function(res) { + if (browser.runtime.lastError || res[0]) { + return; + } + }); + } + + return { requestHeaders: requestHeaders }; +}, { + urls: [''] +}, ['blocking', 'requestHeaders']); + +// remove cookies after page load +browser.webRequest.onCompleted.addListener(function(details) { + for (var domainIndex in remove_cookies) { + var domainVar = remove_cookies[domainIndex]; + if (!enabledSites.includes(domainVar) || details.url.indexOf(domainVar) === -1) { + continue; // don't remove cookies + } + browser.cookies.getAll({domain: domainVar}, function(cookies) { + for (var i=0; i"] +}); + +function isSiteEnabled(details) { + var isEnabled = enabledSites.some(function(enabledSite) { + var useSite = details.url.indexOf("." + enabledSite) !== -1; + if (enabledSite in restrictions) { + return useSite && details.url.indexOf(restrictions[enabledSite]) !== -1; + } + return useSite; + }); + return isEnabled; +} + +function getParameterByName(name, url) { + if (!url) url = window.location.href; + name = name.replace(/[\[\]]/g, '\\$&'); + var regex = new RegExp('[?&]' + name + '(=([^&#]*)|&|#|$)'), + results = regex.exec(url); + if (!results) return null; + if (!results[2]) return ''; + return decodeURIComponent(results[2].replace(/\+/g, ' ')); +} + +function stripQueryStringAndHashFromPath(url) { + return url.split("?")[0].split("#")[0]; +} From 7570338ac69a7b2a6b20abc33a9c4c0cfe9d1948 Mon Sep 17 00:00:00 2001 From: magnolia1234 <7676006-magnolia1234@users.noreply.gitlab.com> Date: Tue, 29 Oct 2019 19:27:54 +0100 Subject: [PATCH 3/3] Fix Quora 'read more' error Fixed 'read more' error by adding quora.com to googlebot settings. --- background.js | 1 + 1 file changed, 1 insertion(+) diff --git a/background.js b/background.js index 882f056..d7d2536 100644 --- a/background.js +++ b/background.js @@ -180,6 +180,7 @@ const remove_cookies_select_drop = { const use_google_bot = [ 'barrons.com', 'nytimes.com', +'quora.com', 'telegraph.co.uk', 'theaustralian.com.au', 'thetimes.co.uk',