Fix WSJ (amp 404 & restore option to disable Googlebot)

README.md

@@ -936,7 +936,7 @@ Grouped in options:\
 ##### Mexico
 [Mexico News Daily](https://mexiconewsdaily.com)
 ##### Peru
-*Grupo El Comercio* sites like
+*Grupo El Comercio* sites like\
 [Diario Correo](https://diariocorreo.pe) -
 [El Comercio](https://elcomercio.pe) -
 [Gestión](https://gestion.pe)

background.js

@@ -427,7 +427,7 @@ ext_api.storage.local.get({
   optIn: false,
   optInUpdate: true
 }, function (items) {
-  var sites = ext_name.includes('Clean') ? items.sites : {};
+  var sites = items.sites;
   optionSites = sites;
   var sites_default = items.sites_default;
   customSites = items.sites_custom;
@@ -496,7 +496,7 @@ ext_api.storage.local.get({
   check_sites_custom_ext();
   if (optin_update)
     check_update();
-  if (!Object.keys(items.sites).length)
+  if (!Object.keys(sites).length)
     ext_api.runtime.openOptionsPage();
 });
 
@@ -906,7 +906,7 @@ ext_api.tabs.onUpdated.addListener((tabId, changeInfo, tab) => {
     if ((tab_status && tab_status === 'complete') || (changeInfo.url)) {
       let timeout = changeInfo.url ? 500 : 0;
       setTimeout(function () {
-        if (matchUrlDomain(enabledSites, tab.url)) {
+        if (isSiteEnabled(tab)) {
           runOnTab(tab);
         }
         runOnTab_once(tab);
@@ -999,7 +999,7 @@ if (matchUrlDomain(change_headers, details.url) && !ignore_types.includes(detail
     !(matchUrlDomain(['economictimes.com', 'economictimes.indiatimes.com'], details.url) && !details.url.split(/\?|#/)[0].endsWith('.cms')) &&
     !(matchUrlDomain(au_news_corp_domains, details.url) && (details.url.includes('?amp') || (!matchUrlDomain(au_news_corp_no_amp_fix, details.url) && enabledSites.includes('#options_disable_gb_au_news_corp')))) &&
     !(matchUrlDomain('uol.com.br', details.url) && !matchUrlDomain('folha.uol.com.br', details.url)) &&
-    !(matchUrlDomain('wsj.com', details.url) && !details.url.match(/((\w)+(\-)+){3,}/) && details.type === 'main_frame' && mobile);
+    !(matchUrlDomain('wsj.com', details.url) && (enabledSites.includes('#options_disable_gb_wsj') || (!details.url.match(/((\w)+[%\-]+){3,}/) && details.type === 'main_frame' && mobile)));
   var bingbotEnabled = matchUrlDomain(use_bing_bot, details.url);
   var facebookbotEnabled = matchUrlDomain(use_facebook_bot, details.url);
 

changelog.txt

@@ -5,6 +5,7 @@ Updates (install signed xpi-file): https://gitlab.com/magnolia1234/bypass-paywal
 Post-release
 Remove DN.no (fix obsolete)
 Fix Mediahuis Noord (json)
+Fix WSJ (amp 404 & restore option to disable Googlebot)
 
 * v3.3.0.0 (2023-08-20)
 Add Cieletespace.fr

contentScript.js

@@ -2045,6 +2045,12 @@ else if (matchDomain('ilmanifesto.it')) {
 
 else if (matchDomain(['iltirreno.it', 'lanuovasardegna.it']) || matchDomain(['gazzettadimodena.it', 'gazzettadireggio.it', 'lanuovaferrara.it'])) {
   if (window.location.pathname.includes('/news/')) {
+    let paywall = document.querySelector('span > img[alt*="Paywall"]');
+    if (paywall) {
+      let header = paywall.parentNode.parentNode;
+      header_nofix(header);
+      removeDOMElement(paywall.parentNode);
+    }
     window.setTimeout(function () {
       let banners = document.querySelectorAll('div.MuiSnackbar-root, div.css-16cchgy');
       removeDOMElement(...banners);
@@ -5196,17 +5202,24 @@ else if (matchDomain('winnipegfreepress.com')) {
 }
 
 else if (matchDomain('wsj.com')) {
-  if (window.location.pathname.includes('/amp/')) {
+  let url_article = window.location.pathname.includes('/articles/');
-    amp_unhide_subscr_section();
+  let path_article = window.location.pathname.match(/((\w)+(\-)+){3,}\w+/);
-    let masthead_link = document.querySelector('div.masthead > a[href*="/articles/"]');
+  if (url_article || path_article) {
-    if (masthead_link)
+    if (window.location.pathname.startsWith('/amp/')) {
-      masthead_link.href = 'https://www.wsj.com';
+      amp_unhide_subscr_section();
-  } else {
+      let masthead_link = document.querySelector('div.masthead > a[href*="-"]');
-    let snippet = false;//document.querySelector('.snippet-promotion, div#cx-snippet-overlay');
+      if (masthead_link)
-    let wsj_pro = document.querySelector('meta[name="page.site"][content="wsjpro"]');
+        masthead_link.href = 'https://www.wsj.com';
-    if (snippet || wsj_pro) {
+    } else {
-      removeDOMElement(snippet, wsj_pro);
+      let snippet = document.querySelector('.snippet-promotion, div#cx-snippet-overlay');
-      window.location.href = window.location.href.replace('wsj.com', 'wsj.com/amp');
+      let wsj_pro = document.querySelector('meta[name="page.site"][content="wsjpro"]');
+      if (snippet || wsj_pro) {
+        removeDOMElement(snippet, wsj_pro);
+        if (url_article)
+          window.location.href = window.location.href.replace('wsj.com', 'wsj.com/amp');
+        else
+          window.location.href = '/amp/articles/' + path_article[0];
+      }
     }
   }
   let ads = document.querySelectorAll('div.wsj-ad, div.adWrapper, div.uds-ad-container');

custom/manifest.json

@@ -51,5 +51,5 @@
     "webRequestBlocking",
     "*://*/*"
   ],
-  "version": "3.3.0.2"
+  "version": "3.3.0.3"
 }

manifest.json

@@ -789,5 +789,5 @@
     "*://*.wyleex.com/*",
     "*://webcache.googleusercontent.com/*"
   ],
-  "version": "3.3.0.2"
+  "version": "3.3.0.3"
 }

sites.js

@@ -2293,7 +2293,7 @@ var defaultSites = {
     allow_cookies: 1,
     cs_dompurify: 1
   },
-  "The Wall Street Journal": {
+  "The Wall Street Journal (when blocked disable Googlebot in BPC-settings)": {
     domain: "wsj.com",
     allow_cookies: 1,
     block_regex: /(cdn\.cxense\.com\/|cdn\.ampproject\.org\/v\d\/amp-(access|subscriptions)-.+\.js)/,
@@ -2601,6 +2601,9 @@ var defaultSites = {
   "Barron's - no Googlebot (http error 500)": {
     domain: "#options_disable_gb_barrons"
   },
+  "The Wall Street Journal - no Googlebot (http error 500)": {
+    domain: "#options_disable_gb_wsj"
+  },
 }
 
 if (typeof browser !== 'object') {

sites_updated.json

@@ -1,22 +1,4 @@
 {
-  "America's Test Kitchen": {
-    "domain": "americastestkitchen.com",
-    "allow_cookies": 1,
-    "block_regex": "\\.tinypass\\.com",
-    "upd_version": "3.2.7.9"
-  },
-  "Autocar.co.uk": {
-    "domain": "autocar.co.uk",
-    "allow_cookies": 1,
-    "ld_google_webcache": "div.ms-block,div.register-block|div.content-wrapper",
-    "upd_version": "3.2.7.5"
-  },
-  "Bar and Bench": {
-    "domain": "barandbench.com",
-    "allow_cookies": 1,
-    "ld_json": "div#paywall-banner|div.arr--story-page-card-wrapper",
-    "upd_version": "3.2.7.7"
-  },
   "Bloomberg": {
     "domain": "bloomberg.com",
     "allow_cookies": 1,
@@ -35,12 +17,6 @@
     "block_regex": "\\.crusoe\\.com\\.br\\/assets\\/js\\/swg-wallcontent-crusoe\\.js",
     "upd_version": "3.2.9.8"
   },
-  "EUobserver": {
-    "domain": "euobserver.com",
-    "allow_cookies": 1,
-    "cs_code": "[{\"cond\":\"div.membership-upsell.show\", \"rm_class\":\"show\"}]",
-    "upd_version": "3.2.7.4"
-  },
   "GenomeWeb": {
     "domain": "###_usa_genomeweb",
     "group": [
@@ -93,13 +69,6 @@
     "block_regex": "\\.precisionmedicineonline\\.com\\/.+\\/js\\/js_.+\\.js",
     "upd_version": "3.2.9.5"
   },
-  "Swarajyamag": {
-    "domain": "swarajyamag.com",
-    "allow_cookies": 1,
-    "remove_cookies": 1,
-    "cs_code": "[{\"cond\":\"div#story-notification\", \"rm_elem\": 1, \"elems\":[{\"cond\":\"div#non-subscriber-text\", \"rm_attrib\": \"id\"}, {\"cond\":\"div.subscriber-text\", \"rm_attrib\": \"class\"}]}]",
-    "upd_version": "3.2.7.8"
-  },
   "The Athletic": {
     "domain": "###_usa_theathletic",
     "group": [
@@ -183,11 +152,5 @@
     "domain": "###_usa_townnews",
     "block_regex_general": "\\/shared-content\\/art\\/tncms\\/api\\/access\\.js",
     "upd_version": "3.2.8.3"
-  },
-  "Washington Examiner": {
-    "domain": "washingtonexaminer.com",
-    "allow_cookies": 1,
-    "block_regex": "\\.zephr\\.com\\/zephr-browser",
-    "upd_version": "3.2.7.8"
   }
 }

updates.json

@@ -2,8 +2,8 @@
   "addons": {
     "magnolia@12.34": {
       "updates": [
-        { "version": "3.2.9.0",
+        { "version": "3.3.0.0",
-            "update_link": "https://gitlab.com/magnolia1234/bpc-uploads/-/raw/master/bypass_paywalls_clean-3.2.9.0.xpi" }
+            "update_link": "https://gitlab.com/magnolia1234/bpc-uploads/-/raw/master/bypass_paywalls_clean-3.3.0.0.xpi" }
         ]
     }
   }