Update custom sites (set useragent Bingbot)

README.md

@@ -611,7 +611,7 @@ Check 'Options'-link in popup-menu and go to custom sites.
 
 Make sure the (new) site is checked under Options (or check on/off-button).  
 By default sites' cookies/local storage are removed after page loads (to bypass article limit).  
-Also you can enable Googlebot user-agent, disable Javascript for (sub)domain(s)/external sources and/or set the referer (to Facebook, Google or Twitter; ignored when Googlebot is set).
+Also you can enable Googlebot/Bingbot user-agent, disable Javascript for (sub)domain(s)/external sources and/or set the referer (to Facebook, Google or Twitter; ignored when Googlebot is set).
 
 PS on Android: enabling custom sites isn't working (no optional permissions).\
 Then you need the 'custom' add-on version (with access to all sites): [Bypass Paywalls Clean (custom)](https://addons.mozilla.org/en-US/firefox/addon/bypass-paywalls-clean-custom)

background.js

@@ -246,12 +246,13 @@ var use_google_bot_default = [
 var use_google_bot = use_google_bot_default.slice();
 
 // Override User-Agent with Bingbot
-var use_bing_bot = [
+var use_bing_bot_default = [
   'haaretz.co.il',
   'haaretz.com',
   'stratfor.com',
   'themarker.com',
 ];
+var use_bing_bot = use_bing_bot_default.slice();
 
 var use_facebook_referer_default = ['clarin.com', 'fd.nl', 'ilmanifesto.it', 'law.com', 'sloanreview.mit.edu'];
 var use_facebook_referer = use_facebook_referer_default.slice();
@@ -576,15 +577,17 @@ function add_grouped_sites(init_rules) {
     }
     for (let domain of nl_pg_domains)
       remove_cookies_select_drop[domain] = ['TID_ID'];
-    use_google_bot_default = use_google_bot.slice();
-    use_facebook_referer_default = use_facebook_referer.slice();
-    use_google_referer_default = use_google_referer.slice();
-    use_twitter_referer_default = use_twitter_referer.slice();
-    change_headers = use_google_bot.concat(use_bing_bot, use_facebook_referer, use_google_referer, use_twitter_referer, use_random_ip);
     for (let domain of usa_genomeweb_domains) {
       allow_cookies.push(domain);
       blockedRegexes[domain] = /crain-platform-.+-prod\.s3\.amazonaws\.com\/s3fs-public\/js\/js_.+\.js/;
     }
+
+    use_google_bot_default = use_google_bot.slice();
+    use_bing_bot_default = use_bing_bot.slice();
+    use_facebook_referer_default = use_facebook_referer.slice();
+    use_google_referer_default = use_google_referer.slice();
+    use_twitter_referer_default = use_twitter_referer.slice();
+    change_headers = use_google_bot.concat(use_bing_bot, use_facebook_referer, use_google_referer, use_twitter_referer, use_random_ip);
   }
 }
 
@@ -599,10 +602,37 @@ ext_api.storage.local.get({
   var sites_custom = items.sites_custom;
   excludedSites = items.sites_excluded;
 
+  enabledSites = Object.keys(sites).filter(function (key) {
+      return (sites[key] !== '' && sites[key] !== '###');
+    }).map(function (key) {
+      return sites[key].toLowerCase();
+    });
+  customSites = sites_custom;
+  customSites_domains = Object.values(sites_custom).map(x => x.domain);
+  disabledSites = defaultSites_domains.concat(customSites_domains).filter(x => !enabledSites.includes(x) && x !== '###');
+  add_grouped_sites(true);  //and exclude sites
+
+  for (let domainVar of enabledSites) {
+    if (!allow_cookies.includes(domainVar) && !remove_cookies.includes(domainVar)) {
+      allow_cookies.push(domainVar);
+      remove_cookies.push(domainVar);
+    }
+  }
+
   for (let key in sites_custom) {
     var domainVar = sites_custom[key]['domain'].toLowerCase();
     if (sites_custom[key]['googlebot'] > 0 && !use_google_bot.includes(domainVar))
       use_google_bot.push(domainVar);
+    switch (sites_custom[key]['useragent']) {
+    case 'googlebot':
+      if (!use_google_bot.includes(domainVar))
+        use_google_bot.push(domainVar);
+      break;
+    case 'bingbot':
+      if (!use_bing_bot.includes(domainVar))
+        use_bing_bot.push(domainVar);
+      break;
+    }
     if (sites_custom[key]['allow_cookies'] > 0 && !allow_cookies.includes(domainVar))
       allow_cookies.push(domainVar);
     if (sites_custom[key]['block_javascript'] > 0)
@@ -621,22 +651,6 @@ ext_api.storage.local.get({
     }
   }
 
-  enabledSites = Object.keys(sites).filter(function (key) {
-      return (sites[key] !== '' && sites[key] !== '###');
-    }).map(function (key) {
-      return sites[key].toLowerCase();
-    });
-  customSites = sites_custom;
-  customSites_domains = Object.values(sites_custom).map(x => x.domain);
-  disabledSites = defaultSites_domains.concat(customSites_domains).filter(x => !enabledSites.includes(x) && x !== '###');
-  add_grouped_sites(true);  //and exclude sites
-
-  for (let domainVar of enabledSites) {
-    if (!allow_cookies.includes(domainVar) && !remove_cookies.includes(domainVar)) {
-      allow_cookies.push(domainVar);
-      remove_cookies.push(domainVar);
-    }
-  }
   disableJavascriptOnListedSites();
 });
 
@@ -701,6 +715,7 @@ ext_api.storage.onChanged.addListener(function (changes, namespace) {
       }
 
       use_google_bot = use_google_bot_default.slice();
+      use_bing_bot = use_bing_bot_default.slice();
       use_facebook_referer = use_facebook_referer_default.slice();
       use_google_referer = use_google_referer_default.slice();
       use_twitter_referer = use_twitter_referer_default.slice();
@@ -711,6 +726,16 @@ ext_api.storage.onChanged.addListener(function (changes, namespace) {
         if (sites_custom[key]['googlebot'] > 0 && !use_google_bot.includes(domainVar)) {
           use_google_bot.push(domainVar);
         }
+        switch (sites_custom[key]['useragent']) {
+        case 'googlebot':
+          if (!use_google_bot.includes(domainVar))
+            use_google_bot.push(domainVar);
+          break;
+        case 'bingbot':
+          if (!use_bing_bot.includes(domainVar))
+            use_bing_bot.push(domainVar);
+          break;
+        }
         if (sites_custom[key]['allow_cookies'] > 0) {
           if (allow_cookies.includes(domainVar)) {
             if (remove_cookies.includes(domainVar))

changelog.txt

@@ -3,6 +3,7 @@ Changelog Bypass Paywalls Clean - Firefox
 
 Post-release
 Fix blocked referer
+Update custom sites (set useragent Bingbot)
 
 * v2.2.6.0 (2021-06-20)
 Add group Mediahuis Nederland Regional

manifest.json

@@ -529,5 +529,5 @@
         "*://*.wallkit.net/*",
         "*://*.wsj.net/*"
     ],
-    "version": "2.2.6.1"
+    "version": "2.2.6.2"
 }

options/options_custom.html

@@ -19,7 +19,7 @@
 <body>
   <h2>Custom Sites</h2>
   <div style="width:90%;">
-    To add a new site, enter an unique title/domain (without www.), select options for Googlebot/block Javascript (block on (sub)domain(s) of site and/or external domains) and/or set referer (ignored when Googlebot is set).<br>
+    To add a new site, enter an unique title/domain (without www.), select options for useragent (like Googlebot)/block Javascript (block on (sub)domain(s) of site and/or external domains) and/or set referer (ignored when Googlebot is set).<br>
     Custom sites are enabled automatically in <small><button><a href="options.html" style="text-decoration:none;color:inherit">Options</a></button></small> (cookies will be removed by default unless you enable allow_cookies).<br>
     If you want to use custom sites (for non-listed sites) enable it in <small><button><a href="optin/opt-in.html" style="text-decoration:none;color:inherit">Opt-in</a></button></small>
     <strong>Custom sites enabled: <span id="custom-enabled"></span></strong><br>

options/options_custom.js

@@ -1,5 +1,6 @@
 var ext_api = chrome || browser;
 
+var useragent_options = ['', 'googlebot', 'bingbot'];
 var referer_options = ['', 'facebook', 'google', 'twitter'];
 
 function capitalize(str) {
@@ -176,7 +177,7 @@ function edit_options() {
         var edit_site = sites_custom[title];
         document.querySelector('input[data-key="title"]').value = title;
         document.querySelector('input[data-key="domain"]').value = edit_site.domain;
-        document.querySelector('input[data-key="googlebot"]').checked = (edit_site.googlebot > 0);
+        document.querySelector('select[data-key="useragent"]').selectedIndex = (edit_site.googlebot > 0) ? 1 : useragent_options.indexOf(edit_site.useragent);
         document.querySelector('input[data-key="allow_cookies"]').checked = (edit_site.allow_cookies > 0);
         document.querySelector('input[data-key="block_javascript"]').checked = (edit_site.block_javascript > 0);
         document.querySelector('input[data-key="block_javascript_ext"]').checked = (edit_site.block_javascript_ext > 0);
@@ -234,7 +235,6 @@ function renderOptions() {
         var add_checkboxes = {
             'title': 0,
             'domain': 0,
-            'googlebot': 1,
             'allow_cookies': 1,
             'block_javascript': 1,
             'block_javascript_ext': 1
@@ -255,6 +255,20 @@ function renderOptions() {
             add_sitesEl.appendChild(labelEl);
         }
 
+        labelEl = document.createElement('label');
+        labelEl.appendChild(document.createTextNode('useragent '));
+        inputEl = document.createElement('select');
+        inputEl.dataset.key = 'useragent';
+        labelEl.appendChild(inputEl);
+
+        for (var i = 0; i < useragent_options.length; i++) {
+            var option = document.createElement("option");
+            option.value = useragent_options[i];
+            option.text = useragent_options[i];
+            inputEl.appendChild(option);
+        }
+        add_sitesEl.appendChild(labelEl);
+
         labelEl = document.createElement('label');
         labelEl.appendChild(document.createTextNode('referer '));
         inputEl = document.createElement('select');
@@ -289,6 +303,7 @@ function renderOptions() {
                 (sites_custom[key]['allow_cookies']>0 ? ' | allow_cookies' : '') + 
                 (sites_custom[key]['block_javascript']>0 ? ' | block javascript' : '') + 
                 (sites_custom[key]['block_javascript_ext']>0 ? ' | block javascript ext' : '') +
+                (sites_custom[key]['useragent'] ? ' | useragent: ' +  sites_custom[key]['useragent'] : '');
                 (sites_custom[key]['referer'] ? ' | referer: ' +  sites_custom[key]['referer'] : '');
             optionEl.value = key;
             selectEl.add(optionEl);