support piped instances in invidious list

.github/workflows/updateInvidous.yml

@@ -11,9 +11,10 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: recursive
-      - name: Download instance list
+      - name: Download instance lists
         run: |
-          wget https://api.invidious.io/instances.json -O ci/data.json
+          wget https://api.invidious.io/instances.json -O ci/invidious_instances.json
+          wget  https://github.com/TeamPiped/piped-uptime/raw/master/history/summary.json -O ci/piped_instances.json
       - name: Install dependencies
         run: npm ci
       - name: "Run CI"
@@ -24,7 +25,7 @@ jobs:
         # v4.2.3
         with:
           commit-message: Update Invidious List
-          author: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
+          author: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
           branch: ci/update_invidious_list
           title: Update Invidious List
           body: Automated Invidious list update

.gitignore

@@ -7,5 +7,6 @@ web-ext-artifacts
 dist/
 tmp/
 .DS_Store
-ci/data.json
+ci/invidious_instances.json
+ci/piped_instances.json
 test-results

ci/generateList.ts

@@ -0,0 +1,63 @@
+/*
+This file is only ran by GitHub Actions in order to populate the Invidious instances list
+
+This file should not be shipped with the extension
+*/
+
+/*
+Criteria for inclusion:
+Invidious
+- 30d uptime >= 90%
+- available for at least 80/90 days
+- must have been up for at least 90 days
+- HTTPS only
+- url includes name (this is to avoid redirects)
+
+Piped
+- 30d uptime >= 90%
+- available for at least 80/90 days
+- must have been up for at least 90 days
+- must not be a wildcard redirect to piped.video
+- must be currently up
+- must have a functioning frontend
+- must have a functioning API
+*/
+
+import { writeFile, existsSync } from "fs"
+import { join } from "path"
+import { getInvidiousList } from "./invidiousCI";
+// import { getPipedList } from "./pipedCI";
+
+const checkPath = (path: string) => existsSync(path);
+const fixArray = (arr: string[]) => [...new Set(arr)].sort()
+
+async function generateList() {
+  // import file from https://api.invidious.io/instances.json
+  const invidiousPath = join(__dirname, "invidious_instances.json");
+  // import file from https://github.com/TeamPiped/piped-uptime/raw/master/history/summary.json
+  const pipedPath = join(__dirname, "piped_instances.json");
+
+  // check if files exist
+  if (!checkPath(invidiousPath) || !checkPath(pipedPath)) {
+    console.log("Missing files")
+    process.exit(1);
+  }
+
+  // static non-invidious instances
+  const staticInstances = ["www.youtubekids.com"];
+  // invidious instances
+  const invidiousList = fixArray(getInvidiousList())
+  // piped instnaces
+  // const pipedList = fixArray(await getPipedList())
+
+  console.log([...staticInstances, ...invidiousList])
+
+  writeFile(
+    join(__dirname, "./invidiouslist.json"),
+    JSON.stringify([...staticInstances, ...invidiousList]),
+    (err) => {
+      if (err) return console.log(err);
+    }
+  );
+}
+generateList()

ci/invidiousCI.ts

@@ -1,23 +1,6 @@
-/*
-This file is only ran by GitHub Actions in order to populate the Invidious instances list
-
-This file should not be shipped with the extension
-*/
-
-import { writeFile, existsSync } from "fs"
-import { join } from "path"
 import { InvidiousInstance, instanceMap } from "./invidiousType"
 
-// import file from https://api.invidious.io/instances.json
+import * as data from "../ci/invidious_instances.json";
-if (!existsSync(join(__dirname, "data.json"))) {
-  process.exit(1);
-}
-// eslint-disable-next-line @typescript-eslint/ban-ts-comment
-// @ts-ignore
-import * as data from "../ci/data.json";
-
-// static non-invidious instances
-const staticInstances = ["www.youtubekids.com"];
 
 // only https servers
 const mapped: instanceMap = data
@@ -33,7 +16,7 @@ const mapped: instanceMap = data
 
 // reliability and sanity checks
 const reliableCheck = mapped
-  .filter((instance) => {
+  .filter(instance => {
     // 30d uptime >= 90%
     const thirtyDayUptime = Number(instance.thirtyDayUptime) >= 90;
     // available for at least 80/90 days
@@ -41,15 +24,8 @@ const reliableCheck = mapped
     return thirtyDayUptime && dailyRatioCheck.length >= 80;
   })
   // url includes name
-  .filter((instance) => instance.url.includes(instance.name));
+  .filter(instance => instance.url.includes(instance.name));
 
-// finally map to array
+export function getInvidiousList(): string[] {
-const result: string[] = reliableCheck.map((instance) => instance.name).sort();
+  return reliableCheck.map(instance => instance.name).sort()
-
+}
-writeFile(
-  join(__dirname, "./invidiouslist.json"),
-  JSON.stringify([...staticInstances, ...result]),
-  (err) => {
-    if (err) return console.log(err);
-  }
-);

ci/pipedCI.ts

@@ -0,0 +1,92 @@
+import * as data from "../ci/piped_instances.json";
+
+type percent = string
+type dailyMinutesDown = Record<string, number>
+
+type PipedInstance = {
+  name: string;
+  url: string;
+  icon: string;
+  slug: string;
+  status: string;
+  uptime: percent;
+  uptimeDay: percent;
+  uptimeWeek: percent;
+  uptimeMonth: percent;
+  uptimeYear: percent;
+  time: number;
+  timeDay: number;
+  timeWeek: number;
+  timeMonth: number;
+  timeYear: number;
+  dailyMinutesDown: dailyMinutesDown
+}
+
+const percentNumber = (percent: percent) => Number(percent.replace("%", ""))
+const ninetyDaysAgo = new Date(Date.now() - 90 * 24 * 60 * 60 * 1000)
+
+function dailyMinuteFilter (dailyMinutesDown: dailyMinutesDown) {
+  let daysDown = 0
+  for (const [date, minsDown] of Object.entries(dailyMinutesDown)) {
+    if (new Date(date) >= ninetyDaysAgo && minsDown > 1000) { // if within 90 days and down for more than 1000 minutes
+      daysDown++
+    }
+  }
+  // return true f less than 10 days down
+  return daysDown < 10
+}
+
+const getHost = (url: string) => new URL(url).host
+
+const getWatchPage = async (instance: PipedInstance) =>
+  fetch(`https://${getHost(instance.url)}`, { redirect: "manual" })
+    .then(res => res.headers.get("Location"))
+    .catch(e => { console.log (e); return null })
+
+const siteOK = async (instance) => {
+  // check if entire site is redirect
+  const notRedirect = await fetch(instance.url, { redirect: "manual" })
+    .then(res => res.status == 200)
+  // only allow kavin to return piped.video
+  // if (instance.url.startsWith("https://piped.video") && instance.slug !== "kavin-rocks-official") return false
+  // check if frontend is OK
+  const watchPageStatus = await fetch(instance.frontendUrl)
+    .then(res => res.ok)
+  // test API - stream returns ok result
+  const streamStatus = await fetch(`${instance.apiUrl}/streams/BaW_jenozKc`)
+    .then(res => res.ok)
+  // get startTime of monitor
+  const age = await fetch(instance.historyUrl)
+    .then(res => res.text())
+    .then(text => { // startTime greater than 90 days ago
+      const date = text.match(/startTime: (.+)/)[1]
+      return Date.parse(date) < ninetyDaysAgo.valueOf()
+    })
+  // console.log(notRedirect, watchPageStatus, streamStatus, age, instance.frontendUrl, instance.apiUrl)
+  return notRedirect && watchPageStatus && streamStatus && age
+}
+
+const staticFilters = (data as PipedInstance[])
+  .filter(instance => {
+    const isup = instance.status === "up"
+    const monthCheck = percentNumber(instance.uptimeMonth) >= 90
+    const dailyMinuteCheck = dailyMinuteFilter(instance.dailyMinutesDown)
+    return isup && monthCheck && dailyMinuteCheck
+  })
+  .map(async instance => {
+    // get frontend url
+    const frontendUrl = await getWatchPage(instance)
+    if (!frontendUrl) return null // return false if frontend doesn't resolve
+    // get api base
+    const apiUrl = instance.url.replace("/healthcheck", "")
+    const historyUrl = `https://raw.githubusercontent.com/TeamPiped/piped-uptime/master/history/${instance.slug}.yml`
+    const pass = await siteOK({ apiUrl, historyUrl, frontendUrl, url: instance.url })
+    const frontendHost = getHost(frontendUrl)
+    return pass ? frontendHost : null
+  })
+
+export async function getPipedList(): Promise<string[]> {
+  const instances = await Promise.all(staticFilters)
+    .then(arr => arr.filter(i => i !== null))
+  return instances
+}

package.json

@@ -55,7 +55,7 @@
     "build:watch": "npm run build:watch:chrome",
     "build:watch:chrome": "webpack --env browser=chrome --config webpack/webpack.dev.js --watch",
     "build:watch:firefox": "webpack --env browser=firefox --config webpack/webpack.dev.js --watch",
-    "ci:invidious": "ts-node ci/invidiousCI.ts",
+    "ci:invidious": "ts-node ci/generateList.ts",
     "dev": "npm run build:dev && concurrently \"npm run web-run\" \"npm run build:watch\"",
     "dev:firefox": "npm run build:dev:firefox && concurrently \"npm run web-run:firefox\" \"npm run build:watch:firefox\"",
     "dev:firefox-android": "npm run build:dev:firefox && concurrently \"npm run web-run:firefox-android\" \"npm run build:watch:firefox\"",