support piped instances in invidious list

This commit is contained in:
Michael C 2023-07-07 03:14:49 -04:00
parent 0d9c3dc807
commit 21bce341f0
No known key found for this signature in database
GPG key ID: FFB04FB3B878B7B4
6 changed files with 168 additions and 35 deletions

View file

@ -11,9 +11,10 @@ jobs:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
with: with:
submodules: recursive submodules: recursive
- name: Download instance list - name: Download instance lists
run: | run: |
wget https://api.invidious.io/instances.json -O ci/data.json wget https://api.invidious.io/instances.json -O ci/invidious_instances.json
wget https://github.com/TeamPiped/piped-uptime/raw/master/history/summary.json -O ci/piped_instances.json
- name: Install dependencies - name: Install dependencies
run: npm ci run: npm ci
- name: "Run CI" - name: "Run CI"
@ -24,7 +25,7 @@ jobs:
# v4.2.3 # v4.2.3
with: with:
commit-message: Update Invidious List commit-message: Update Invidious List
author: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> author: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
branch: ci/update_invidious_list branch: ci/update_invidious_list
title: Update Invidious List title: Update Invidious List
body: Automated Invidious list update body: Automated Invidious list update

3
.gitignore vendored
View file

@ -7,5 +7,6 @@ web-ext-artifacts
dist/ dist/
tmp/ tmp/
.DS_Store .DS_Store
ci/data.json ci/invidious_instances.json
ci/piped_instances.json
test-results test-results

63
ci/generateList.ts Normal file
View file

@ -0,0 +1,63 @@
/*
This file is only ran by GitHub Actions in order to populate the Invidious instances list
This file should not be shipped with the extension
*/
/*
Criteria for inclusion:
Invidious
- 30d uptime >= 90%
- available for at least 80/90 days
- must have been up for at least 90 days
- HTTPS only
- url includes name (this is to avoid redirects)
Piped
- 30d uptime >= 90%
- available for at least 80/90 days
- must have been up for at least 90 days
- must not be a wildcard redirect to piped.video
- must be currently up
- must have a functioning frontend
- must have a functioning API
*/
import { writeFile, existsSync } from "fs"
import { join } from "path"
import { getInvidiousList } from "./invidiousCI";
// import { getPipedList } from "./pipedCI";
const checkPath = (path: string) => existsSync(path);
const fixArray = (arr: string[]) => [...new Set(arr)].sort()
async function generateList() {
// import file from https://api.invidious.io/instances.json
const invidiousPath = join(__dirname, "invidious_instances.json");
// import file from https://github.com/TeamPiped/piped-uptime/raw/master/history/summary.json
const pipedPath = join(__dirname, "piped_instances.json");
// check if files exist
if (!checkPath(invidiousPath) || !checkPath(pipedPath)) {
console.log("Missing files")
process.exit(1);
}
// static non-invidious instances
const staticInstances = ["www.youtubekids.com"];
// invidious instances
const invidiousList = fixArray(getInvidiousList())
// piped instnaces
// const pipedList = fixArray(await getPipedList())
console.log([...staticInstances, ...invidiousList])
writeFile(
join(__dirname, "./invidiouslist.json"),
JSON.stringify([...staticInstances, ...invidiousList]),
(err) => {
if (err) return console.log(err);
}
);
}
generateList()

View file

@ -1,23 +1,6 @@
/*
This file is only ran by GitHub Actions in order to populate the Invidious instances list
This file should not be shipped with the extension
*/
import { writeFile, existsSync } from "fs"
import { join } from "path"
import { InvidiousInstance, instanceMap } from "./invidiousType" import { InvidiousInstance, instanceMap } from "./invidiousType"
// import file from https://api.invidious.io/instances.json import * as data from "../ci/invidious_instances.json";
if (!existsSync(join(__dirname, "data.json"))) {
process.exit(1);
}
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
import * as data from "../ci/data.json";
// static non-invidious instances
const staticInstances = ["www.youtubekids.com"];
// only https servers // only https servers
const mapped: instanceMap = data const mapped: instanceMap = data
@ -33,7 +16,7 @@ const mapped: instanceMap = data
// reliability and sanity checks // reliability and sanity checks
const reliableCheck = mapped const reliableCheck = mapped
.filter((instance) => { .filter(instance => {
// 30d uptime >= 90% // 30d uptime >= 90%
const thirtyDayUptime = Number(instance.thirtyDayUptime) >= 90; const thirtyDayUptime = Number(instance.thirtyDayUptime) >= 90;
// available for at least 80/90 days // available for at least 80/90 days
@ -41,15 +24,8 @@ const reliableCheck = mapped
return thirtyDayUptime && dailyRatioCheck.length >= 80; return thirtyDayUptime && dailyRatioCheck.length >= 80;
}) })
// url includes name // url includes name
.filter((instance) => instance.url.includes(instance.name)); .filter(instance => instance.url.includes(instance.name));
// finally map to array export function getInvidiousList(): string[] {
const result: string[] = reliableCheck.map((instance) => instance.name).sort(); return reliableCheck.map(instance => instance.name).sort()
}
writeFile(
join(__dirname, "./invidiouslist.json"),
JSON.stringify([...staticInstances, ...result]),
(err) => {
if (err) return console.log(err);
}
);

92
ci/pipedCI.ts Normal file
View file

@ -0,0 +1,92 @@
import * as data from "../ci/piped_instances.json";
type percent = string
type dailyMinutesDown = Record<string, number>
type PipedInstance = {
name: string;
url: string;
icon: string;
slug: string;
status: string;
uptime: percent;
uptimeDay: percent;
uptimeWeek: percent;
uptimeMonth: percent;
uptimeYear: percent;
time: number;
timeDay: number;
timeWeek: number;
timeMonth: number;
timeYear: number;
dailyMinutesDown: dailyMinutesDown
}
const percentNumber = (percent: percent) => Number(percent.replace("%", ""))
const ninetyDaysAgo = new Date(Date.now() - 90 * 24 * 60 * 60 * 1000)
function dailyMinuteFilter (dailyMinutesDown: dailyMinutesDown) {
let daysDown = 0
for (const [date, minsDown] of Object.entries(dailyMinutesDown)) {
if (new Date(date) >= ninetyDaysAgo && minsDown > 1000) { // if within 90 days and down for more than 1000 minutes
daysDown++
}
}
// return true f less than 10 days down
return daysDown < 10
}
const getHost = (url: string) => new URL(url).host
const getWatchPage = async (instance: PipedInstance) =>
fetch(`https://${getHost(instance.url)}`, { redirect: "manual" })
.then(res => res.headers.get("Location"))
.catch(e => { console.log (e); return null })
const siteOK = async (instance) => {
// check if entire site is redirect
const notRedirect = await fetch(instance.url, { redirect: "manual" })
.then(res => res.status == 200)
// only allow kavin to return piped.video
// if (instance.url.startsWith("https://piped.video") && instance.slug !== "kavin-rocks-official") return false
// check if frontend is OK
const watchPageStatus = await fetch(instance.frontendUrl)
.then(res => res.ok)
// test API - stream returns ok result
const streamStatus = await fetch(`${instance.apiUrl}/streams/BaW_jenozKc`)
.then(res => res.ok)
// get startTime of monitor
const age = await fetch(instance.historyUrl)
.then(res => res.text())
.then(text => { // startTime greater than 90 days ago
const date = text.match(/startTime: (.+)/)[1]
return Date.parse(date) < ninetyDaysAgo.valueOf()
})
// console.log(notRedirect, watchPageStatus, streamStatus, age, instance.frontendUrl, instance.apiUrl)
return notRedirect && watchPageStatus && streamStatus && age
}
const staticFilters = (data as PipedInstance[])
.filter(instance => {
const isup = instance.status === "up"
const monthCheck = percentNumber(instance.uptimeMonth) >= 90
const dailyMinuteCheck = dailyMinuteFilter(instance.dailyMinutesDown)
return isup && monthCheck && dailyMinuteCheck
})
.map(async instance => {
// get frontend url
const frontendUrl = await getWatchPage(instance)
if (!frontendUrl) return null // return false if frontend doesn't resolve
// get api base
const apiUrl = instance.url.replace("/healthcheck", "")
const historyUrl = `https://raw.githubusercontent.com/TeamPiped/piped-uptime/master/history/${instance.slug}.yml`
const pass = await siteOK({ apiUrl, historyUrl, frontendUrl, url: instance.url })
const frontendHost = getHost(frontendUrl)
return pass ? frontendHost : null
})
export async function getPipedList(): Promise<string[]> {
const instances = await Promise.all(staticFilters)
.then(arr => arr.filter(i => i !== null))
return instances
}

View file

@ -55,7 +55,7 @@
"build:watch": "npm run build:watch:chrome", "build:watch": "npm run build:watch:chrome",
"build:watch:chrome": "webpack --env browser=chrome --config webpack/webpack.dev.js --watch", "build:watch:chrome": "webpack --env browser=chrome --config webpack/webpack.dev.js --watch",
"build:watch:firefox": "webpack --env browser=firefox --config webpack/webpack.dev.js --watch", "build:watch:firefox": "webpack --env browser=firefox --config webpack/webpack.dev.js --watch",
"ci:invidious": "ts-node ci/invidiousCI.ts", "ci:invidious": "ts-node ci/generateList.ts",
"dev": "npm run build:dev && concurrently \"npm run web-run\" \"npm run build:watch\"", "dev": "npm run build:dev && concurrently \"npm run web-run\" \"npm run build:watch\"",
"dev:firefox": "npm run build:dev:firefox && concurrently \"npm run web-run:firefox\" \"npm run build:watch:firefox\"", "dev:firefox": "npm run build:dev:firefox && concurrently \"npm run web-run:firefox\" \"npm run build:watch:firefox\"",
"dev:firefox-android": "npm run build:dev:firefox && concurrently \"npm run web-run:firefox-android\" \"npm run build:watch:firefox\"", "dev:firefox-android": "npm run build:dev:firefox && concurrently \"npm run web-run:firefox-android\" \"npm run build:watch:firefox\"",