From 2ad52e70bbd48ad586e53bc681f65801e9dc73ea Mon Sep 17 00:00:00 2001 From: Michael C Date: Tue, 5 Oct 2021 23:31:05 -0400 Subject: [PATCH 01/22] enforce gzip for downloads --- nginx/nginx.conf | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 3204402..4f85aed 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -17,6 +17,10 @@ http { include /etc/nginx/mime.types; include /etc/nginx/proxy.conf; include /etc/nginx/fastcgi.conf; + ## Custom MIME definition + types { + text/csv csv; + } upstream backend_GET { least_conn; @@ -124,7 +128,8 @@ http { location /download/ { access_log /etc/nginx/logs/download.log no_ip; gzip on; - gzip_types text/plain application/json; + gzip_types text/csv; + gzip_comp_level 1; alias /home/sbadmin/sponsor/docker/database-export/; #return 307 https://cdnsponsor.ajay.app$request_uri; } From ac15686b47e47c74a91f9773b408a404aa3a22ef Mon Sep 17 00:00:00 2001 From: Michael C Date: Wed, 20 Oct 2021 00:16:27 -0400 Subject: [PATCH 02/22] add loadAvg to status --- src/routes/getStatus.ts | 4 +++- test/cases/getStatus.ts | 13 +++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/routes/getStatus.ts b/src/routes/getStatus.ts index 704ffaf..b4ea254 100644 --- a/src/routes/getStatus.ts +++ b/src/routes/getStatus.ts @@ -1,6 +1,7 @@ import { db } from "../databases/databases"; import { Logger } from "../utils/logger"; import { Request, Response } from "express"; +import os from "os"; export async function getStatus(req: Request, res: Response): Promise { const startTime = Date.now(); @@ -14,8 +15,9 @@ export async function getStatus(req: Request, res: Response): Promise db: Number(dbVersion), startTime, processTime: Date.now() - startTime, + loadavg: os.loadavg().slice(1) // only return 5 & 15 minute load average }; - return value ? res.send(String(statusValues[value])) : res.send(statusValues); + return value ? res.send(JSON.stringify(statusValues[value])) : res.send(statusValues); } catch (err) { Logger.error(err as string); return res.sendStatus(500); diff --git a/test/cases/getStatus.ts b/test/cases/getStatus.ts index 5d175d7..10b10db 100644 --- a/test/cases/getStatus.ts +++ b/test/cases/getStatus.ts @@ -19,6 +19,7 @@ describe("getStatus", () => { assert.strictEqual(data.db, Number(dbVersion)); assert.ok(data.startTime); assert.ok(data.processTime >= 0); + assert.ok(data.loadavg.length == 2); done(); }) .catch(err => done(err)); @@ -74,4 +75,16 @@ describe("getStatus", () => { }) .catch(err => done(err)); }); + + it("Should be able to get loadavg only", (done) => { + client.get(`${endpoint}/loadavg`) + .then(res => { + console.log(res.data); + assert.strictEqual(res.status, 200); + assert.ok(Number(res.data[0]) >= 0); + assert.ok(Number(res.data[1]) >= 0); + done(); + }) + .catch(err => done(err)); + }); }); From 109578a3eda14418c9af8fbba96f14a5e447c086 Mon Sep 17 00:00:00 2001 From: Michael C Date: Wed, 20 Oct 2021 00:59:08 -0400 Subject: [PATCH 03/22] remove extra console.log lines --- test/cases/getStatus.ts | 1 - test/cases/setUsername.ts | 1 - test/cases/voteOnSponsorTime.ts | 2 -- 3 files changed, 4 deletions(-) diff --git a/test/cases/getStatus.ts b/test/cases/getStatus.ts index 10b10db..b515427 100644 --- a/test/cases/getStatus.ts +++ b/test/cases/getStatus.ts @@ -79,7 +79,6 @@ describe("getStatus", () => { it("Should be able to get loadavg only", (done) => { client.get(`${endpoint}/loadavg`) .then(res => { - console.log(res.data); assert.strictEqual(res.status, 200); assert.ok(Number(res.data[0]) >= 0); assert.ok(Number(res.data[1]) >= 0); diff --git a/test/cases/setUsername.ts b/test/cases/setUsername.ts index f1f3f15..89af032 100644 --- a/test/cases/setUsername.ts +++ b/test/cases/setUsername.ts @@ -92,7 +92,6 @@ describe("setUsername", () => { it("Should be able to set username that has never been set", (done) => { postSetUserName(user00PrivateUserID, username00) .then(async res => { - console.log(res.data); const usernameInfo = await getUsernameInfo(getHash(user00PrivateUserID)); assert.strictEqual(res.status, 200); assert.strictEqual(usernameInfo.userName, username00); diff --git a/test/cases/voteOnSponsorTime.ts b/test/cases/voteOnSponsorTime.ts index 377c7d2..fcb823d 100644 --- a/test/cases/voteOnSponsorTime.ts +++ b/test/cases/voteOnSponsorTime.ts @@ -299,7 +299,6 @@ describe("voteOnSponsorTime", () => { .then(async res => { assert.strictEqual(res.status, 200); const row = await getSegmentCategory(UUID); - console.log(row.category); assert.strictEqual(row.category, category); done(); }) @@ -314,7 +313,6 @@ describe("voteOnSponsorTime", () => { .then(async res => { assert.strictEqual(res.status, 200); const row = await getSegmentCategory(UUID); - console.log(row.category); assert.strictEqual(row.category, "intro"); done(); }) From 815df94db4cd2097297fd5be7199dcedc363079c Mon Sep 17 00:00:00 2001 From: Michael C Date: Wed, 20 Oct 2021 01:01:58 -0400 Subject: [PATCH 04/22] add eslint rule for no-console --- .eslintrc.js | 1 + src/utils/logger.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/.eslintrc.js b/.eslintrc.js index 14b67b2..e96d7da 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -27,5 +27,6 @@ module.exports = { "indent": ["warn", 4, { "SwitchCase": 1 }], "object-curly-spacing": ["warn", "always"], "require-await": "warn", + "no-console": "error" }, }; diff --git a/src/utils/logger.ts b/src/utils/logger.ts index 56a7e95..8507a46 100644 --- a/src/utils/logger.ts +++ b/src/utils/logger.ts @@ -79,6 +79,7 @@ class Logger { if (levelStr.length === 4) { levelStr += " "; // ensure logs are aligned } + // eslint-disable-next-line no-console console.log(colors.Dim, `${levelStr} ${new Date().toISOString()}: `, color, str, colors.Reset); } } From 0163a52e5599a2f729412ebbeeaecaa15a25faa4 Mon Sep 17 00:00:00 2001 From: Michael C Date: Thu, 21 Oct 2021 03:49:25 -0400 Subject: [PATCH 05/22] if in mirror mode, import CSV files --- src/databases/databases.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/databases/databases.ts b/src/databases/databases.ts index 96a5b5f..be15a32 100644 --- a/src/databases/databases.ts +++ b/src/databases/databases.ts @@ -3,7 +3,7 @@ import { Sqlite } from "./Sqlite"; import { Mysql } from "./Mysql"; import { Postgres } from "./Postgres"; import { IDatabase } from "./IDatabase"; - +import { readdirSync } from "fs"; let db: IDatabase; let privateDB: IDatabase; @@ -68,6 +68,14 @@ async function initDb(): Promise { // Attach private db to main db (db as Sqlite).attachDatabase(config.privateDB, "privateDB"); } + + if (config.mode === "mirror") { + readdirSync("/mirror").forEach(async file => { + const fileName = file.slice(0,-4); + const filePath = `/mirror/${file}`; + await db.prepare("run", `COPY "${fileName}" FROM '${filePath}' WITH (FORMAT CSV, HEADER true);`); + }); + } } export { From c6428bf9e45aa9e9e222ecd19079c0dd0c75a76f Mon Sep 17 00:00:00 2001 From: Michael C Date: Thu, 21 Oct 2021 23:15:36 -0400 Subject: [PATCH 06/22] import without fs --- src/databases/databases.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/databases/databases.ts b/src/databases/databases.ts index be15a32..5e2ae0e 100644 --- a/src/databases/databases.ts +++ b/src/databases/databases.ts @@ -3,7 +3,6 @@ import { Sqlite } from "./Sqlite"; import { Mysql } from "./Mysql"; import { Postgres } from "./Postgres"; import { IDatabase } from "./IDatabase"; -import { readdirSync } from "fs"; let db: IDatabase; let privateDB: IDatabase; @@ -69,12 +68,13 @@ async function initDb(): Promise { (db as Sqlite).attachDatabase(config.privateDB, "privateDB"); } - if (config.mode === "mirror") { - readdirSync("/mirror").forEach(async file => { - const fileName = file.slice(0,-4); - const filePath = `/mirror/${file}`; - await db.prepare("run", `COPY "${fileName}" FROM '${filePath}' WITH (FORMAT CSV, HEADER true);`); - }); + if (config.mode === "mirror" && db instanceof Postgres) { + const tables = config?.dumpDatabase?.tables ?? []; + const tableNames = tables.map(table => table.name); + for (const table of tableNames) { + const filePath = `${config?.dumpDatabase?.postgresExportPath}/${table}.csv`; + await db.prepare("run", `COPY "${table}" FROM '${filePath}' WITH (FORMAT CSV, HEADER true);`); + } } } From 2d10dd6c9c304764e6cd80a2be05142d7f3912a6 Mon Sep 17 00:00:00 2001 From: Michael C Date: Fri, 22 Oct 2021 01:05:08 -0400 Subject: [PATCH 07/22] add extension to db.sql --- databases/_sponsorTimes.db.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/databases/_sponsorTimes.db.sql b/databases/_sponsorTimes.db.sql index 38ed203..1ab5210 100644 --- a/databases/_sponsorTimes.db.sql +++ b/databases/_sponsorTimes.db.sql @@ -38,5 +38,6 @@ CREATE TABLE IF NOT EXISTS "config" ( ); CREATE EXTENSION IF NOT EXISTS pgcrypto; --!sqlite-ignore +CREATE EXTENSION IF NOT EXISTS pg_trgm; --!sqlite-ignore COMMIT; \ No newline at end of file From fd6ae8fc0e32b5b8ef15a2bfc8b2c31fbb5d9f99 Mon Sep 17 00:00:00 2001 From: Michael C Date: Fri, 22 Oct 2021 01:26:08 -0400 Subject: [PATCH 08/22] bargaining with postgres CI - fix tests with lockCategories - new unique naming scheme for video - super janky somehow working method for comparing arrays out of order --- config.json.backup | 65 +++++++++++++++++++++++++++++++++ test/cases/getLockCategories.ts | 29 ++++++++------- test/cases/postSkipSegments.ts | 48 ++++++++++++------------ test/utils/partialDeepEquals.ts | 33 ++++++++++++++++- 4 files changed, 136 insertions(+), 39 deletions(-) create mode 100644 config.json.backup diff --git a/config.json.backup b/config.json.backup new file mode 100644 index 0000000..1b0ced2 --- /dev/null +++ b/config.json.backup @@ -0,0 +1,65 @@ +{ + "port": 8080, + "mockPort": 8081, + "globalSalt": "testSalt", + "adminUserID": "4bdfdc9cddf2c7d07a8a87b57bf6d25389fb75d1399674ee0e0938a6a60f4c3b", + "newLeafURLs": ["placeholder"], + "discordReportChannelWebhookURL": "http://127.0.0.1:8081/ReportChannelWebhook", + "discordFirstTimeSubmissionsWebhookURL": "http://127.0.0.1:8081/FirstTimeSubmissionsWebhook", + "discordCompletelyIncorrectReportWebhookURL": "http://127.0.0.1:8081/CompletelyIncorrectReportWebhook", + "discordNeuralBlockRejectWebhookURL": "http://127.0.0.1:8081/NeuralBlockRejectWebhook", + "neuralBlockURL": "http://127.0.0.1:8081/NeuralBlock", + "behindProxy": true, + "postgres": { + "user": "ci_db_user", + "password": "ci_db_pass", + "host": "bf.mchang.icu", + "port": 5432 + }, + "createDatabaseIfNotExist": true, + "schemaFolder": "./databases", + "dbSchema": "./databases/_sponsorTimes.db.sql", + "privateDBSchema": "./databases/_private.db.sql", + "mode": "test", + "readOnly": false, + "webhooks": [ + { + "url": "http://127.0.0.1:8081/CustomWebhook", + "key": "superSecretKey", + "scopes": [ + "vote.up", + "vote.down" + ] + }, { + "url": "http://127.0.0.1:8081/FailedWebhook", + "key": "superSecretKey", + "scopes": [ + "vote.up", + "vote.down" + ] + }, { + "url": "http://127.0.0.1:8099/WrongPort", + "key": "superSecretKey", + "scopes": [ + "vote.up", + "vote.down" + ] + } + ], + "categoryList": ["sponsor", "selfpromo", "interaction", "intro", "outro", "preview", "music_offtopic", "poi_highlight"], + "maxNumberOfActiveWarnings": 3, + "hoursAfterWarningExpires": 24, + "rateLimit": { + "vote": { + "windowMs": 900000, + "max": 20, + "message": "Too many votes, please try again later", + "statusCode": 429 + }, + "view": { + "windowMs": 900000, + "max": 20, + "statusCode": 200 + } + } +} diff --git a/test/cases/getLockCategories.ts b/test/cases/getLockCategories.ts index b8285bf..9666bda 100644 --- a/test/cases/getLockCategories.ts +++ b/test/cases/getLockCategories.ts @@ -2,6 +2,7 @@ import { getHash } from "../../src/utils/getHash"; import { db } from "../../src/databases/databases"; import assert from "assert"; import { client } from "../utils/httpClient"; +import { mixedDeepEquals } from "../utils/partialDeepEquals"; const endpoint = "/api/lockCategories"; const getLockCategories = (videoID: string) => client.get(endpoint, { params: { videoID } }); const getLockCategoriesWithService = (videoID: string, service: string) => client.get(endpoint, { params: { videoID, service } }); @@ -12,13 +13,13 @@ describe("getLockCategories", () => { await db.prepare("run", insertVipUserQuery, [getHash("getLockCategoriesVIP")]); const insertLockCategoryQuery = 'INSERT INTO "lockCategories" ("userID", "videoID", "category", "reason", "service") VALUES (?, ?, ?, ?, ?)'; - await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLock1", "sponsor", "1-short", "YouTube"]); - await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLock1", "interaction", "1-longer-reason", "YouTube"]); + await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLockCategory1", "sponsor", "1-short", "YouTube"]); + await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLockCategory1", "interaction", "1-longer-reason", "YouTube"]); - await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLock2", "preview", "2-reason", "YouTube"]); + await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLockCategory2", "preview", "2-reason", "YouTube"]); - await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLock3", "nonmusic", "3-reason", "PeerTube"]); - await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLock3", "sponsor", "3-reason", "YouTube"]); + await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLockCategory3", "nonmusic", "3-reason", "PeerTube"]); + await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLockCategory3", "sponsor", "3-reason", "YouTube"]); }); it("Should update the database version when starting the application", async () => { @@ -27,7 +28,7 @@ describe("getLockCategories", () => { }); it("Should be able to get multiple locks", (done) => { - getLockCategories("getLock1") + getLockCategories("getLockCategory1") .then(res => { assert.strictEqual(res.status, 200); const expected = { @@ -37,14 +38,14 @@ describe("getLockCategories", () => { ], reason: "1-longer-reason" }; - assert.deepStrictEqual(res.data, expected); + assert.ok(mixedDeepEquals(res.data, expected)); done(); }) .catch(err => done(err)); }); it("Should be able to get single locks", (done) => { - getLockCategories("getLock2") + getLockCategories("getLockCategory2") .then(res => { assert.strictEqual(res.status, 200); const expected = { @@ -60,7 +61,7 @@ describe("getLockCategories", () => { }); it("should return 404 if no lock exists", (done) => { - getLockCategories("getLockNull") + getLockCategories("getLockCategoryNull") .then(res => { assert.strictEqual(res.status, 404); done(); @@ -78,7 +79,7 @@ describe("getLockCategories", () => { }); it("Should be able to get multiple locks with service", (done) => { - getLockCategoriesWithService("getLock1", "YouTube") + getLockCategoriesWithService("getLockCategory1", "YouTube") .then(res => { assert.strictEqual(res.status, 200); const expected = { @@ -88,14 +89,14 @@ describe("getLockCategories", () => { ], reason: "1-longer-reason" }; - assert.deepStrictEqual(res.data, expected); + assert.ok(mixedDeepEquals(res.data, expected)); done(); }) .catch(err => done(err)); }); it("Should be able to get single locks with service", (done) => { - getLockCategoriesWithService("getLock3", "PeerTube") + getLockCategoriesWithService("getLockCategory3", "PeerTube") .then(res => { assert.strictEqual(res.status, 200); const expected = { @@ -111,7 +112,7 @@ describe("getLockCategories", () => { }); it("Should be able to get single locks with service", (done) => { - getLockCategoriesWithService("getLock3", "Youtube") + getLockCategoriesWithService("getLockCategory3", "Youtube") .then(res => { assert.strictEqual(res.status, 200); const expected = { @@ -127,7 +128,7 @@ describe("getLockCategories", () => { }); it("should return result from Youtube service if service not match", (done) => { - getLockCategoriesWithService("getLock3", "Dailymotion") + getLockCategoriesWithService("getLockCategory3", "Dailymotion") .then(res => { assert.strictEqual(res.status, 200); const expected = { diff --git a/test/cases/postSkipSegments.ts b/test/cases/postSkipSegments.ts index abc7ce7..27c6d3c 100644 --- a/test/cases/postSkipSegments.ts +++ b/test/cases/postSkipSegments.ts @@ -1,6 +1,6 @@ import { config } from "../../src/config"; import { getHash } from "../../src/utils/getHash"; -import { partialDeepEquals } from "../utils/partialDeepEquals"; +import { partialDeepEquals, arrayDeepEquals } from "../utils/partialDeepEquals"; import { db } from "../../src/databases/databases"; import { ImportMock } from "ts-mock-imports"; import * as YouTubeAPIModule from "../../src/utils/youtubeApi"; @@ -29,7 +29,7 @@ describe("postSkipSegments", () => { const submitUserOneHash = getHash(submitUserOne); const submitVIPuser = `VIPPostSkipUser${".".repeat(16)}`; - const warnVideoID = "dQw4w9WgXcF"; + const warnVideoID = "postSkip2"; const badInputVideoID = "dQw4w9WgXcQ"; const queryDatabase = (videoID: string) => db.prepare("get", `SELECT "startTime", "endTime", "locked", "category" FROM "sponsorTimes" WHERE "videoID" = ?`, [videoID]); @@ -91,7 +91,7 @@ describe("postSkipSegments", () => { }); it("Should be able to submit a single time (Params method)", (done) => { - const videoID = "dQw4w9WgXcR"; + const videoID = "postSkip1"; postSkipSegmentParam({ videoID, startTime: 2, @@ -125,7 +125,7 @@ describe("postSkipSegments", () => { }); it("Should be able to submit a single time (JSON method)", (done) => { - const videoID = "dQw4w9WgXcF"; + const videoID = "postSkip2"; postSkipSegmentJSON({ userID: submitUserOne, videoID, @@ -150,7 +150,7 @@ describe("postSkipSegments", () => { }); it("Should be able to submit a single time with an action type (JSON method)", (done) => { - const videoID = "dQw4w9WgXcV"; + const videoID = "postSkip3"; postSkipSegmentJSON({ userID: submitUserOne, videoID, @@ -176,7 +176,7 @@ describe("postSkipSegments", () => { }); it("Should not be able to submit an intro with mute action type (JSON method)", (done) => { - const videoID = "dQw4w9WgXpQ"; + const videoID = "postSkip4"; postSkipSegmentJSON({ userID: submitUserOne, videoID, @@ -196,7 +196,7 @@ describe("postSkipSegments", () => { }); it("Should be able to submit a single time with a duration from the YouTube API (JSON method)", (done) => { - const videoID = "dQw4w9WgXZX"; + const videoID = "postSkip5"; postSkipSegmentJSON({ userID: submitUserOne, videoID, @@ -222,7 +222,7 @@ describe("postSkipSegments", () => { }); it("Should be able to submit a single time with a precise duration close to the one from the YouTube API (JSON method)", (done) => { - const videoID = "dQw4w9WgXZH"; + const videoID = "postSkip6"; postSkipSegmentJSON({ userID: submitUserOne, videoID, @@ -331,7 +331,7 @@ describe("postSkipSegments", () => { }); it("Should be able to submit a single time under a different service (JSON method)", (done) => { - const videoID = "dQw4w9WgXcG"; + const videoID = "postSkip7"; postSkipSegmentJSON({ userID: submitUserOne, videoID, @@ -383,7 +383,7 @@ describe("postSkipSegments", () => { }); it("Should be able to submit multiple times (JSON method)", (done) => { - const videoID = "dQw4w9WgXcT"; + const videoID = "postSkip11"; postSkipSegmentJSON({ userID: submitUserOne, videoID, @@ -407,14 +407,14 @@ describe("postSkipSegments", () => { endTime: 60, category: "intro" }]; - assert.deepStrictEqual(rows, expected); + assert.ok(arrayDeepEquals(rows, expected)); done(); }) .catch(err => done(err)); }).timeout(5000); it("Should allow multiple times if total is under 80% of video(JSON method)", (done) => { - const videoID = "L_jWHffIx5E"; + const videoID = "postSkip9"; postSkipSegmentJSON({ userID: submitUserOne, videoID, @@ -452,7 +452,7 @@ describe("postSkipSegments", () => { endTime: 170, category: "sponsor" }]; - assert.deepStrictEqual(rows, expected); + assert.ok(arrayDeepEquals(rows, expected)); done(); }) .catch(err => done(err)); @@ -505,20 +505,20 @@ describe("postSkipSegments", () => { .then(async res => { assert.strictEqual(res.status, 403); const expected = [{ - category: "sponsor", - startTime: 2000, - endTime: 4000 + category: "interaction", + startTime: 0, + endTime: 1000 }, { - category: "sponsor", - startTime: 1500, - endTime: 2750 + category: "interaction", + startTime: 1001, + endTime: 1005 }, { - category: "sponsor", - startTime: 4050, - endTime: 4750 + category: "interaction", + startTime: 0, + endTime: 5000 }]; - const rows = await queryDatabase(videoID); - assert.notDeepStrictEqual(rows, expected); + const rows = await db.prepare("all", `SELECT "category", "startTime", "endTime" FROM "sponsorTimes" WHERE "videoID" = ?`, [videoID]); + assert.ok(arrayDeepEquals(rows, expected)); done(); }) .catch(err => done(err)); diff --git a/test/utils/partialDeepEquals.ts b/test/utils/partialDeepEquals.ts index 6571794..200252a 100644 --- a/test/utils/partialDeepEquals.ts +++ b/test/utils/partialDeepEquals.ts @@ -21,4 +21,35 @@ export const partialDeepEquals = (actual: Record, expected: Record< } } return true; -}; \ No newline at end of file +}; + +export const arrayDeepEquals = (actual: Record, expected: Record, print = true): boolean => { + if (actual.length !== expected.length) return false; + let flag = true; + const actualString = JSON.stringify(actual); + const expectedString = JSON.stringify(expected); + // check every value in arr1 for match in arr2 + actual.every((value: any) => { if (flag && !expectedString.includes(JSON.stringify(value))) flag = false; }); + // check arr2 for match in arr1 + expected.every((value: any) => { if (flag && !actualString.includes(JSON.stringify(value))) flag = false; }); + + if (!flag && print) printActualExpected(actual, expected); + return flag; +}; + +export const mixedDeepEquals = (actual: Record, expected: Record, print = true): boolean => { + for (const [ key, value ] of Object.entries(expected)) { + // if value is object or array, recurse + if (Array.isArray(value)) { + if (!arrayDeepEquals(actual?.[key], value, false)) { + if (print) printActualExpected(actual, expected); + return false; + } + } + else if (actual?.[key] !== value) { + if (print) printActualExpected(actual, expected); + return false; + } + } + return true; +}; From a3ea73287031c0dd7fdca5d7a6a55b88e91a697e Mon Sep 17 00:00:00 2001 From: Michael C Date: Fri, 22 Oct 2021 03:54:17 -0400 Subject: [PATCH 09/22] delete config.json.backup - thankfully no private information --- config.json.backup | 65 ---------------------------------------------- 1 file changed, 65 deletions(-) delete mode 100644 config.json.backup diff --git a/config.json.backup b/config.json.backup deleted file mode 100644 index 1b0ced2..0000000 --- a/config.json.backup +++ /dev/null @@ -1,65 +0,0 @@ -{ - "port": 8080, - "mockPort": 8081, - "globalSalt": "testSalt", - "adminUserID": "4bdfdc9cddf2c7d07a8a87b57bf6d25389fb75d1399674ee0e0938a6a60f4c3b", - "newLeafURLs": ["placeholder"], - "discordReportChannelWebhookURL": "http://127.0.0.1:8081/ReportChannelWebhook", - "discordFirstTimeSubmissionsWebhookURL": "http://127.0.0.1:8081/FirstTimeSubmissionsWebhook", - "discordCompletelyIncorrectReportWebhookURL": "http://127.0.0.1:8081/CompletelyIncorrectReportWebhook", - "discordNeuralBlockRejectWebhookURL": "http://127.0.0.1:8081/NeuralBlockRejectWebhook", - "neuralBlockURL": "http://127.0.0.1:8081/NeuralBlock", - "behindProxy": true, - "postgres": { - "user": "ci_db_user", - "password": "ci_db_pass", - "host": "bf.mchang.icu", - "port": 5432 - }, - "createDatabaseIfNotExist": true, - "schemaFolder": "./databases", - "dbSchema": "./databases/_sponsorTimes.db.sql", - "privateDBSchema": "./databases/_private.db.sql", - "mode": "test", - "readOnly": false, - "webhooks": [ - { - "url": "http://127.0.0.1:8081/CustomWebhook", - "key": "superSecretKey", - "scopes": [ - "vote.up", - "vote.down" - ] - }, { - "url": "http://127.0.0.1:8081/FailedWebhook", - "key": "superSecretKey", - "scopes": [ - "vote.up", - "vote.down" - ] - }, { - "url": "http://127.0.0.1:8099/WrongPort", - "key": "superSecretKey", - "scopes": [ - "vote.up", - "vote.down" - ] - } - ], - "categoryList": ["sponsor", "selfpromo", "interaction", "intro", "outro", "preview", "music_offtopic", "poi_highlight"], - "maxNumberOfActiveWarnings": 3, - "hoursAfterWarningExpires": 24, - "rateLimit": { - "vote": { - "windowMs": 900000, - "max": 20, - "message": "Too many votes, please try again later", - "statusCode": 429 - }, - "view": { - "windowMs": 900000, - "max": 20, - "statusCode": 200 - } - } -} From 6bcc4cdfa35d009960e0243e7f4943d2b46c7583 Mon Sep 17 00:00:00 2001 From: Michael C Date: Fri, 22 Oct 2021 11:37:32 -0400 Subject: [PATCH 10/22] sturcture for init and exit --- src/index.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/index.ts b/src/index.ts index 8c57217..d945178 100644 --- a/src/index.ts +++ b/src/index.ts @@ -12,6 +12,9 @@ async function init() { }); await initDb(); + // edge case clause for creating compatible .db files, do not enable + if (config.mode === "init-db-and-exit") process.exit(0); + // do not enable init-db-only mode for usage. (global as any).HEADCOMMIT = config.mode === "development" ? "development" : config.mode === "test" ? "test" : getCommit() as string; From b715b30ae64563354966d2ff5936b47b83008f09 Mon Sep 17 00:00:00 2001 From: Felix Hoang Date: Sun, 24 Oct 2021 16:04:18 +0700 Subject: [PATCH 11/22] add username in get lock reason route --- src/routes/getLockReason.ts | 50 ++++++++++++++++++++--------- test/cases/getLockReason.ts | 63 +++++++++++++++++++++++-------------- 2 files changed, 75 insertions(+), 38 deletions(-) diff --git a/src/routes/getLockReason.ts b/src/routes/getLockReason.ts index cf57828..049c1b7 100644 --- a/src/routes/getLockReason.ts +++ b/src/routes/getLockReason.ts @@ -8,7 +8,9 @@ const possibleCategoryList = config.categoryList; interface lockArray { category: Category; locked: number, - reason: string + reason: string, + userID: string, + userName: string, } export async function getLockReason(req: Request, res: Response): Promise { @@ -38,31 +40,51 @@ export async function getLockReason(req: Request, res: Response): Promise !lockedCategories.includes(x)); - for (const noLock of noLockCategories) { - locks.push({ - category: noLock, - locked: 0, - reason: "" - } as lockArray); + // all userName from userIDs + const userNames = await db.prepare( + "all", + `SELECT "userName", "userID" FROM "userNames" WHERE "userID" IN (${Array.from("?".repeat(userIDs.size)).join()}) LIMIT ?`, + [...userIDs, userIDs.size] + ) as { userName: string, userID: string }[]; + + const results = []; + for (const category of searchCategories) { + const lock = locks.find(l => l.category === category); + if (lock?.userID) { + // mapping userName to locks + const user = userNames.find(u => u.userID === lock.userID); + lock.userName = user?.userName || ""; + results.push(lock); + } else { + // add empty locks for categories requested but not locked + results.push({ + category, + locked: 0, + reason: "", + userID: "", + userName: "", + } as lockArray); + } } - // return real and fake locks that were requested - const filtered = locks.filter(lock => searchCategories.includes(lock.category)); - return res.send(filtered); + + return res.send(results); } catch (err) { Logger.error(err as string); return res.sendStatus(500); diff --git a/test/cases/getLockReason.ts b/test/cases/getLockReason.ts index 2941077..bf517d5 100644 --- a/test/cases/getLockReason.ts +++ b/test/cases/getLockReason.ts @@ -5,19 +5,33 @@ import { client } from "../utils/httpClient"; const endpoint = "/api/lockReason"; +const vipUserName1 = "getLockReason-vipUserName_1"; +const vipUserID1 = getHash("getLockReason-vipUserID_1"); +const vipUserName2 = "getLockReason-vipUserName_2"; +const vipUserID2 = getHash("getLockReason-vipUserID_2"); + describe("getLockReason", () => { before(async () => { - const vipUserID = "getLockReasonVIP"; - const vipUserHash = getHash(vipUserID); const insertVipUserQuery = 'INSERT INTO "vipUsers" ("userID") VALUES (?)'; - await db.prepare("run", insertVipUserQuery, [vipUserHash]); - await db.prepare("run", insertVipUserQuery, [vipUserHash]); + await db.prepare("run", insertVipUserQuery, [vipUserID1]); + await db.prepare("run", insertVipUserQuery, [vipUserID2]); + + const insertVipUserNameQuery = 'INSERT INTO "userNames" ("userID", "userName") VALUES (?, ?)'; + await db.prepare("run", insertVipUserNameQuery, [vipUserID1, vipUserName1]); + await db.prepare("run", insertVipUserNameQuery, [vipUserID2, vipUserName2]); const insertLockCategoryQuery = 'INSERT INTO "lockCategories" ("userID", "videoID", "category", "reason") VALUES (?, ?, ?, ?)'; - await db.prepare("run", insertLockCategoryQuery, [vipUserHash, "getLockReason", "sponsor", "sponsor-reason"]); - await db.prepare("run", insertLockCategoryQuery, [vipUserHash, "getLockReason", "interaction", "interaction-reason"]); - await db.prepare("run", insertLockCategoryQuery, [vipUserHash, "getLockReason", "preview", "preview-reason"]); - await db.prepare("run", insertLockCategoryQuery, [vipUserHash, "getLockReason", "music_offtopic", "nonmusic-reason"]); + await db.prepare("run", insertLockCategoryQuery, [vipUserID1, "getLockReason", "sponsor", "sponsor-reason"]); + await db.prepare("run", insertLockCategoryQuery, [vipUserID1, "getLockReason", "interaction", "interaction-reason"]); + await db.prepare("run", insertLockCategoryQuery, [vipUserID1, "getLockReason", "preview", "preview-reason"]); + await db.prepare("run", insertLockCategoryQuery, [vipUserID1, "getLockReason", "music_offtopic", "nonmusic-reason"]); + await db.prepare("run", insertLockCategoryQuery, [vipUserID2, "getLockReason", "outro", "outro-reason"]); + }); + + after(async () => { + const deleteUserNameQuery = 'DELETE FROM "userNames" WHERE "userID" = ? AND "userName" = ? LIMIT 1'; + await db.prepare("run", deleteUserNameQuery, [vipUserID1, vipUserName1]); + await db.prepare("run", deleteUserNameQuery, [vipUserID2, vipUserName2]); }); it("Should update the database version when starting the application", async () => { @@ -31,7 +45,7 @@ describe("getLockReason", () => { .then(res => { assert.strictEqual(res.status, 200); const expected = [ - { category: "sponsor", locked: 1, reason: "sponsor-reason" } + { category: "sponsor", locked: 1, reason: "sponsor-reason", userID: vipUserID1, userName: vipUserName1 } ]; assert.deepStrictEqual(res.data, expected); done(); @@ -44,7 +58,7 @@ describe("getLockReason", () => { .then(res => { assert.strictEqual(res.status, 200); const expected = [ - { category: "intro", locked: 0, reason: "" } + { category: "intro", locked: 0, reason: "", userID: "", userName: "" } ]; assert.deepStrictEqual(res.data, expected); done(); @@ -53,12 +67,13 @@ describe("getLockReason", () => { }); it("should get multiple locks with array", (done) => { - client.get(endpoint, { params: { videoID: "getLockReason", categories: `["intro","sponsor"]` } }) + client.get(endpoint, { params: { videoID: "getLockReason", categories: `["intro","sponsor","outro"]` } }) .then(res => { assert.strictEqual(res.status, 200); const expected = [ - { category: "sponsor", locked: 1, reason: "sponsor-reason" }, - { category: "intro", locked: 0, reason: "" } + { category: "intro", locked: 0, reason: "", userID: "", userName: "" }, + { category: "sponsor", locked: 1, reason: "sponsor-reason", userID: vipUserID1, userName: vipUserName1 }, + { category: "outro", locked: 1, reason: "outro-reason", userID: vipUserID2, userName: vipUserName2 } ]; assert.deepStrictEqual(res.data, expected); done(); @@ -71,9 +86,9 @@ describe("getLockReason", () => { .then(res => { assert.strictEqual(res.status, 200); const expected = [ - { category: "interaction", locked: 1, reason: "interaction-reason" }, - { category: "music_offtopic", locked: 1, reason: "nonmusic-reason" }, - { category: "intro", locked: 0, reason: "" } + { category: "interaction", locked: 1, reason: "interaction-reason", userID: vipUserID1, userName: vipUserName1 }, + { category: "music_offtopic", locked: 1, reason: "nonmusic-reason", userID: vipUserID1, userName: vipUserName1 }, + { category: "intro", locked: 0, reason: "", userID: "", userName: "" } ]; assert.deepStrictEqual(res.data, expected); done(); @@ -86,14 +101,14 @@ describe("getLockReason", () => { .then(res => { assert.strictEqual(res.status, 200); const expected = [ - { category: "sponsor", locked: 1, reason: "sponsor-reason" }, - { category: "interaction", locked: 1, reason: "interaction-reason" }, - { category: "preview", locked: 1, reason: "preview-reason" }, - { category: "music_offtopic", locked: 1, reason: "nonmusic-reason" }, - { category: "selfpromo", locked: 0, reason: "" }, - { category: "intro", locked: 0, reason: "" }, - { category: "outro", locked: 0, reason: "" }, - { category: "poi_highlight", locked: 0, reason: "" } + { category: "sponsor", locked: 1, reason: "sponsor-reason", userID: vipUserID1, userName: vipUserName1 }, + { category: "selfpromo", locked: 0, reason: "", userID: "", userName: "" }, + { category: "interaction", locked: 1, reason: "interaction-reason", userID: vipUserID1, userName: vipUserName1 }, + { category: "intro", locked: 0, reason: "", userID: "", userName: "" }, + { category: "outro", locked: 1, reason: "outro-reason", userID: vipUserID2, userName: vipUserName2 }, + { category: "preview", locked: 1, reason: "preview-reason", userID: vipUserID1, userName: vipUserName1 }, + { category: "music_offtopic", locked: 1, reason: "nonmusic-reason", userID: vipUserID1, userName: vipUserName1 }, + { category: "poi_highlight", locked: 0, reason: "", userID: "", userName: "" } ]; assert.deepStrictEqual(res.data, expected); done(); From 103280ca59c70e94dfe98d587fe84c8933032fd3 Mon Sep 17 00:00:00 2001 From: Felix Hoang Date: Sun, 24 Oct 2021 16:27:12 +0700 Subject: [PATCH 12/22] remove unuse variable --- src/routes/getLockReason.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/routes/getLockReason.ts b/src/routes/getLockReason.ts index 049c1b7..30aa839 100644 --- a/src/routes/getLockReason.ts +++ b/src/routes/getLockReason.ts @@ -43,7 +43,6 @@ export async function getLockReason(req: Request, res: Response): Promise Date: Sun, 24 Oct 2021 16:19:40 -0400 Subject: [PATCH 13/22] don't queue dump if generate=false --- src/routes/dumpDatabase.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/dumpDatabase.ts b/src/routes/dumpDatabase.ts index e876449..ba85f00 100644 --- a/src/routes/dumpDatabase.ts +++ b/src/routes/dumpDatabase.ts @@ -184,7 +184,7 @@ export async function redirectLink(req: Request, res: Response): Promise { res.sendStatus(404); } - await queueDump(); + if (req.query.generate !== "false") await queueDump(); } function updateQueueTime(): void { From a9adfbc06dafee579411c2358894ba5f6a27f81e Mon Sep 17 00:00:00 2001 From: Michael C Date: Tue, 26 Oct 2021 20:37:49 -0400 Subject: [PATCH 14/22] disable eslint for error dump --- src/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/index.ts b/src/index.ts index d945178..e58aacd 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,6 +7,7 @@ import { getCommit } from "./utils/getCommit"; async function init() { process.on("unhandledRejection", (error: any) => { + // eslint-disable-next-line no-console console.dir(error?.stack); process.exit(1); }); From 59b1d02075b17f6798012cec443fe98149cc53f7 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Tue, 26 Oct 2021 20:54:25 -0400 Subject: [PATCH 15/22] Add new vps --- nginx/nginx.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 4f85aed..9fdb9d3 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -36,6 +36,9 @@ http { server 10.0.0.3:4441; server 10.0.0.3:4442; + server 10.0.0.5:4441; + server 10.0.0.5:4442; + #server 134.209.69.251:80 backup; #server 116.203.32.253:80 backup; From 65fa663a1a8c569a7af63b6baa6d43536be4ea6d Mon Sep 17 00:00:00 2001 From: Michael C Date: Tue, 26 Oct 2021 21:55:32 -0400 Subject: [PATCH 16/22] add rsyncd config --- docker/docker-compose.yml | 9 +++++++++ docker/rsync/rsyncd.conf | 15 +++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 docker/rsync/rsyncd.conf diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 0523aef..2618eff 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -28,6 +28,15 @@ services: - 3241:3000 volumes: - ./newleaf/configuration.py:/workdir/configuration.py + rsync: + image: mchangrh/rsync:latest + container_name: rsync + restart: always + ports: + - 873:873 + volumes: + - ./rsync/rsync.conf:/etc/rsync.conf + - ./database-export/:/mirror volumes: database-data: diff --git a/docker/rsync/rsyncd.conf b/docker/rsync/rsyncd.conf new file mode 100644 index 0000000..4d09d5d --- /dev/null +++ b/docker/rsync/rsyncd.conf @@ -0,0 +1,15 @@ +pid file = /var/run/rsyncd.pid +lock file = /var/run/rsync.lock +log file = /var/log/rsync.log +# replace with user accessing the files + +[sponsorblock] +use chroot = no +max connections = 10 +# path to mirrored files +path = /mirror +comment = sponsorblock-database +read only = true +refuse options = c delete zl +# disallow checksumming and compression level to reduce CPU/IO load +# disallow deleting files clientside \ No newline at end of file From 6db89778a9b6b0caf1b7e020acec394edec44c21 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Tue, 26 Oct 2021 22:00:32 -0400 Subject: [PATCH 17/22] enable testing server --- nginx/nginx.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 9fdb9d3..7e14eb7 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -95,8 +95,8 @@ http { } location /test/ { - return 404 ""; - #proxy_pass http://localhost:4440/; + # return 404 ""; + proxy_pass http://10.0.0.3:4445; #proxy_pass https://sbtest.etcinit.com/; } From 452d8a47f50451105e285c570af8d479e3ac7a58 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Tue, 26 Oct 2021 22:02:50 -0400 Subject: [PATCH 18/22] Switch testing server ip --- nginx/nginx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 7e14eb7..3433b60 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -96,7 +96,7 @@ http { location /test/ { # return 404 ""; - proxy_pass http://10.0.0.3:4445; + proxy_pass http://10.0.0.5:4445; #proxy_pass https://sbtest.etcinit.com/; } From 58de2a786d5b44c7db4bcd4138c48443d3121958 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Tue, 26 Oct 2021 22:05:51 -0400 Subject: [PATCH 19/22] Fix testing server getting full path --- nginx/nginx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 3433b60..d6f71d9 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -96,7 +96,7 @@ http { location /test/ { # return 404 ""; - proxy_pass http://10.0.0.5:4445; + proxy_pass http://10.0.0.5:4445/; #proxy_pass https://sbtest.etcinit.com/; } From dc7831c31f418aa1fc4520988fa0b101c397430b Mon Sep 17 00:00:00 2001 From: Michael C Date: Tue, 26 Oct 2021 22:38:46 -0400 Subject: [PATCH 20/22] fix rsync file name --- docker/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 2618eff..639f2c4 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -35,7 +35,7 @@ services: ports: - 873:873 volumes: - - ./rsync/rsync.conf:/etc/rsync.conf + - ./rsync/rsyncd.conf:/etc/rsyncd.conf - ./database-export/:/mirror volumes: From 6cf268330f0be692ddc92677b453867234fb18e9 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Wed, 27 Oct 2021 19:36:33 -0400 Subject: [PATCH 21/22] Remove old auto-ban on low trust --- src/routes/postSkipSegments.ts | 10 +--------- src/utils/isUserTrustworthy.ts | 20 -------------------- 2 files changed, 1 insertion(+), 29 deletions(-) delete mode 100644 src/utils/isUserTrustworthy.ts diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index aa97798..ae68983 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -6,7 +6,6 @@ import { getSubmissionUUID } from "../utils/getSubmissionUUID"; import { getHash } from "../utils/getHash"; import { getIP } from "../utils/getIP"; import { getFormattedTime } from "../utils/getFormattedTime"; -import { isUserTrustworthy } from "../utils/isUserTrustworthy"; import { dispatchEvent } from "../utils/webhookUtils"; import { Request, Response } from "express"; import { ActionType, Category, CategoryActionType, IncomingSegment, SegmentUUID, Service, VideoDuration, VideoID } from "../types/segments.model"; @@ -622,13 +621,6 @@ export async function postSkipSegments(req: Request, res: Response): Promise { - //check to see if this user how many submissions this user has submitted - const totalSubmissionsRow = await db.prepare("get", `SELECT count(*) as "totalSubmissions", sum(votes) as "voteSum" FROM "sponsorTimes" WHERE "userID" = ?`, [userID]); - - if (totalSubmissionsRow.totalSubmissions > 5) { - //check if they have a high downvote ratio - const downvotedSubmissionsRow = await db.prepare("get", `SELECT count(*) as "downvotedSubmissions" FROM "sponsorTimes" WHERE "userID" = ? AND (votes < 0 OR "shadowHidden" > 0)`, [userID]); - - return (downvotedSubmissionsRow.downvotedSubmissions / totalSubmissionsRow.totalSubmissions) < 0.6 || - (totalSubmissionsRow.voteSum > downvotedSubmissionsRow.downvotedSubmissions); - } - - return true; -} From 19f7bbcde5d3800439aca4934f53176a0544a5b3 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Wed, 27 Oct 2021 21:25:46 -0400 Subject: [PATCH 22/22] Add info about using sb mirror Closes #373 --- src/routes/dumpDatabase.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/routes/dumpDatabase.ts b/src/routes/dumpDatabase.ts index ba85f00..f03446f 100644 --- a/src/routes/dumpDatabase.ts +++ b/src/routes/dumpDatabase.ts @@ -118,6 +118,12 @@ export default async function dumpDatabase(req: Request, res: Response, showPage Then, you can download the csv files below, or use the links returned from the JSON request. A dump will also be triggered by making a request to one of these urls. +

Keeping your dump up to date

+ + If you want a live dump, please do not continually fetch this url. + Please instead use the sb-mirror project. + This can automatically fetch new data and will not require a redownload each time, saving bandwith. +

Links