From da482054a435176a8800d42b47b671c892e6532d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 11 Nov 2023 08:17:36 +0000 Subject: [PATCH 1/3] Bump axios from 1.1.3 to 1.6.0 Bumps [axios](https://github.com/axios/axios) from 1.1.3 to 1.6.0. - [Release notes](https://github.com/axios/axios/releases) - [Changelog](https://github.com/axios/axios/blob/v1.x/CHANGELOG.md) - [Commits](https://github.com/axios/axios/compare/v1.1.3...v1.6.0) --- updated-dependencies: - dependency-name: axios dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index 06ddadc..c12e9ae 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "0.1.0", "license": "AGPL-3.0-only", "dependencies": { - "axios": "^1.1.3", + "axios": "^1.6.0", "better-sqlite3": "^8.0.1", "cron": "^2.1.0", "express": "^4.18.2", @@ -1465,9 +1465,9 @@ "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, "node_modules/axios": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.1.3.tgz", - "integrity": "sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.0.tgz", + "integrity": "sha512-EZ1DYihju9pwVB+jg67ogm+Tmqc6JmhamRN6I4Zt8DfZu5lbcQGw3ozH9lFejSJgs/ibaef3A9PMXPLeefFGJg==", "dependencies": { "follow-redirects": "^1.15.0", "form-data": "^4.0.0", @@ -6818,9 +6818,9 @@ "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, "axios": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.1.3.tgz", - "integrity": "sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.0.tgz", + "integrity": "sha512-EZ1DYihju9pwVB+jg67ogm+Tmqc6JmhamRN6I4Zt8DfZu5lbcQGw3ozH9lFejSJgs/ibaef3A9PMXPLeefFGJg==", "requires": { "follow-redirects": "^1.15.0", "form-data": "^4.0.0", diff --git a/package.json b/package.json index ac5864d..89db73e 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "author": "Ajay Ramachandran", "license": "AGPL-3.0-only", "dependencies": { - "axios": "^1.1.3", + "axios": "^1.6.0", "better-sqlite3": "^8.0.1", "cron": "^2.1.0", "express": "^4.18.2", From b3a28f7df32e19d400b688dbe68022681cf33cbc Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Wed, 6 Dec 2023 00:16:55 -0500 Subject: [PATCH 2/3] Revert "Bump axios from 1.1.3 to 1.6.0" --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index c12e9ae..06ddadc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "0.1.0", "license": "AGPL-3.0-only", "dependencies": { - "axios": "^1.6.0", + "axios": "^1.1.3", "better-sqlite3": "^8.0.1", "cron": "^2.1.0", "express": "^4.18.2", @@ -1465,9 +1465,9 @@ "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, "node_modules/axios": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.0.tgz", - "integrity": "sha512-EZ1DYihju9pwVB+jg67ogm+Tmqc6JmhamRN6I4Zt8DfZu5lbcQGw3ozH9lFejSJgs/ibaef3A9PMXPLeefFGJg==", + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.1.3.tgz", + "integrity": "sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA==", "dependencies": { "follow-redirects": "^1.15.0", "form-data": "^4.0.0", @@ -6818,9 +6818,9 @@ "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, "axios": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.0.tgz", - "integrity": "sha512-EZ1DYihju9pwVB+jg67ogm+Tmqc6JmhamRN6I4Zt8DfZu5lbcQGw3ozH9lFejSJgs/ibaef3A9PMXPLeefFGJg==", + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.1.3.tgz", + "integrity": "sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA==", "requires": { "follow-redirects": "^1.15.0", "form-data": "^4.0.0", diff --git a/package.json b/package.json index 89db73e..ac5864d 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "author": "Ajay Ramachandran", "license": "AGPL-3.0-only", "dependencies": { - "axios": "^1.6.0", + "axios": "^1.1.3", "better-sqlite3": "^8.0.1", "cron": "^2.1.0", "express": "^4.18.2", From 15f19df8a4446335ab01bf80f8219991adeb0809 Mon Sep 17 00:00:00 2001 From: Michael C Date: Thu, 21 Dec 2023 18:37:24 -0500 Subject: [PATCH 3/3] clean up shadowban code, exclude long running categories query when possible --- src/routes/shadowBanUser.ts | 114 ++++----------------------- test/cases/shadowBanUser.ts | 153 +++++++----------------------------- 2 files changed, 41 insertions(+), 226 deletions(-) diff --git a/src/routes/shadowBanUser.ts b/src/routes/shadowBanUser.ts index b0b5e8a..93164a4 100644 --- a/src/routes/shadowBanUser.ts +++ b/src/routes/shadowBanUser.ts @@ -1,8 +1,8 @@ -import { db, privateDB } from "../databases/databases"; +import { db } from "../databases/databases"; import { getHashCache } from "../utils/getHashCache"; import { Request, Response } from "express"; import { config } from "../config"; -import { Category, DeArrowType, HashedIP, Service, VideoID, VideoIDHash } from "../types/segments.model"; +import { Category, DeArrowType, Service, VideoID, VideoIDHash } from "../types/segments.model"; import { UserID } from "../types/user.model"; import { QueryCacher } from "../utils/queryCacher"; import { isUserVIP } from "../utils/isUserVIP"; @@ -11,7 +11,6 @@ import { Logger } from "../utils/logger"; export async function shadowBanUser(req: Request, res: Response): Promise { const userID = req.query.userID as UserID; - const hashedIP = req.query.hashedIP as HashedIP; const adminUserIDInput = req.query.adminUserID as UserID; const type = Number.parseInt(req.query.type as string ?? "1"); if (isNaN(type)) { @@ -21,10 +20,6 @@ export async function shadowBanUser(req: Request, res: Response): Promise ? AND "userID" = ?`, [ipLoggingFixedTime, userID])) as { timeSubmitted: number }[]; - const ips = (await Promise.all(timeSubmitted.map((s) => { - return privateDB.prepare("all", `SELECT "hashedIP" FROM "sponsorTimes" WHERE "timeSubmitted" = ?`, [s.timeSubmitted]) as Promise<{ hashedIP: HashedIP }[]>; - }))).flat(); - - await Promise.all([...new Set(ips.map((ip) => ip.hashedIP))].map((ip) => { - return banIP(ip, enabled, unHideOldSubmissions, type, categories, deArrowTypes, true); - })); - } - - if (result) { - res.sendStatus(result); - return; - } - } else if (hashedIP) { - const result = await banIP(hashedIP, enabled, unHideOldSubmissions, type, categories, deArrowTypes, banUsers); - if (result) { - res.sendStatus(result); - return; - } + const result = await banUser(userID, enabled, unHideOldSubmissions, type, categories, deArrowTypes); + if (result) { + res.sendStatus(result); + return; } return res.sendStatus(200); } catch (e) { @@ -115,58 +88,20 @@ export async function banUser(userID: UserID, enabled: boolean, unHideOldSubmiss // already not shadowbanned return 400; } - - return 200; -} - -export async function banIP(hashedIP: HashedIP, enabled: boolean, unHideOldSubmissions: boolean, type: number, - categories: Category[], deArrowTypes: DeArrowType[], banUsers: boolean): Promise { - - //check to see if this user is already shadowbanned - const row = await db.prepare("get", `SELECT count(*) as "userCount" FROM "shadowBannedIPs" WHERE "hashedIP" = ?`, [hashedIP]); - - if (enabled) { - if (row.userCount == 0) { - await db.prepare("run", `INSERT INTO "shadowBannedIPs" VALUES(?)`, [hashedIP]); - } - - //find all previous submissions and hide them - if (unHideOldSubmissions) { - const users = await unHideSubmissionsByIP(categories, hashedIP, type); - - if (banUsers) { - await Promise.all([...users].map((user) => { - return banUser(user, enabled, unHideOldSubmissions, type, categories, deArrowTypes); - })); - } - } else if (row.userCount > 0) { - // Nothing to do, and already added - return 409; - } - } else if (!enabled) { - if (row.userCount > 0) { - //remove them from the shadow ban list - await db.prepare("run", `DELETE FROM "shadowBannedIPs" WHERE "hashedIP" = ?`, [hashedIP]); - } - - //find all previous submissions and unhide them - if (unHideOldSubmissions) { - await unHideSubmissionsByIP(categories, hashedIP, 0); - } - } - return 200; } async function unHideSubmissionsByUser(categories: string[], deArrowTypes: DeArrowType[], userID: UserID, type = 1) { - await db.prepare("run", `UPDATE "sponsorTimes" SET "shadowHidden" = '${type}' WHERE "userID" = ? AND "category" in (${categories.map((c) => `'${c}'`).join(",")}) - AND NOT EXISTS ( SELECT "videoID", "category" FROM "lockCategories" WHERE - "sponsorTimes"."videoID" = "lockCategories"."videoID" AND "sponsorTimes"."service" = "lockCategories"."service" AND "sponsorTimes"."category" = "lockCategories"."category")`, [userID]); + if (categories.length) { + await db.prepare("run", `UPDATE "sponsorTimes" SET "shadowHidden" = '${type}' WHERE "userID" = ? AND "category" in (${categories.map((c) => `'${c}'`).join(",")}) + AND NOT EXISTS ( SELECT "videoID", "category" FROM "lockCategories" WHERE + "sponsorTimes"."videoID" = "lockCategories"."videoID" AND "sponsorTimes"."service" = "lockCategories"."service" AND "sponsorTimes"."category" = "lockCategories"."category")`, [userID]); + } // clear cache for all old videos - (await db.prepare("all", `SELECT "videoID", "hashedVideoID", "service", "votes", "views" FROM "sponsorTimes" WHERE "userID" = ?`, [userID])) + (await db.prepare("all", `SELECT "category", "videoID", "hashedVideoID", "service", "userID" FROM "sponsorTimes" WHERE "userID" = ?`, [userID])) .forEach((videoInfo: { category: Category; videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; userID: UserID; }) => { QueryCacher.clearSegmentCache(videoInfo); }); @@ -181,7 +116,6 @@ async function unHideSubmissionsByUser(categories: string[], deArrowTypes: DeArr [userID]); } - (await db.prepare("all", `SELECT "videoID", "hashedVideoID", "service" FROM "titles" WHERE "userID" = ?`, [userID])) .forEach((videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; }) => { QueryCacher.clearBrandingCache(videoInfo); @@ -190,24 +124,4 @@ async function unHideSubmissionsByUser(categories: string[], deArrowTypes: DeArr .forEach((videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; }) => { QueryCacher.clearBrandingCache(videoInfo); }); -} - -async function unHideSubmissionsByIP(categories: string[], hashedIP: HashedIP, type = 1): Promise> { - const submissions = await privateDB.prepare("all", `SELECT "timeSubmitted" FROM "sponsorTimes" WHERE "hashedIP" = ?`, [hashedIP]) as { timeSubmitted: number }[]; - - const users: Set = new Set(); - await Promise.all(submissions.map(async (submission) => { - (await db.prepare("all", `SELECT "videoID", "hashedVideoID", "service", "votes", "views", "userID" FROM "sponsorTimes" WHERE "timeSubmitted" = ? AND "category" in (${categories.map((c) => `'${c}'`).join(",")})`, [submission.timeSubmitted])) - .forEach((videoInfo: { category: Category, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, userID: UserID }) => { - QueryCacher.clearSegmentCache(videoInfo); - users.add(videoInfo.userID); - } - ); - - await db.prepare("run", `UPDATE "sponsorTimes" SET "shadowHidden" = ${type} WHERE "timeSubmitted" = ? AND "category" in (${categories.map((c) => `'${c}'`).join(",")}) - AND NOT EXISTS ( SELECT "videoID", "category" FROM "lockCategories" WHERE - "sponsorTimes"."videoID" = "lockCategories"."videoID" AND "sponsorTimes"."service" = "lockCategories"."service" AND "sponsorTimes"."category" = "lockCategories"."category")`, [submission.timeSubmitted]); - })); - - return users; -} +} \ No newline at end of file diff --git a/test/cases/shadowBanUser.ts b/test/cases/shadowBanUser.ts index 6efc13d..b50c094 100644 --- a/test/cases/shadowBanUser.ts +++ b/test/cases/shadowBanUser.ts @@ -11,8 +11,6 @@ describe("shadowBanUser", () => { const getShadowBanTitles = (userID: string, status: number) => db.prepare("all", `SELECT tv."shadowHidden" FROM "titles" t JOIN "titleVotes" tv ON t."UUID" = tv."UUID" WHERE t."userID" = ? AND tv."shadowHidden" = ?`, [userID, status]); const getShadowBanThumbnails = (userID: string, status: number) => db.prepare("all", `SELECT tv."shadowHidden" FROM "thumbnails" t JOIN "thumbnailVotes" tv ON t."UUID" = tv."UUID" WHERE t."userID" = ? AND tv."shadowHidden" = ?`, [userID, status]); - const getIPShadowBan = (hashedIP: string) => db.prepare("get", `SELECT * FROM "shadowBannedIPs" WHERE "hashedIP" = ?`, [hashedIP]); - const endpoint = "/api/shadowBanUser"; const VIPuserID = "shadow-ban-vip"; const video = "shadowBanVideo"; @@ -58,18 +56,6 @@ describe("shadowBanUser", () => { await db.prepare("run", `INSERT INTO "vipUsers" ("userID") VALUES(?)`, [getHash(VIPuserID)]); - const privateInsertQuery = `INSERT INTO "sponsorTimes" ("videoID", "hashedIP", "timeSubmitted", "service") VALUES(?, ?, ?, ?)`; - await privateDB.prepare("run", privateInsertQuery, [video, "shadowBannedIP7", 383848, "YouTube"]); - await privateDB.prepare("run", privateInsertQuery, [video, "shadowBannedIP7", 2332, "YouTube"]); - await privateDB.prepare("run", privateInsertQuery, [video, "shadowBannedIP7", 4923, "YouTube"]); - - await privateDB.prepare("run", privateInsertQuery, [video, "shadowBannedIP8", 1674590916068933, "YouTube"]); - await privateDB.prepare("run", privateInsertQuery, [video, "shadowBannedIP8", 1674590916062936, "YouTube"]); - await privateDB.prepare("run", privateInsertQuery, [video, "shadowBannedIP8", 1674590916064324, "YouTube"]); - await privateDB.prepare("run", privateInsertQuery, [video, "shadowBannedIP8", 1674590916062443, "YouTube"]); - await privateDB.prepare("run", privateInsertQuery, [video, "shadowBannedIP8", 1674590916062342, "YouTube"]); - await privateDB.prepare("run", privateInsertQuery, [video, "shadowBannedIP8", 1674590916069491, "YouTube"]); - const titleQuery = `INSERT INTO "titles" ("videoID", "title", "original", "userID", "service", "hashedVideoID", "timeSubmitted", "UUID") VALUES (?, ?, ?, ?, ?, ?, ?, ?)`; const titleVotesQuery = `INSERT INTO "titleVotes" ("UUID", "votes", "locked", "shadowHidden", "verification") VALUES (?, ?, ?, ?, ?)`; const thumbnailQuery = `INSERT INTO "thumbnails" ("videoID", "original", "userID", "service", "hashedVideoID", "timeSubmitted", "UUID") VALUES (?, ?, ?, ?, ?, ?, ?)`; @@ -93,7 +79,7 @@ describe("shadowBanUser", () => { db.prepare("run", thumbnailTimestampsQuery, ["UUID3T-ban", 3]), db.prepare("run", thumbnailVotesQuery, ["UUID1T-ban", 3, 0, 0]), db.prepare("run", thumbnailVotesQuery, ["UUID2T-ban", 2, 0, 0]), - db.prepare("run", thumbnailVotesQuery, ["UUID3T-ban", 1, 0, 0]) + db.prepare("run", thumbnailVotesQuery, ["UUID3T-ban", 1, 0, 0]), ]); await Promise.all([ @@ -115,6 +101,11 @@ describe("shadowBanUser", () => { db.prepare("run", thumbnailVotesQuery, ["UUID2T-ban2", 2, 0, 0]), db.prepare("run", thumbnailVotesQuery, ["UUID3T-ban2", 1, 0, 0]) ]); + + await Promise.all([ + db.prepare("run", titleQuery, [video, "title31", 0, "userID3-ban", Service.YouTube, videohash, 1, "UUID1-ban3"]), + db.prepare("run", thumbnailQuery, [video, 0, "userID3-ban", Service.YouTube, videohash, 1, "UUID1T-ban3"]), + ]); }); it("Should be able to ban user and hide submissions", (done) => { @@ -380,84 +371,6 @@ describe("shadowBanUser", () => { .catch(err => done(err)); }); - it("Should be able to ban user by IP and hide submissions of a specific category", (done) => { - const hashedIP = "shadowBannedIP7"; - const userID = "shadowBanned7"; - client({ - method: "POST", - url: endpoint, - params: { - hashedIP, - categories: `["sponsor", "intro"]`, - adminUserID: VIPuserID, - } - }) - .then(async res => { - assert.strictEqual(res.status, 200); - const videoRow = await getShadowBanSegments(userID, 1); - const normalShadowRow = await getShadowBan(userID); - const ipShadowRow = await getIPShadowBan(hashedIP); - assert.ok(ipShadowRow); - assert.ok(normalShadowRow); - assert.strictEqual(videoRow.length, 2); - done(); - }) - .catch(err => done(err)); - }); - - it("Should be able to unban user by IP", (done) => { - const hashedIP = "shadowBannedIP7"; - const userID = "shadowBanned7"; - client({ - method: "POST", - url: endpoint, - params: { - hashedIP, - enabled: false, - unHideOldSubmissions: false, - adminUserID: VIPuserID, - } - }) - .then(async res => { - assert.strictEqual(res.status, 200); - const videoRow = await getShadowBanSegments(userID, 1); - const normalShadowRow = await getShadowBan(userID); - const ipShadowRow = await getIPShadowBan(hashedIP); - assert.ok(!ipShadowRow); - assert.ok(normalShadowRow); - assert.strictEqual(videoRow.length, 2); - done(); - }) - .catch(err => done(err)); - }); - - it("Should be able to unban user by IP and unhide specific category", (done) => { - const hashedIP = "shadowBannedIP7"; - const userID = "shadowBanned7"; - client({ - method: "POST", - url: endpoint, - params: { - hashedIP, - enabled: false, - categories: `["sponsor"]`, - unHideOldSubmissions: true, - adminUserID: VIPuserID, - } - }) - .then(async res => { - assert.strictEqual(res.status, 200); - const videoRow = await getShadowBanSegments(userID, 1); - const normalShadowRow = await getShadowBan(userID); - const ipShadowRow = await getIPShadowBan(hashedIP); - assert.ok(!ipShadowRow); - assert.ok(normalShadowRow); - assert.strictEqual(videoRow.length, 1); - done(); - }) - .catch(err => done(err)); - }); - it("Should be possible to ban self", (done) => { const userID = VIPuserID; const hashUserID = getHash(userID); @@ -479,39 +392,6 @@ describe("shadowBanUser", () => { .catch(err => done(err)); }); - it("Should be able to ban user by userID and other users who used that IP and hide specific category", (done) => { - const hashedIP = "shadowBannedIP8"; - const userID = "shadowBanned8"; - const userID2 = "shadowBanned9"; - client({ - method: "POST", - url: endpoint, - params: { - userID, - enabled: true, - categories: `["sponsor", "intro"]`, - unHideOldSubmissions: true, - adminUserID: VIPuserID, - lookForIPs: true - } - }) - .then(async res => { - assert.strictEqual(res.status, 200); - const videoRow = await getShadowBanSegments(userID, 1); - const videoRow2 = await getShadowBanSegments(userID2, 1); - const normalShadowRow = await getShadowBan(userID); - const normalShadowRow2 = await getShadowBan(userID2); - const ipShadowRow = await getIPShadowBan(hashedIP); - assert.ok(ipShadowRow); - assert.ok(normalShadowRow); - assert.ok(normalShadowRow2); - assert.strictEqual(videoRow.length, 2); - assert.strictEqual(videoRow2.length, 2); - done(); - }) - .catch(err => done(err)); - }); - it("Should be able to ban user and hide dearrow submissions", (done) => { const userID = "userID1-ban"; client({ @@ -558,4 +438,25 @@ describe("shadowBanUser", () => { }) .catch(err => done(err)); }); + + it("Should be able to ban user with bad/ empty categories", (done) => { + const userID = "userID4-ban"; + client({ + method: "POST", + url: endpoint, + params: { + userID, + adminUserID: VIPuserID, + enabled: true, + unHideOldSubmissions: true, + categories: `[]`, + deArrowTypes: `["title","thumbnail"]` + } + }) + .then(res => { + assert.strictEqual(res.status, 200); + done(); + }) + .catch(err => done(err)); + }); });