mirror of
https://github.com/ajayyy/SponsorBlockServer.git
synced 2024-11-10 01:02:30 +01:00
Merge pull request #569 from mchangrh/fix-shadowban
clean up shadowban code, exclude long running categories query
This commit is contained in:
Ajay Ramachandran
GitHub
commit
951d678640
2 changed files with 41 additions and 226 deletions
|
|
@ -1,8 +1,8 @@
|
|||
import { db, privateDB } from "../databases/databases";
|
||||
import { db } from "../databases/databases";
|
||||
import { getHashCache } from "../utils/getHashCache";
|
||||
import { Request, Response } from "express";
|
||||
import { config } from "../config";
|
||||
import { Category, DeArrowType, HashedIP, Service, VideoID, VideoIDHash } from "../types/segments.model";
|
||||
import { Category, DeArrowType, Service, VideoID, VideoIDHash } from "../types/segments.model";
|
||||
import { UserID } from "../types/user.model";
|
||||
import { QueryCacher } from "../utils/queryCacher";
|
||||
import { isUserVIP } from "../utils/isUserVIP";
|
||||
|
|
@ -11,7 +11,6 @@ import { Logger } from "../utils/logger";
|
|||
|
||||
export async function shadowBanUser(req: Request, res: Response): Promise<Response> {
|
||||
const userID = req.query.userID as UserID;
|
||||
const hashedIP = req.query.hashedIP as HashedIP;
|
||||
const adminUserIDInput = req.query.adminUserID as UserID;
|
||||
const type = Number.parseInt(req.query.type as string ?? "1");
|
||||
if (isNaN(type)) {
|
||||
|
|
@ -21,10 +20,6 @@ export async function shadowBanUser(req: Request, res: Response): Promise<Respon
|
|||
const enabled = req.query.enabled === undefined
|
||||
? true
|
||||
: req.query.enabled === "true";
|
||||
const lookForIPs = req.query.lookForIPs === "true";
|
||||
const banUsers = req.query.banUsers === undefined
|
||||
? true
|
||||
: req.query.banUsers === "true";
|
||||
|
||||
//if enabled is false and the old submissions should be made visible again
|
||||
const unHideOldSubmissions = req.query.unHideOldSubmissions !== "false";
|
||||
|
|
@ -32,7 +27,7 @@ export async function shadowBanUser(req: Request, res: Response): Promise<Respon
|
|||
const categories: Category[] = parseCategories(req, config.categoryList as Category[]);
|
||||
const deArrowTypes: DeArrowType[] = parseDeArrowTypes(req, config.deArrowTypes);
|
||||
|
||||
if (adminUserIDInput == undefined || (userID == undefined && hashedIP == undefined || type <= 0)) {
|
||||
if (adminUserIDInput == undefined || (userID == undefined || type <= 0)) {
|
||||
//invalid request
|
||||
return res.sendStatus(400);
|
||||
}
|
||||
|
|
@ -46,33 +41,11 @@ export async function shadowBanUser(req: Request, res: Response): Promise<Respon
|
|||
//not authorized
|
||||
return res.sendStatus(403);
|
||||
}
|
||||
|
||||
if (userID) {
|
||||
const result = await banUser(userID, enabled, unHideOldSubmissions, type, categories, deArrowTypes);
|
||||
|
||||
if (enabled && lookForIPs) {
|
||||
const ipLoggingFixedTime = 1675295716000;
|
||||
const timeSubmitted = (await db.prepare("all", `SELECT "timeSubmitted" FROM "sponsorTimes" WHERE "timeSubmitted" > ? AND "userID" = ?`, [ipLoggingFixedTime, userID])) as { timeSubmitted: number }[];
|
||||
const ips = (await Promise.all(timeSubmitted.map((s) => {
|
||||
return privateDB.prepare("all", `SELECT "hashedIP" FROM "sponsorTimes" WHERE "timeSubmitted" = ?`, [s.timeSubmitted]) as Promise<{ hashedIP: HashedIP }[]>;
|
||||
}))).flat();
|
||||
|
||||
await Promise.all([...new Set(ips.map((ip) => ip.hashedIP))].map((ip) => {
|
||||
return banIP(ip, enabled, unHideOldSubmissions, type, categories, deArrowTypes, true);
|
||||
}));
|
||||
}
|
||||
|
||||
if (result) {
|
||||
res.sendStatus(result);
|
||||
return;
|
||||
}
|
||||
} else if (hashedIP) {
|
||||
const result = await banIP(hashedIP, enabled, unHideOldSubmissions, type, categories, deArrowTypes, banUsers);
|
||||
if (result) {
|
||||
res.sendStatus(result);
|
||||
return;
|
||||
}
|
||||
}
|
||||
return res.sendStatus(200);
|
||||
} catch (e) {
|
||||
Logger.error(e as string);
|
||||
|
|
@ -115,58 +88,20 @@ export async function banUser(userID: UserID, enabled: boolean, unHideOldSubmiss
|
|||
// already not shadowbanned
|
||||
return 400;
|
||||
}
|
||||
|
||||
return 200;
|
||||
}
|
||||
|
||||
export async function banIP(hashedIP: HashedIP, enabled: boolean, unHideOldSubmissions: boolean, type: number,
|
||||
categories: Category[], deArrowTypes: DeArrowType[], banUsers: boolean): Promise<number> {
|
||||
|
||||
//check to see if this user is already shadowbanned
|
||||
const row = await db.prepare("get", `SELECT count(*) as "userCount" FROM "shadowBannedIPs" WHERE "hashedIP" = ?`, [hashedIP]);
|
||||
|
||||
if (enabled) {
|
||||
if (row.userCount == 0) {
|
||||
await db.prepare("run", `INSERT INTO "shadowBannedIPs" VALUES(?)`, [hashedIP]);
|
||||
}
|
||||
|
||||
//find all previous submissions and hide them
|
||||
if (unHideOldSubmissions) {
|
||||
const users = await unHideSubmissionsByIP(categories, hashedIP, type);
|
||||
|
||||
if (banUsers) {
|
||||
await Promise.all([...users].map((user) => {
|
||||
return banUser(user, enabled, unHideOldSubmissions, type, categories, deArrowTypes);
|
||||
}));
|
||||
}
|
||||
} else if (row.userCount > 0) {
|
||||
// Nothing to do, and already added
|
||||
return 409;
|
||||
}
|
||||
} else if (!enabled) {
|
||||
if (row.userCount > 0) {
|
||||
//remove them from the shadow ban list
|
||||
await db.prepare("run", `DELETE FROM "shadowBannedIPs" WHERE "hashedIP" = ?`, [hashedIP]);
|
||||
}
|
||||
|
||||
//find all previous submissions and unhide them
|
||||
if (unHideOldSubmissions) {
|
||||
await unHideSubmissionsByIP(categories, hashedIP, 0);
|
||||
}
|
||||
}
|
||||
|
||||
return 200;
|
||||
}
|
||||
|
||||
async function unHideSubmissionsByUser(categories: string[], deArrowTypes: DeArrowType[],
|
||||
userID: UserID, type = 1) {
|
||||
|
||||
if (categories.length) {
|
||||
await db.prepare("run", `UPDATE "sponsorTimes" SET "shadowHidden" = '${type}' WHERE "userID" = ? AND "category" in (${categories.map((c) => `'${c}'`).join(",")})
|
||||
AND NOT EXISTS ( SELECT "videoID", "category" FROM "lockCategories" WHERE
|
||||
"sponsorTimes"."videoID" = "lockCategories"."videoID" AND "sponsorTimes"."service" = "lockCategories"."service" AND "sponsorTimes"."category" = "lockCategories"."category")`, [userID]);
|
||||
}
|
||||
|
||||
// clear cache for all old videos
|
||||
(await db.prepare("all", `SELECT "videoID", "hashedVideoID", "service", "votes", "views" FROM "sponsorTimes" WHERE "userID" = ?`, [userID]))
|
||||
(await db.prepare("all", `SELECT "category", "videoID", "hashedVideoID", "service", "userID" FROM "sponsorTimes" WHERE "userID" = ?`, [userID]))
|
||||
.forEach((videoInfo: { category: Category; videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; userID: UserID; }) => {
|
||||
QueryCacher.clearSegmentCache(videoInfo);
|
||||
});
|
||||
|
|
@ -181,7 +116,6 @@ async function unHideSubmissionsByUser(categories: string[], deArrowTypes: DeArr
|
|||
[userID]);
|
||||
}
|
||||
|
||||
|
||||
(await db.prepare("all", `SELECT "videoID", "hashedVideoID", "service" FROM "titles" WHERE "userID" = ?`, [userID]))
|
||||
.forEach((videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; }) => {
|
||||
QueryCacher.clearBrandingCache(videoInfo);
|
||||
|
|
@ -191,23 +125,3 @@ async function unHideSubmissionsByUser(categories: string[], deArrowTypes: DeArr
|
|||
QueryCacher.clearBrandingCache(videoInfo);
|
||||
});
|
||||
}
|
||||
|
||||
async function unHideSubmissionsByIP(categories: string[], hashedIP: HashedIP, type = 1): Promise<Set<UserID>> {
|
||||
const submissions = await privateDB.prepare("all", `SELECT "timeSubmitted" FROM "sponsorTimes" WHERE "hashedIP" = ?`, [hashedIP]) as { timeSubmitted: number }[];
|
||||
|
||||
const users: Set<UserID> = new Set();
|
||||
await Promise.all(submissions.map(async (submission) => {
|
||||
(await db.prepare("all", `SELECT "videoID", "hashedVideoID", "service", "votes", "views", "userID" FROM "sponsorTimes" WHERE "timeSubmitted" = ? AND "category" in (${categories.map((c) => `'${c}'`).join(",")})`, [submission.timeSubmitted]))
|
||||
.forEach((videoInfo: { category: Category, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, userID: UserID }) => {
|
||||
QueryCacher.clearSegmentCache(videoInfo);
|
||||
users.add(videoInfo.userID);
|
||||
}
|
||||
);
|
||||
|
||||
await db.prepare("run", `UPDATE "sponsorTimes" SET "shadowHidden" = ${type} WHERE "timeSubmitted" = ? AND "category" in (${categories.map((c) => `'${c}'`).join(",")})
|
||||
AND NOT EXISTS ( SELECT "videoID", "category" FROM "lockCategories" WHERE
|
||||
"sponsorTimes"."videoID" = "lockCategories"."videoID" AND "sponsorTimes"."service" = "lockCategories"."service" AND "sponsorTimes"."category" = "lockCategories"."category")`, [submission.timeSubmitted]);
|
||||
}));
|
||||
|
||||
return users;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,8 +11,6 @@ describe("shadowBanUser", () => {
|
|||
const getShadowBanTitles = (userID: string, status: number) => db.prepare("all", `SELECT tv."shadowHidden" FROM "titles" t JOIN "titleVotes" tv ON t."UUID" = tv."UUID" WHERE t."userID" = ? AND tv."shadowHidden" = ?`, [userID, status]);
|
||||
const getShadowBanThumbnails = (userID: string, status: number) => db.prepare("all", `SELECT tv."shadowHidden" FROM "thumbnails" t JOIN "thumbnailVotes" tv ON t."UUID" = tv."UUID" WHERE t."userID" = ? AND tv."shadowHidden" = ?`, [userID, status]);
|
||||
|
||||
const getIPShadowBan = (hashedIP: string) => db.prepare("get", `SELECT * FROM "shadowBannedIPs" WHERE "hashedIP" = ?`, [hashedIP]);
|
||||
|
||||
const endpoint = "/api/shadowBanUser";
|
||||
const VIPuserID = "shadow-ban-vip";
|
||||
const video = "shadowBanVideo";
|
||||
|
|
@ -58,18 +56,6 @@ describe("shadowBanUser", () => {
|
|||
|
||||
await db.prepare("run", `INSERT INTO "vipUsers" ("userID") VALUES(?)`, [getHash(VIPuserID)]);
|
||||
|
||||
const privateInsertQuery = `INSERT INTO "sponsorTimes" ("videoID", "hashedIP", "timeSubmitted", "service") VALUES(?, ?, ?, ?)`;
|
||||
await privateDB.prepare("run", privateInsertQuery, [video, "shadowBannedIP7", 383848, "YouTube"]);
|
||||
await privateDB.prepare("run", privateInsertQuery, [video, "shadowBannedIP7", 2332, "YouTube"]);
|
||||
await privateDB.prepare("run", privateInsertQuery, [video, "shadowBannedIP7", 4923, "YouTube"]);
|
||||
|
||||
await privateDB.prepare("run", privateInsertQuery, [video, "shadowBannedIP8", 1674590916068933, "YouTube"]);
|
||||
await privateDB.prepare("run", privateInsertQuery, [video, "shadowBannedIP8", 1674590916062936, "YouTube"]);
|
||||
await privateDB.prepare("run", privateInsertQuery, [video, "shadowBannedIP8", 1674590916064324, "YouTube"]);
|
||||
await privateDB.prepare("run", privateInsertQuery, [video, "shadowBannedIP8", 1674590916062443, "YouTube"]);
|
||||
await privateDB.prepare("run", privateInsertQuery, [video, "shadowBannedIP8", 1674590916062342, "YouTube"]);
|
||||
await privateDB.prepare("run", privateInsertQuery, [video, "shadowBannedIP8", 1674590916069491, "YouTube"]);
|
||||
|
||||
const titleQuery = `INSERT INTO "titles" ("videoID", "title", "original", "userID", "service", "hashedVideoID", "timeSubmitted", "UUID") VALUES (?, ?, ?, ?, ?, ?, ?, ?)`;
|
||||
const titleVotesQuery = `INSERT INTO "titleVotes" ("UUID", "votes", "locked", "shadowHidden", "verification") VALUES (?, ?, ?, ?, ?)`;
|
||||
const thumbnailQuery = `INSERT INTO "thumbnails" ("videoID", "original", "userID", "service", "hashedVideoID", "timeSubmitted", "UUID") VALUES (?, ?, ?, ?, ?, ?, ?)`;
|
||||
|
|
@ -93,7 +79,7 @@ describe("shadowBanUser", () => {
|
|||
db.prepare("run", thumbnailTimestampsQuery, ["UUID3T-ban", 3]),
|
||||
db.prepare("run", thumbnailVotesQuery, ["UUID1T-ban", 3, 0, 0]),
|
||||
db.prepare("run", thumbnailVotesQuery, ["UUID2T-ban", 2, 0, 0]),
|
||||
db.prepare("run", thumbnailVotesQuery, ["UUID3T-ban", 1, 0, 0])
|
||||
db.prepare("run", thumbnailVotesQuery, ["UUID3T-ban", 1, 0, 0]),
|
||||
]);
|
||||
|
||||
await Promise.all([
|
||||
|
|
@ -115,6 +101,11 @@ describe("shadowBanUser", () => {
|
|||
db.prepare("run", thumbnailVotesQuery, ["UUID2T-ban2", 2, 0, 0]),
|
||||
db.prepare("run", thumbnailVotesQuery, ["UUID3T-ban2", 1, 0, 0])
|
||||
]);
|
||||
|
||||
await Promise.all([
|
||||
db.prepare("run", titleQuery, [video, "title31", 0, "userID3-ban", Service.YouTube, videohash, 1, "UUID1-ban3"]),
|
||||
db.prepare("run", thumbnailQuery, [video, 0, "userID3-ban", Service.YouTube, videohash, 1, "UUID1T-ban3"]),
|
||||
]);
|
||||
});
|
||||
|
||||
it("Should be able to ban user and hide submissions", (done) => {
|
||||
|
|
@ -380,84 +371,6 @@ describe("shadowBanUser", () => {
|
|||
.catch(err => done(err));
|
||||
});
|
||||
|
||||
it("Should be able to ban user by IP and hide submissions of a specific category", (done) => {
|
||||
const hashedIP = "shadowBannedIP7";
|
||||
const userID = "shadowBanned7";
|
||||
client({
|
||||
method: "POST",
|
||||
url: endpoint,
|
||||
params: {
|
||||
hashedIP,
|
||||
categories: `["sponsor", "intro"]`,
|
||||
adminUserID: VIPuserID,
|
||||
}
|
||||
})
|
||||
.then(async res => {
|
||||
assert.strictEqual(res.status, 200);
|
||||
const videoRow = await getShadowBanSegments(userID, 1);
|
||||
const normalShadowRow = await getShadowBan(userID);
|
||||
const ipShadowRow = await getIPShadowBan(hashedIP);
|
||||
assert.ok(ipShadowRow);
|
||||
assert.ok(normalShadowRow);
|
||||
assert.strictEqual(videoRow.length, 2);
|
||||
done();
|
||||
})
|
||||
.catch(err => done(err));
|
||||
});
|
||||
|
||||
it("Should be able to unban user by IP", (done) => {
|
||||
const hashedIP = "shadowBannedIP7";
|
||||
const userID = "shadowBanned7";
|
||||
client({
|
||||
method: "POST",
|
||||
url: endpoint,
|
||||
params: {
|
||||
hashedIP,
|
||||
enabled: false,
|
||||
unHideOldSubmissions: false,
|
||||
adminUserID: VIPuserID,
|
||||
}
|
||||
})
|
||||
.then(async res => {
|
||||
assert.strictEqual(res.status, 200);
|
||||
const videoRow = await getShadowBanSegments(userID, 1);
|
||||
const normalShadowRow = await getShadowBan(userID);
|
||||
const ipShadowRow = await getIPShadowBan(hashedIP);
|
||||
assert.ok(!ipShadowRow);
|
||||
assert.ok(normalShadowRow);
|
||||
assert.strictEqual(videoRow.length, 2);
|
||||
done();
|
||||
})
|
||||
.catch(err => done(err));
|
||||
});
|
||||
|
||||
it("Should be able to unban user by IP and unhide specific category", (done) => {
|
||||
const hashedIP = "shadowBannedIP7";
|
||||
const userID = "shadowBanned7";
|
||||
client({
|
||||
method: "POST",
|
||||
url: endpoint,
|
||||
params: {
|
||||
hashedIP,
|
||||
enabled: false,
|
||||
categories: `["sponsor"]`,
|
||||
unHideOldSubmissions: true,
|
||||
adminUserID: VIPuserID,
|
||||
}
|
||||
})
|
||||
.then(async res => {
|
||||
assert.strictEqual(res.status, 200);
|
||||
const videoRow = await getShadowBanSegments(userID, 1);
|
||||
const normalShadowRow = await getShadowBan(userID);
|
||||
const ipShadowRow = await getIPShadowBan(hashedIP);
|
||||
assert.ok(!ipShadowRow);
|
||||
assert.ok(normalShadowRow);
|
||||
assert.strictEqual(videoRow.length, 1);
|
||||
done();
|
||||
})
|
||||
.catch(err => done(err));
|
||||
});
|
||||
|
||||
it("Should be possible to ban self", (done) => {
|
||||
const userID = VIPuserID;
|
||||
const hashUserID = getHash(userID);
|
||||
|
|
@ -479,39 +392,6 @@ describe("shadowBanUser", () => {
|
|||
.catch(err => done(err));
|
||||
});
|
||||
|
||||
it("Should be able to ban user by userID and other users who used that IP and hide specific category", (done) => {
|
||||
const hashedIP = "shadowBannedIP8";
|
||||
const userID = "shadowBanned8";
|
||||
const userID2 = "shadowBanned9";
|
||||
client({
|
||||
method: "POST",
|
||||
url: endpoint,
|
||||
params: {
|
||||
userID,
|
||||
enabled: true,
|
||||
categories: `["sponsor", "intro"]`,
|
||||
unHideOldSubmissions: true,
|
||||
adminUserID: VIPuserID,
|
||||
lookForIPs: true
|
||||
}
|
||||
})
|
||||
.then(async res => {
|
||||
assert.strictEqual(res.status, 200);
|
||||
const videoRow = await getShadowBanSegments(userID, 1);
|
||||
const videoRow2 = await getShadowBanSegments(userID2, 1);
|
||||
const normalShadowRow = await getShadowBan(userID);
|
||||
const normalShadowRow2 = await getShadowBan(userID2);
|
||||
const ipShadowRow = await getIPShadowBan(hashedIP);
|
||||
assert.ok(ipShadowRow);
|
||||
assert.ok(normalShadowRow);
|
||||
assert.ok(normalShadowRow2);
|
||||
assert.strictEqual(videoRow.length, 2);
|
||||
assert.strictEqual(videoRow2.length, 2);
|
||||
done();
|
||||
})
|
||||
.catch(err => done(err));
|
||||
});
|
||||
|
||||
it("Should be able to ban user and hide dearrow submissions", (done) => {
|
||||
const userID = "userID1-ban";
|
||||
client({
|
||||
|
|
@ -558,4 +438,25 @@ describe("shadowBanUser", () => {
|
|||
})
|
||||
.catch(err => done(err));
|
||||
});
|
||||
|
||||
it("Should be able to ban user with bad/ empty categories", (done) => {
|
||||
const userID = "userID4-ban";
|
||||
client({
|
||||
method: "POST",
|
||||
url: endpoint,
|
||||
params: {
|
||||
userID,
|
||||
adminUserID: VIPuserID,
|
||||
enabled: true,
|
||||
unHideOldSubmissions: true,
|
||||
categories: `[]`,
|
||||
deArrowTypes: `["title","thumbnail"]`
|
||||
}
|
||||
})
|
||||
.then(res => {
|
||||
assert.strictEqual(res.status, 200);
|
||||
done();
|
||||
})
|
||||
.catch(err => done(err));
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in a new issue