external/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2024-11-10 01:02:30 +01:00
Code Issues Projects Releases Packages Wiki Activity

Add locking by action type

Browse source
This commit is contained in:
Ajay 2022-01-02 14:00:54 -05:00
parent aaa3179d42
commit 09eec5a4a5
6 changed files with 183 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
DatabaseSchema.md
View file

@ -94,6 +94,7 @@
| -- | :--: | -- | | -- | :--: | -- |
| videoID | TEXT | not null | | videoID | TEXT | not null |
| userID | TEXT | not null | | userID | TEXT | not null |
| actionType | TEXT | not null, default 'skip' |
| category | TEXT | not null | | category | TEXT | not null |
| hashedVideoID | TEXT | not null, default '' | | hashedVideoID | TEXT | not null, default '' |
| reason | TEXT | not null, default '' | | reason | TEXT | not null, default '' |

21
databases/_upgrade_sponsorTimes_29.sql Normal file
View file

@ -0,0 +1,21 @@
BEGIN TRANSACTION;
CREATE TABLE "sqlb_temp_table_29" (
"videoID" TEXT NOT NULL,
"userID" TEXT NOT NULL,
"actionType" TEXT NOT NULL DEFAULT 'skip',
"category" TEXT NOT NULL,
"hashedVideoID" TEXT NOT NULL default '',
"reason" TEXT NOT NULL default '',
"service" TEXT NOT NULL default 'YouTube'
);
INSERT INTO sqlb_temp_table_29 SELECT "videoID","userID",'skip',"category","hashedVideoID","reason","service" FROM "lockCategories";
INSERT INTO sqlb_temp_table_29 SELECT "videoID","userID",'mute',"category","hashedVideoID","reason","service" FROM "lockCategories";
DROP TABLE "lockCategories";
ALTER TABLE sqlb_temp_table_29 RENAME TO "lockCategories";
UPDATE "config" SET value = 29 WHERE key = 'version';
COMMIT;

81
src/routes/postLockCategories.ts
View file

@ -3,14 +3,15 @@ import { getHashCache } from "../utils/getHashCache";
import { isUserVIP } from "../utils/isUserVIP"; import { isUserVIP } from "../utils/isUserVIP";
import { db } from "../databases/databases"; import { db } from "../databases/databases";
import { Request, Response } from "express"; import { Request, Response } from "express";
import { VideoIDHash } from "../types/segments.model"; import { ActionType, Category, VideoIDHash } from "../types/segments.model";
import { getService } from "../utils/getService"; import { getService } from "../utils/getService";
export async function postLockCategories(req: Request, res: Response): Promise<string[]> { export async function postLockCategories(req: Request, res: Response): Promise<string[]> {
// Collect user input data // Collect user input data
const videoID = req.body.videoID; const videoID = req.body.videoID;
let userID = req.body.userID; let userID = req.body.userID;
const categories = req.body.categories; const categories = req.body.categories as Category[];
const actionTypes = req.body.actionTypes as ActionType[] || [ActionType.Skip, ActionType.Mute];
const reason: string = req.body.reason ?? ""; const reason: string = req.body.reason ?? "";
const service = getService(req.body.service); const service = getService(req.body.service);
@ -20,6 +21,8 @@ export async function postLockCategories(req: Request, res: Response): Promise<s
|| !categories || !categories
|| !Array.isArray(categories) || !Array.isArray(categories)
|| categories.length === 0 || categories.length === 0
|| !Array.isArray(actionTypes)
|| actionTypes.length === 0
) { ) {
res.status(400).json({ res.status(400).json({
message: "Bad Format", message: "Bad Format",
@ -38,38 +41,39 @@ export async function postLockCategories(req: Request, res: Response): Promise<s
return; return;
} }
// Get existing lock categories markers const existingLocks = (await db.prepare("all", 'SELECT "category", "actionType" from "lockCategories" where "videoID" = ? AND "service" = ?', [videoID, service])) as
let noCategoryList = await db.prepare("all", 'SELECT "category" from "lockCategories" where "videoID" = ? AND "service" = ?', [videoID, service]); { category: Category, actionType: ActionType }[];
if (!noCategoryList || noCategoryList.length === 0) {
noCategoryList = []; const filteredCategories = filterData(categories);
const filteredActionTypes = filterData(actionTypes);
const locksToApply: { category: Category, actionType: ActionType }[] = [];
const overwrittenLocks: { category: Category, actionType: ActionType }[] = [];
for (const category of filteredCategories) {
for (const actionType of filteredActionTypes) {
if (!existingLocks.some((lock) => lock.category === category && lock.actionType === actionType)) {
locksToApply.push({
category,
actionType
});
} else { } else {
noCategoryList = noCategoryList.map((obj: any) => { overwrittenLocks.push({
return obj.category; category,
actionType
}); });
} }
}
// get user categories not already submitted that match accepted format }
let filteredCategories = categories.filter((category) => {
return !!category.match(/^[_a-zA-Z]+$/);
});
// remove any duplicates
filteredCategories = filteredCategories.filter((category, index) => {
return filteredCategories.indexOf(category) === index;
});
const categoriesToMark = filteredCategories.filter((category) => {
return noCategoryList.indexOf(category) === -1;
});
// calculate hash of videoID // calculate hash of videoID
const hashedVideoID: VideoIDHash = await getHashCache(videoID, 1); const hashedVideoID: VideoIDHash = await getHashCache(videoID, 1);
// create database entry // create database entry
for (const category of categoriesToMark) { for (const lock of locksToApply) {
try { try {
await db.prepare("run", `INSERT INTO "lockCategories" ("videoID", "userID", "category", "hashedVideoID", "reason", "service") VALUES(?, ?, ?, ?, ?, ?)`, [videoID, userID, category, hashedVideoID, reason, service]); await db.prepare("run", `INSERT INTO "lockCategories" ("videoID", "userID", "actionType", "category", "hashedVideoID", "reason", "service") VALUES(?, ?, ?, ?, ?, ?, ?)`, [videoID, userID, lock.actionType, lock.category, hashedVideoID, reason, service]);
} catch (err) { } catch (err) {
Logger.error(`Error submitting 'lockCategories' marker for category '${category}' for video '${videoID}' (${service})`); Logger.error(`Error submitting 'lockCategories' marker for category '${lock.category}' and actionType '${lock.actionType}' for video '${videoID}' (${service})`);
Logger.error(err as string); Logger.error(err as string);
res.status(500).json({ res.status(500).json({
message: "Internal Server Error: Could not write marker to the database.", message: "Internal Server Error: Could not write marker to the database.",
@ -78,19 +82,14 @@ export async function postLockCategories(req: Request, res: Response): Promise<s
} }
// update reason for existed categories // update reason for existed categories
let overlapCategories = [];
if (reason.length !== 0) { if (reason.length !== 0) {
overlapCategories = filteredCategories.filter((category) => { for (const lock of overwrittenLocks) {
return noCategoryList.indexOf(category) !== -1;
});
for (const category of overlapCategories) {
try { try {
await db.prepare("run", await db.prepare("run",
'UPDATE "lockCategories" SET "reason" = ?, "userID" = ? WHERE "videoID" = ? AND "category" = ? AND "service" = ?', 'UPDATE "lockCategories" SET "reason" = ?, "userID" = ? WHERE "videoID" = ? AND "actionType" = ? AND "category" = ? AND "service" = ?',
[reason, userID, videoID, category, service]); [reason, userID, videoID, lock.actionType, lock.category, service]);
} catch (err) { } catch (err) {
Logger.error(`Error submitting 'lockCategories' marker for category '${category}' for video '${videoID} (${service})'`); Logger.error(`Error submitting 'lockCategories' marker for category '${lock.category}' and actionType '${lock.actionType}' for video '${videoID}' (${service})`);
Logger.error(err as string); Logger.error(err as string);
res.status(500).json({ res.status(500).json({
message: "Internal Server Error: Could not write marker to the database.", message: "Internal Server Error: Could not write marker to the database.",
@ -100,6 +99,20 @@ export async function postLockCategories(req: Request, res: Response): Promise<s
} }
res.status(200).json({ res.status(200).json({
submitted: [...categoriesToMark, ...overlapCategories], submitted: reason.length === 0
? [...filteredCategories.filter(((category) => locksToApply.some((lock) => category === lock.category)))]
: [...filteredCategories], // Legacy
submittedValues: [...locksToApply, ...overwrittenLocks],
});
}
function filterData<T extends string>(data: T[]): T[] {
// get user categories not already submitted that match accepted format
const filtered = data.filter((elem) => {
return !!elem.match(/^[_a-zA-Z]+$/);
});
// remove any duplicates
return filtered.filter((elem, index) => {
return filtered.indexOf(elem) === index;
}); });
} }

4
src/routes/postSkipSegments.ts
View file

@ -357,7 +357,7 @@ async function checkEachSegmentValid(userID: string, videoID: VideoID,
} }
// Reject segment if it's in the locked categories list // Reject segment if it's in the locked categories list
const lockIndex = lockedCategoryList.findIndex(c => segments[i].category === c.category); const lockIndex = lockedCategoryList.findIndex(c => segments[i].category === c.category && segments[i].actionType === c.actionType);
if (!isVIP && lockIndex !== -1) { if (!isVIP && lockIndex !== -1) {
// TODO: Do something about the fradulent submission // TODO: Do something about the fradulent submission
Logger.warn(`Caught a submission for a locked category. userID: '${userID}', videoID: '${videoID}', category: '${segments[i].category}', times: ${segments[i].segment}`); Logger.warn(`Caught a submission for a locked category. userID: '${userID}', videoID: '${videoID}', category: '${segments[i].category}', times: ${segments[i].segment}`);
@ -439,7 +439,7 @@ async function checkByAutoModerator(videoID: any, userID: any, segments: Array<a
} }
async function updateDataIfVideoDurationChange(videoID: VideoID, service: Service, videoDuration: VideoDuration, videoDurationParam: VideoDuration) { async function updateDataIfVideoDurationChange(videoID: VideoID, service: Service, videoDuration: VideoDuration, videoDurationParam: VideoDuration) {
let lockedCategoryList = await db.prepare("all", 'SELECT category, reason from "lockCategories" where "videoID" = ? AND "service" = ?', [videoID, service]); let lockedCategoryList = await db.prepare("all", 'SELECT category, "actionType", reason from "lockCategories" where "videoID" = ? AND "service" = ?', [videoID, service]);
const previousSubmissions = await db.prepare("all", const previousSubmissions = await db.prepare("all",
`SELECT "videoDuration", "UUID" `SELECT "videoDuration", "UUID"

107
test/cases/lockCategoriesRecords.ts
View file

@ -3,6 +3,7 @@ import { db } from "../../src/databases/databases";
import assert from "assert"; import assert from "assert";
import { LockCategory } from "../../src/types/segments.model"; import { LockCategory } from "../../src/types/segments.model";
import { client } from "../utils/httpClient"; import { client } from "../utils/httpClient";
import { partialDeepEquals } from "../utils/partialDeepEquals";
const stringDeepEquals = (a: string[], b: string[]): boolean => { const stringDeepEquals = (a: string[], b: string[]): boolean => {
let result = true; let result = true;
@ -23,18 +24,27 @@ describe("lockCategoriesRecords", () => {
const insertVipUserQuery = 'INSERT INTO "vipUsers" ("userID") VALUES (?)'; const insertVipUserQuery = 'INSERT INTO "vipUsers" ("userID") VALUES (?)';
await db.prepare("run", insertVipUserQuery, [lockVIPUserHash]); await db.prepare("run", insertVipUserQuery, [lockVIPUserHash]);
const insertLockCategoryQuery = 'INSERT INTO "lockCategories" ("userID", "videoID", "category", "reason", "service") VALUES (?, ?, ?, ?, ?)'; const insertLockCategoryQuery = 'INSERT INTO "lockCategories" ("userID", "videoID", "actionType", "category", "reason", "service") VALUES (?, ?, ?, ?, ?, ?)';
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id", "sponsor", "reason-1", "YouTube"]); await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id", "skip", "sponsor", "reason-1", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id", "intro", "reason-1", "YouTube"]); await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id", "mute", "sponsor", "reason-1", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id", "skip", "intro", "reason-1", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id", "mute", "intro", "reason-1", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id-1", "sponsor", "reason-2", "YouTube"]); await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id-1", "skip", "sponsor", "reason-2", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id-1", "intro", "reason-2", "YouTube"]); await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id-1", "mute", "sponsor", "reason-2", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "lockCategoryVideo", "sponsor", "reason-3", "YouTube"]); await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id-1", "skip", "intro", "reason-2", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id-1", "mute", "intro", "reason-2", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "lockCategoryVideo", "skip", "sponsor", "reason-3", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "lockCategoryVideo", "mute", "sponsor", "reason-3", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "delete-record", "sponsor", "reason-4", "YouTube"]); await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "lockCategoryVideo-2", "skip", "sponsor", "reason-4", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "delete-record-1", "sponsor", "reason-5", "YouTube"]); await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "delete-record", "skip", "sponsor", "reason-4", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "delete-record-1", "intro", "reason-5", "YouTube"]); await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "delete-record", "mute", "sponsor", "reason-4", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "delete-record-1", "skip", "sponsor", "reason-5", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "delete-record-1", "mute", "sponsor", "reason-5", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "delete-record-1", "skip", "intro", "reason-5", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "delete-record-1", "mute", "intro", "reason-5", "YouTube"]);
}); });
it("Should update the database version when starting the application", async () => { it("Should update the database version when starting the application", async () => {
@ -60,6 +70,32 @@ describe("lockCategoriesRecords", () => {
"outro", "outro",
"shilling", "shilling",
], ],
submittedValues: [
{
actionType: "skip",
category: "outro"
},
{
actionType: "mute",
category: "outro"
},
{
actionType: "skip",
category: "shilling"
},
{
actionType: "mute",
category: "shilling"
},
{
actionType: "skip",
category: "intro"
},
{
actionType: "mute",
category: "intro"
}
]
}; };
client.post(endpoint, json) client.post(endpoint, json)
.then(res => { .then(res => {
@ -88,15 +124,15 @@ describe("lockCategoriesRecords", () => {
.then(async res => { .then(async res => {
assert.strictEqual(res.status, 200); assert.strictEqual(res.status, 200);
const result = await checkLockCategories(videoID); const result = await checkLockCategories(videoID);
assert.strictEqual(result.length, 4); assert.strictEqual(result.length, 8);
const oldRecordNotChangeReason = result.filter(item => const oldRecordNotChangeReason = result.filter(item =>
item.reason === "reason-2" && ["sponsor", "intro"].includes(item.category) item.reason === "reason-2" && ["sponsor", "intro"].includes(item.category)
); );
const newRecordWithEmptyReason = result.filter(item => const newRecordWithEmptyReason = result.filter(item =>
item.reason === "" && ["outro", "shilling"].includes(item.category) item.reason === "" && ["outro", "shilling"].includes(item.category)
); );
assert.strictEqual(newRecordWithEmptyReason.length, 2); assert.strictEqual(newRecordWithEmptyReason.length, 4);
assert.strictEqual(oldRecordNotChangeReason.length, 2); assert.strictEqual(oldRecordNotChangeReason.length, 4);
done(); done();
}) })
.catch(err => done(err)); .catch(err => done(err));
@ -160,7 +196,7 @@ describe("lockCategoriesRecords", () => {
.then(async res => { .then(async res => {
assert.strictEqual(res.status, 200); assert.strictEqual(res.status, 200);
const result = await checkLockCategories(videoID); const result = await checkLockCategories(videoID);
assert.strictEqual(result.length, 4); assert.strictEqual(result.length, 8);
const newRecordWithNewReason = result.filter(item => const newRecordWithNewReason = result.filter(item =>
expectedWithNewReason.includes(item.category) && item.reason === "new reason" expectedWithNewReason.includes(item.category) && item.reason === "new reason"
); );
@ -168,8 +204,8 @@ describe("lockCategoriesRecords", () => {
item.reason === "reason-2" item.reason === "reason-2"
); );
assert.strictEqual(newRecordWithNewReason.length, 3); assert.strictEqual(newRecordWithNewReason.length, 6);
assert.strictEqual(oldRecordNotChangeReason.length, 1); assert.strictEqual(oldRecordNotChangeReason.length, 2);
done(); done();
}) })
.catch(err => done(err)); .catch(err => done(err));
@ -187,7 +223,7 @@ describe("lockCategoriesRecords", () => {
.then(async res => { .then(async res => {
assert.strictEqual(res.status, 200); assert.strictEqual(res.status, 200);
const result = await checkLockCategories("underscore"); const result = await checkLockCategories("underscore");
assert.strictEqual(result.length, 1); assert.strictEqual(result.length, 2);
done(); done();
}) })
.catch(err => done(err)); .catch(err => done(err));
@ -205,7 +241,7 @@ describe("lockCategoriesRecords", () => {
.then(async res => { .then(async res => {
assert.strictEqual(res.status, 200); assert.strictEqual(res.status, 200);
const result = await checkLockCategories("bothCases"); const result = await checkLockCategories("bothCases");
assert.strictEqual(result.length, 1); assert.strictEqual(result.length, 2);
done(); done();
}) })
.catch(err => done(err)); .catch(err => done(err));
@ -231,6 +267,41 @@ describe("lockCategoriesRecords", () => {
.catch(err => done(err)); .catch(err => done(err));
}); });
it("Should be able to submit specific action type not in video (sql check)", (done) => {
const videoID = "lockCategoryVideo-2";
const json = {
videoID,
userID: lockVIPUser,
categories: [
"sponsor",
],
actionTypes: [
"mute"
],
reason: "custom-reason",
};
client.post(endpoint, json)
.then(async res => {
assert.strictEqual(res.status, 200);
const result = await checkLockCategories(videoID);
assert.strictEqual(result.length, 2);
assert.ok(partialDeepEquals(result, [
{
category: "sponsor",
actionType: "skip",
reason: "reason-4",
},
{
category: "sponsor",
actionType: "mute",
reason: "custom-reason",
}
]));
done();
})
.catch(err => done(err));
});
it("Should return 400 for missing params", (done) => { it("Should return 400 for missing params", (done) => {
client.post(endpoint, {}) client.post(endpoint, {})
.then(res => { .then(res => {
@ -365,7 +436,7 @@ describe("lockCategoriesRecords", () => {
.then(async res => { .then(async res => {
assert.strictEqual(res.status, 200); assert.strictEqual(res.status, 200);
const result = await checkLockCategories(videoID); const result = await checkLockCategories(videoID);
assert.strictEqual(result.length, 1); assert.strictEqual(result.length, 2);
done(); done();
}) })
.catch(err => done(err)); .catch(err => done(err));

20
test/cases/postSkipSegments.ts
View file

@ -901,6 +901,26 @@ describe("postSkipSegments", () => {
.catch(err => done(err)); .catch(err => done(err));
}); });
it("Should return not be 403 when submitting with locked category but unlocked actionType", (done) => {
const videoID = "lockedVideo";
db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category", "reason")
VALUES(?, ?, ?, ?)`, [getHash("VIPUser-lockCategories"), videoID, "sponsor", "Custom Reason"])
.then(() => postSkipSegmentJSON({
userID: submitUserOne,
videoID,
segments: [{
segment: [1, 10],
category: "sponsor",
actionType: "mute"
}],
}))
.then(res => {
assert.strictEqual(res.status, 200);
done();
})
.catch(err => done(err));
});
it("Should return 403 for submiting in lockedCategory", (done) => { it("Should return 403 for submiting in lockedCategory", (done) => {
const videoID = "lockedVideo1"; const videoID = "lockedVideo1";
db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category", "reason") db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category", "reason")