external/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2024-11-10 01:02:30 +01:00
Code Issues Projects Releases Packages Wiki Activity

Add locking by action type

Browse source
This commit is contained in:
Ajay 2022-01-02 14:00:54 -05:00
parent aaa3179d42
commit 09eec5a4a5
6 changed files with 183 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
DatabaseSchema.md
View file

@ -94,6 +94,7 @@
| -- | :--: | -- |
| videoID | TEXT | not null |
| userID | TEXT | not null |
| actionType | TEXT | not null, default 'skip' |
| category | TEXT | not null |
| hashedVideoID | TEXT | not null, default '' |
| reason | TEXT | not null, default '' |

21
databases/_upgrade_sponsorTimes_29.sql Normal file
View file

@ -0,0 +1,21 @@
BEGIN TRANSACTION;
CREATE TABLE "sqlb_temp_table_29" (
"videoID" TEXT NOT NULL,
"userID" TEXT NOT NULL,
"actionType" TEXT NOT NULL DEFAULT 'skip',
"category" TEXT NOT NULL,
"hashedVideoID" TEXT NOT NULL default '',
"reason" TEXT NOT NULL default '',
"service" TEXT NOT NULL default 'YouTube'
);
INSERT INTO sqlb_temp_table_29 SELECT "videoID","userID",'skip',"category","hashedVideoID","reason","service" FROM "lockCategories";
INSERT INTO sqlb_temp_table_29 SELECT "videoID","userID",'mute',"category","hashedVideoID","reason","service" FROM "lockCategories";
DROP TABLE "lockCategories";
ALTER TABLE sqlb_temp_table_29 RENAME TO "lockCategories";
UPDATE "config" SET value = 29 WHERE key = 'version';
COMMIT;

85
src/routes/postLockCategories.ts
View file

@ -3,14 +3,15 @@ import { getHashCache } from "../utils/getHashCache";
import { isUserVIP } from "../utils/isUserVIP";
import { db } from "../databases/databases";
import { Request, Response } from "express";
import { VideoIDHash } from "../types/segments.model";
import { ActionType, Category, VideoIDHash } from "../types/segments.model";
import { getService } from "../utils/getService";
export async function postLockCategories(req: Request, res: Response): Promise<string[]> {
// Collect user input data
const videoID = req.body.videoID;
let userID = req.body.userID;
const categories = req.body.categories;
const categories = req.body.categories as Category[];
const actionTypes = req.body.actionTypes as ActionType[] || [ActionType.Skip, ActionType.Mute];
const reason: string = req.body.reason ?? "";
const service = getService(req.body.service);
@ -20,6 +21,8 @@ export async function postLockCategories(req: Request, res: Response): Promise<s
|| !categories
|| !Array.isArray(categories)
|| categories.length === 0
|| !Array.isArray(actionTypes)
|| actionTypes.length === 0
) {
res.status(400).json({
message: "Bad Format",
@ -38,38 +41,39 @@ export async function postLockCategories(req: Request, res: Response): Promise<s
return;
}
// Get existing lock categories markers
let noCategoryList = await db.prepare("all", 'SELECT "category" from "lockCategories" where "videoID" = ? AND "service" = ?', [videoID, service]);
if (!noCategoryList || noCategoryList.length === 0) {
noCategoryList = [];
} else {
noCategoryList = noCategoryList.map((obj: any) => {
return obj.category;
});
const existingLocks = (await db.prepare("all", 'SELECT "category", "actionType" from "lockCategories" where "videoID" = ? AND "service" = ?', [videoID, service])) as
{ category: Category, actionType: ActionType }[];
const filteredCategories = filterData(categories);
const filteredActionTypes = filterData(actionTypes);
const locksToApply: { category: Category, actionType: ActionType }[] = [];
const overwrittenLocks: { category: Category, actionType: ActionType }[] = [];
for (const category of filteredCategories) {
for (const actionType of filteredActionTypes) {
if (!existingLocks.some((lock) => lock.category === category && lock.actionType === actionType)) {
locksToApply.push({
category,
actionType
});
} else {
overwrittenLocks.push({
category,
actionType
});
}
}
}
// get user categories not already submitted that match accepted format
let filteredCategories = categories.filter((category) => {
return !!category.match(/^[_a-zA-Z]+$/);
});
// remove any duplicates
filteredCategories = filteredCategories.filter((category, index) => {
return filteredCategories.indexOf(category) === index;
});
const categoriesToMark = filteredCategories.filter((category) => {
return noCategoryList.indexOf(category) === -1;
});
// calculate hash of videoID
const hashedVideoID: VideoIDHash = await getHashCache(videoID, 1);
// create database entry
for (const category of categoriesToMark) {
for (const lock of locksToApply) {
try {
await db.prepare("run", `INSERT INTO "lockCategories" ("videoID", "userID", "category", "hashedVideoID", "reason", "service") VALUES(?, ?, ?, ?, ?, ?)`, [videoID, userID, category, hashedVideoID, reason, service]);
await db.prepare("run", `INSERT INTO "lockCategories" ("videoID", "userID", "actionType", "category", "hashedVideoID", "reason", "service") VALUES(?, ?, ?, ?, ?, ?, ?)`, [videoID, userID, lock.actionType, lock.category, hashedVideoID, reason, service]);
} catch (err) {
Logger.error(`Error submitting 'lockCategories' marker for category '${category}' for video '${videoID}' (${service})`);
Logger.error(`Error submitting 'lockCategories' marker for category '${lock.category}' and actionType '${lock.actionType}' for video '${videoID}' (${service})`);
Logger.error(err as string);
res.status(500).json({
message: "Internal Server Error: Could not write marker to the database.",
@ -78,19 +82,14 @@ export async function postLockCategories(req: Request, res: Response): Promise<s
}
// update reason for existed categories
let overlapCategories = [];
if (reason.length !== 0) {
overlapCategories = filteredCategories.filter((category) => {
return noCategoryList.indexOf(category) !== -1;
});
for (const category of overlapCategories) {
for (const lock of overwrittenLocks) {
try {
await db.prepare("run",
'UPDATE "lockCategories" SET "reason" = ?, "userID" = ? WHERE "videoID" = ? AND "category" = ? AND "service" = ?',
[reason, userID, videoID, category, service]);
'UPDATE "lockCategories" SET "reason" = ?, "userID" = ? WHERE "videoID" = ? AND "actionType" = ? AND "category" = ? AND "service" = ?',
[reason, userID, videoID, lock.actionType, lock.category, service]);
} catch (err) {
Logger.error(`Error submitting 'lockCategories' marker for category '${category}' for video '${videoID} (${service})'`);
Logger.error(`Error submitting 'lockCategories' marker for category '${lock.category}' and actionType '${lock.actionType}' for video '${videoID}' (${service})`);
Logger.error(err as string);
res.status(500).json({
message: "Internal Server Error: Could not write marker to the database.",
@ -100,6 +99,20 @@ export async function postLockCategories(req: Request, res: Response): Promise<s
}
res.status(200).json({
submitted: [...categoriesToMark, ...overlapCategories],
submitted: reason.length === 0
? [...filteredCategories.filter(((category) => locksToApply.some((lock) => category === lock.category)))]
: [...filteredCategories], // Legacy
submittedValues: [...locksToApply, ...overwrittenLocks],
});
}
function filterData<T extends string>(data: T[]): T[] {
// get user categories not already submitted that match accepted format
const filtered = data.filter((elem) => {
return !!elem.match(/^[_a-zA-Z]+$/);
});
// remove any duplicates
return filtered.filter((elem, index) => {
return filtered.indexOf(elem) === index;
});
}

4
src/routes/postSkipSegments.ts
View file

@ -357,7 +357,7 @@ async function checkEachSegmentValid(userID: string, videoID: VideoID,
}
// Reject segment if it's in the locked categories list
const lockIndex = lockedCategoryList.findIndex(c => segments[i].category === c.category);
const lockIndex = lockedCategoryList.findIndex(c => segments[i].category === c.category && segments[i].actionType === c.actionType);
if (!isVIP && lockIndex !== -1) {
// TODO: Do something about the fradulent submission
Logger.warn(`Caught a submission for a locked category. userID: '${userID}', videoID: '${videoID}', category: '${segments[i].category}', times: ${segments[i].segment}`);
@ -439,7 +439,7 @@ async function checkByAutoModerator(videoID: any, userID: any, segments: Array<a
}
async function updateDataIfVideoDurationChange(videoID: VideoID, service: Service, videoDuration: VideoDuration, videoDurationParam: VideoDuration) {
let lockedCategoryList = await db.prepare("all", 'SELECT category, reason from "lockCategories" where "videoID" = ? AND "service" = ?', [videoID, service]);
let lockedCategoryList = await db.prepare("all", 'SELECT category, "actionType", reason from "lockCategories" where "videoID" = ? AND "service" = ?', [videoID, service]);
const previousSubmissions = await db.prepare("all",
`SELECT "videoDuration", "UUID"

109
test/cases/lockCategoriesRecords.ts
View file

@ -3,8 +3,9 @@ import { db } from "../../src/databases/databases";
import assert from "assert";
import { LockCategory } from "../../src/types/segments.model";
import { client } from "../utils/httpClient";
import { partialDeepEquals } from "../utils/partialDeepEquals";
const stringDeepEquals = (a: string[] ,b: string[]): boolean => {
const stringDeepEquals = (a: string[], b: string[]): boolean => {
let result = true;
b.forEach((e) => {
if (!a.includes(e)) result = false;
@ -23,18 +24,27 @@ describe("lockCategoriesRecords", () => {
const insertVipUserQuery = 'INSERT INTO "vipUsers" ("userID") VALUES (?)';
await db.prepare("run", insertVipUserQuery, [lockVIPUserHash]);
const insertLockCategoryQuery = 'INSERT INTO "lockCategories" ("userID", "videoID", "category", "reason", "service") VALUES (?, ?, ?, ?, ?)';
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id", "sponsor", "reason-1", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id", "intro", "reason-1", "YouTube"]);
const insertLockCategoryQuery = 'INSERT INTO "lockCategories" ("userID", "videoID", "actionType", "category", "reason", "service") VALUES (?, ?, ?, ?, ?, ?)';
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id", "skip", "sponsor", "reason-1", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id", "mute", "sponsor", "reason-1", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id", "skip", "intro", "reason-1", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id", "mute", "intro", "reason-1", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id-1", "sponsor", "reason-2", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id-1", "intro", "reason-2", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "lockCategoryVideo", "sponsor", "reason-3", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id-1", "skip", "sponsor", "reason-2", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id-1", "mute", "sponsor", "reason-2", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id-1", "skip", "intro", "reason-2", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id-1", "mute", "intro", "reason-2", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "lockCategoryVideo", "skip", "sponsor", "reason-3", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "lockCategoryVideo", "mute", "sponsor", "reason-3", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "delete-record", "sponsor", "reason-4", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "lockCategoryVideo-2", "skip", "sponsor", "reason-4", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "delete-record-1", "sponsor", "reason-5", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "delete-record-1", "intro", "reason-5", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "delete-record", "skip", "sponsor", "reason-4", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "delete-record", "mute", "sponsor", "reason-4", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "delete-record-1", "skip", "sponsor", "reason-5", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "delete-record-1", "mute", "sponsor", "reason-5", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "delete-record-1", "skip", "intro", "reason-5", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "delete-record-1", "mute", "intro", "reason-5", "YouTube"]);
});
it("Should update the database version when starting the application", async () => {
@ -60,6 +70,32 @@ describe("lockCategoriesRecords", () => {
"outro",
"shilling",
],
submittedValues: [
{
actionType: "skip",
category: "outro"
},
{
actionType: "mute",
category: "outro"
},
{
actionType: "skip",
category: "shilling"
},
{
actionType: "mute",
category: "shilling"
},
{
actionType: "skip",
category: "intro"
},
{
actionType: "mute",
category: "intro"
}
]
};
client.post(endpoint, json)
.then(res => {
@ -88,15 +124,15 @@ describe("lockCategoriesRecords", () => {
.then(async res => {
assert.strictEqual(res.status, 200);
const result = await checkLockCategories(videoID);
assert.strictEqual(result.length, 4);
assert.strictEqual(result.length, 8);
const oldRecordNotChangeReason = result.filter(item =>
item.reason === "reason-2" && ["sponsor", "intro"].includes(item.category)
);
const newRecordWithEmptyReason = result.filter(item =>
item.reason === "" && ["outro", "shilling"].includes(item.category)
);
assert.strictEqual(newRecordWithEmptyReason.length, 2);
assert.strictEqual(oldRecordNotChangeReason.length, 2);
assert.strictEqual(newRecordWithEmptyReason.length, 4);
assert.strictEqual(oldRecordNotChangeReason.length, 4);
done();
})
.catch(err => done(err));
@ -160,7 +196,7 @@ describe("lockCategoriesRecords", () => {
.then(async res => {
assert.strictEqual(res.status, 200);
const result = await checkLockCategories(videoID);
assert.strictEqual(result.length, 4);
assert.strictEqual(result.length, 8);
const newRecordWithNewReason = result.filter(item =>
expectedWithNewReason.includes(item.category) && item.reason === "new reason"
);
@ -168,8 +204,8 @@ describe("lockCategoriesRecords", () => {
item.reason === "reason-2"
);
assert.strictEqual(newRecordWithNewReason.length, 3);
assert.strictEqual(oldRecordNotChangeReason.length, 1);
assert.strictEqual(newRecordWithNewReason.length, 6);
assert.strictEqual(oldRecordNotChangeReason.length, 2);
done();
})
.catch(err => done(err));
@ -187,7 +223,7 @@ describe("lockCategoriesRecords", () => {
.then(async res => {
assert.strictEqual(res.status, 200);
const result = await checkLockCategories("underscore");
assert.strictEqual(result.length, 1);
assert.strictEqual(result.length, 2);
done();
})
.catch(err => done(err));
@ -205,7 +241,7 @@ describe("lockCategoriesRecords", () => {
.then(async res => {
assert.strictEqual(res.status, 200);
const result = await checkLockCategories("bothCases");
assert.strictEqual(result.length, 1);
assert.strictEqual(result.length, 2);
done();
})
.catch(err => done(err));
@ -231,6 +267,41 @@ describe("lockCategoriesRecords", () => {
.catch(err => done(err));
});
it("Should be able to submit specific action type not in video (sql check)", (done) => {
const videoID = "lockCategoryVideo-2";
const json = {
videoID,
userID: lockVIPUser,
categories: [
"sponsor",
],
actionTypes: [
"mute"
],
reason: "custom-reason",
};
client.post(endpoint, json)
.then(async res => {
assert.strictEqual(res.status, 200);
const result = await checkLockCategories(videoID);
assert.strictEqual(result.length, 2);
assert.ok(partialDeepEquals(result, [
{
category: "sponsor",
actionType: "skip",
reason: "reason-4",
},
{
category: "sponsor",
actionType: "mute",
reason: "custom-reason",
}
]));
done();
})
.catch(err => done(err));
});
it("Should return 400 for missing params", (done) => {
client.post(endpoint, {})
.then(res => {
@ -365,7 +436,7 @@ describe("lockCategoriesRecords", () => {
.then(async res => {
assert.strictEqual(res.status, 200);
const result = await checkLockCategories(videoID);
assert.strictEqual(result.length, 1);
assert.strictEqual(result.length, 2);
done();
})
.catch(err => done(err));

20
test/cases/postSkipSegments.ts
View file

@ -901,6 +901,26 @@ describe("postSkipSegments", () => {
.catch(err => done(err));
});
it("Should return not be 403 when submitting with locked category but unlocked actionType", (done) => {
const videoID = "lockedVideo";
db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category", "reason")
VALUES(?, ?, ?, ?)`, [getHash("VIPUser-lockCategories"), videoID, "sponsor", "Custom Reason"])
.then(() => postSkipSegmentJSON({
userID: submitUserOne,
videoID,
segments: [{
segment: [1, 10],
category: "sponsor",
actionType: "mute"
}],
}))
.then(res => {
assert.strictEqual(res.status, 200);
done();
})
.catch(err => done(err));
});
it("Should return 403 for submiting in lockedCategory", (done) => {
const videoID = "lockedVideo1";
db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category", "reason")