external/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2024-11-10 01:02:30 +01:00
Code Issues Projects Releases Packages Wiki Activity

Unlock videos and hide segments if duration changed

Browse source
This commit is contained in:
Ajay Ramachandran 2021-03-29 19:16:18 -04:00
parent c17f0b1e6e
commit c9a8dc21b1
8 changed files with 175 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
databases/_upgrade_sponsorTimes_10.sql Normal file
View file

@ -0,0 +1,30 @@
BEGIN TRANSACTION;
/* Add Hidden field */
CREATE TABLE "sqlb_temp_table_10" (
"videoID" TEXT NOT NULL,
"startTime" REAL NOT NULL,
"endTime" REAL NOT NULL,
"votes" INTEGER NOT NULL,
"locked" INTEGER NOT NULL default '0',
"incorrectVotes" INTEGER NOT NULL default '1',
"UUID" TEXT NOT NULL UNIQUE,
"userID" TEXT NOT NULL,
"timeSubmitted" INTEGER NOT NULL,
"views" INTEGER NOT NULL,
"category" TEXT NOT NULL DEFAULT 'sponsor',
"service" TEXT NOT NULL DEFAULT 'YouTube',
"videoDuration" REAL NOT NULL DEFAULT '0',
"hidden" INTEGER NOT NULL DEFAULT '0',
"shadowHidden" INTEGER NOT NULL,
"hashedVideoID" TEXT NOT NULL default ''
);
INSERT INTO sqlb_temp_table_10 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category","service","videoDuration",0,"shadowHidden","hashedVideoID" FROM "sponsorTimes";
DROP TABLE "sponsorTimes";
ALTER TABLE sqlb_temp_table_10 RENAME TO "sponsorTimes";
UPDATE "config" SET value = 10 WHERE key = 'version';
COMMIT;

4
src/app.ts
View file

@ -5,7 +5,7 @@ import {oldGetVideoSponsorTimes} from './routes/oldGetVideoSponsorTimes';
import {postSegmentShift} from './routes/postSegmentShift';
import {postWarning} from './routes/postWarning';
import {getIsUserVIP} from './routes/getIsUserVIP';
import {deleteNoSegments} from './routes/deleteNoSegments';
import {deleteNoSegmentsEndpoint} from './routes/deleteNoSegments';
import {postNoSegments} from './routes/postNoSegments';
import {getUserInfo} from './routes/getUserInfo';
import {getDaysSavedFormatted} from './routes/getDaysSavedFormatted';
@ -117,7 +117,7 @@ function setupRoutes(app: Express) {
//submit video containing no segments
app.post('/api/noSegments', postNoSegments);
app.delete('/api/noSegments', deleteNoSegments);
app.delete('/api/noSegments', deleteNoSegmentsEndpoint);
//get if user is a vip
app.get('/api/isUserVIP', getIsUserVIP);

29
src/routes/deleteNoSegments.ts
View file

@ -2,12 +2,14 @@ import {Request, Response} from 'express';
import {isUserVIP} from '../utils/isUserVIP';
import {getHash} from '../utils/getHash';
import {db} from '../databases/databases';
import { Category, VideoID } from '../types/segments.model';
import { UserID } from '../types/user.model';
export async function deleteNoSegments(req: Request, res: Response) {
export async function deleteNoSegmentsEndpoint(req: Request, res: Response) {
// Collect user input data
const videoID = req.body.videoID;
let userID = req.body.userID;
const categories = req.body.categories;
const videoID = req.body.videoID as VideoID;
const userID = req.body.userID as UserID;
const categories = req.body.categories as Category[];
// Check input data is valid
if (!videoID
@ -23,8 +25,8 @@ export async function deleteNoSegments(req: Request, res: Response) {
}
// Check if user is VIP
userID = getHash(userID);
const userIsVIP = await isUserVIP(userID);
const hashedUserID = getHash(userID);
const userIsVIP = await isUserVIP(hashedUserID);
if (!userIsVIP) {
res.status(403).json({
@ -33,13 +35,22 @@ export async function deleteNoSegments(req: Request, res: Response) {
return;
}
deleteNoSegments(videoID, categories);
res.status(200).json({message: 'Removed no segments entrys for video ' + videoID});
}
/**
*
* @param videoID
* @param categories If null, will remove all
*/
export async function deleteNoSegments(videoID: VideoID, categories: Category[]): Promise<void> {
const entries = (await db.prepare("all", 'SELECT * FROM "noSegments" WHERE "videoID" = ?', [videoID])).filter((entry: any) => {
return (categories.indexOf(entry.category) !== -1);
return categories === null || categories.indexOf(entry.category) !== -1;
});
for (const entry of entries) {
await db.prepare('run', 'DELETE FROM "noSegments" WHERE "videoID" = ? AND "category" = ?', [videoID, entry.category]);
}
res.status(200).json({message: 'Removed no segments entrys for video ' + videoID});
}

4
src/routes/getSkipSegments.ts
View file

@ -60,7 +60,7 @@ async function getSegmentsByVideoID(req: Request, videoID: string, categories: C
.prepare(
'all',
`SELECT "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden" FROM "sponsorTimes"
WHERE "videoID" = ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) AND "service" = ? ORDER BY "startTime"`,
WHERE "videoID" = ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`,
[videoID, service]
)).reduce((acc: SBRecord<Category, DBSegment[]>, segment: DBSegment) => {
acc[segment.category] = acc[segment.category] || [];
@ -132,7 +132,7 @@ async function getSegmentsFromDB(hashedVideoIDPrefix: VideoIDHash, service: Serv
.prepare(
'all',
`SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden", "hashedVideoID" FROM "sponsorTimes"
WHERE "hashedVideoID" LIKE ? AND "service" = ? ORDER BY "startTime"`,
WHERE "hashedVideoID" LIKE ? AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`,
[hashedVideoIDPrefix + '%', service]
);

40
src/routes/postSkipSegments.ts
View file

@ -13,7 +13,8 @@ import {dispatchEvent} from '../utils/webhookUtils';
import {Request, Response} from 'express';
import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys';
import redis from '../utils/redis';
import { Category, IncomingSegment, Segment, Service, VideoDuration, VideoID } from '../types/segments.model';
import { Category, IncomingSegment, Segment, SegmentUUID, Service, VideoDuration, VideoID } from '../types/segments.model';
import { deleteNoSegments } from './deleteNoSegments';
interface APIVideoInfo {
err: string | boolean,
@ -357,7 +358,7 @@ export async function postSkipSegments(req: Request, res: Response) {
return res.status(403).send('Submission rejected due to a warning from a moderator. This means that we noticed you were making some common mistakes that are not malicious, and we just want to clarify the rules. Could you please send a message in Discord or Matrix so we can further help you?');
}
const noSegmentList = (await db.prepare('all', 'SELECT category from "noSegments" where "videoID" = ?', [videoID])).map((list: any) => {
let noSegmentList = (await db.prepare('all', 'SELECT category from "noSegments" where "videoID" = ?', [videoID])).map((list: any) => {
return list.category;
});
@ -366,6 +367,31 @@ export async function postSkipSegments(req: Request, res: Response) {
const decreaseVotes = 0;
let apiVideoInfo: APIVideoInfo = null;
if (service == Service.YouTube) {
apiVideoInfo = await getYouTubeVideoInfo(videoID);
}
const apiVideoDuration = getYouTubeVideoDuration(apiVideoInfo);
if (!videoDuration || (apiVideoDuration && Math.abs(videoDuration - apiVideoDuration) > 2)) {
// If api duration is far off, take that one instead (it is only precise to seconds, not millis)
videoDuration = apiVideoDuration || 0 as VideoDuration;
}
const previousSubmissions = await db.prepare('all', `SELECT "videoDuration", "UUID" FROM "sponsorTimes" WHERE "videoID" = ? AND "service" = ? AND "hidden" = 0 AND "shadowHidden" = 0 AND "votes" >= 0`, [videoID, service]) as
{videoDuration: VideoDuration, UUID: SegmentUUID}[];
// If the video's duration is changed, then the video should be unlocked and old submissions should be hidden
const videoDurationChanged = previousSubmissions.length > 0 && !previousSubmissions.some((e) => Math.abs(videoDuration - e.videoDuration) < 2);
if (videoDurationChanged) {
// Hide all previous submissions
for (const submission of previousSubmissions) {
await db.prepare('run', `UPDATE "sponsorTimes" SET "hidden" = 1 WHERE "UUID" = ?`, [submission.UUID]);
}
// Reset no segments
noSegmentList = [];
deleteNoSegments(videoID, null);
}
// Check if all submissions are correct
for (let i = 0; i < segments.length; i++) {
if (segments[i] === undefined || segments[i].segment === undefined || segments[i].category === undefined) {
@ -419,16 +445,6 @@ export async function postSkipSegments(req: Request, res: Response) {
}
}
let apiVideoInfo: APIVideoInfo = null;
if (service == Service.YouTube) {
apiVideoInfo = await getYouTubeVideoInfo(videoID);
}
const apiVideoDuration = getYouTubeVideoDuration(apiVideoInfo);
if (!videoDuration || (apiVideoDuration && Math.abs(videoDuration - apiVideoDuration) > 2)) {
// If api duration is far off, take that one instead (it is only precise to seconds, not millis)
videoDuration = apiVideoDuration || 0 as VideoDuration;
}
// Auto moderator check
if (!isVIP && service == Service.YouTube) {
const autoModerateResult = await autoModerateSubmission(apiVideoInfo, {userID, videoID, segments});//startTime, endTime, category: segments[i].category});

41
test/cases/getSkipSegments.ts
View file

@ -5,18 +5,19 @@ import {getHash} from '../../src/utils/getHash';
describe('getSkipSegments', () => {
before(async () => {
let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "service", "videoDuration", "shadowHidden", "hashedVideoID") VALUES';
await db.prepare("run", startOfQuery + "('testtesttest', 1, 11, 2, 0, '1-uuid-0', 'testman', 0, 50, 'sponsor', 'YouTube', 100, 0, '" + getHash('testtesttest', 1) + "')");
await db.prepare("run", startOfQuery + "('testtesttest2', 1, 11, 2, 0, '1-uuid-0-1', 'testman', 0, 50, 'sponsor', 'PeerTube', 120, 0, '" + getHash('testtesttest2', 1) + "')");
await db.prepare("run", startOfQuery + "('testtesttest', 20, 33, 2, 0, '1-uuid-2', 'testman', 0, 50, 'intro', 'YouTube', 101, 0, '" + getHash('testtesttest', 1) + "')");
await db.prepare("run", startOfQuery + "('testtesttest,test', 1, 11, 2, 0, '1-uuid-1', 'testman', 0, 50, 'sponsor', 'YouTube', 140, 0, '" + getHash('testtesttest,test', 1) + "')");
await db.prepare("run", startOfQuery + "('test3', 1, 11, 2, 0, '1-uuid-4', 'testman', 0, 50, 'sponsor', 'YouTube', 200, 0, '" + getHash('test3', 1) + "')");
await db.prepare("run", startOfQuery + "('test3', 7, 22, -3, 0, '1-uuid-5', 'testman', 0, 50, 'sponsor', 'YouTube', 300, 0, '" + getHash('test3', 1) + "')");
await db.prepare("run", startOfQuery + "('multiple', 1, 11, 2, 0, '1-uuid-6', 'testman', 0, 50, 'intro', 'YouTube', 400, 0, '" + getHash('multiple', 1) + "')");
await db.prepare("run", startOfQuery + "('multiple', 20, 33, 2, 0, '1-uuid-7', 'testman', 0, 50, 'intro', 'YouTube', 500, 0, '" + getHash('multiple', 1) + "')");
await db.prepare("run", startOfQuery + "('locked', 20, 33, 2, 1, '1-uuid-locked-8', 'testman', 0, 50, 'intro', 'YouTube', 230, 0, '" + getHash('locked', 1) + "')");
await db.prepare("run", startOfQuery + "('locked', 20, 34, 100000, 0, '1-uuid-9', 'testman', 0, 50, 'intro', 'YouTube', 190, 0, '" + getHash('locked', 1) + "')");
let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "service", "videoDuration", "hidden", "shadowHidden", "hashedVideoID") VALUES';
await db.prepare("run", startOfQuery + "('testtesttest', 1, 11, 2, 0, '1-uuid-0', 'testman', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '" + getHash('testtesttest', 1) + "')");
await db.prepare("run", startOfQuery + "('testtesttest2', 1, 11, 2, 0, '1-uuid-0-1', 'testman', 0, 50, 'sponsor', 'PeerTube', 120, 0, 0, '" + getHash('testtesttest2', 1) + "')");
await db.prepare("run", startOfQuery + "('testtesttest', 20, 33, 2, 0, '1-uuid-2', 'testman', 0, 50, 'intro', 'YouTube', 101, 0, 0, '" + getHash('testtesttest', 1) + "')");
await db.prepare("run", startOfQuery + "('testtesttest,test', 1, 11, 2, 0, '1-uuid-1', 'testman', 0, 50, 'sponsor', 'YouTube', 140, 0, 0, '" + getHash('testtesttest,test', 1) + "')");
await db.prepare("run", startOfQuery + "('test3', 1, 11, 2, 0, '1-uuid-4', 'testman', 0, 50, 'sponsor', 'YouTube', 200, 0, 0, '" + getHash('test3', 1) + "')");
await db.prepare("run", startOfQuery + "('test3', 7, 22, -3, 0, '1-uuid-5', 'testman', 0, 50, 'sponsor', 'YouTube', 300, 0, 0, '" + getHash('test3', 1) + "')");
await db.prepare("run", startOfQuery + "('multiple', 1, 11, 2, 0, '1-uuid-6', 'testman', 0, 50, 'intro', 'YouTube', 400, 0, 0, '" + getHash('multiple', 1) + "')");
await db.prepare("run", startOfQuery + "('multiple', 20, 33, 2, 0, '1-uuid-7', 'testman', 0, 50, 'intro', 'YouTube', 500, 0, 0, '" + getHash('multiple', 1) + "')");
await db.prepare("run", startOfQuery + "('locked', 20, 33, 2, 1, '1-uuid-locked-8', 'testman', 0, 50, 'intro', 'YouTube', 230, 0, 0, '" + getHash('locked', 1) + "')");
await db.prepare("run", startOfQuery + "('locked', 20, 34, 100000, 0, '1-uuid-9', 'testman', 0, 50, 'intro', 'YouTube', 190, 0, 0, '" + getHash('locked', 1) + "')");
await db.prepare("run", startOfQuery + "('onlyHiddenSegments', 20, 34, 100000, 0, 'onlyHiddenSegments', 'testman', 0, 50, 'sponsor', 'YouTube', 190, 1, 0, '" + getHash('onlyHiddenSegments', 1) + "')");
return;
});
@ -106,6 +107,22 @@ describe('getSkipSegments', () => {
.catch(err => ("Couldn't call endpoint"));
});
it('Should be empty if all submissions are hidden', () => {
fetch(getbaseURL() + "/api/skipSegments?videoID=onlyHiddenSegments")
.then(async res => {
if (res.status !== 200) return ("Status code was: " + res.status);
else {
const data = await res.json();
if (data.length === 0) {
return;
} else {
return ("Received incorrect body: " + (await res.text()));
}
}
})
.catch(err => ("Couldn't call endpoint"));
});
it('Should be able to get multiple times by category', () => {
fetch(getbaseURL() + "/api/skipSegments?videoID=multiple&categories=[\"intro\"]")
.then(async res => {

26
test/cases/getSkipSegmentsByHash.ts
View file

@ -12,12 +12,13 @@ sinonStub.callsFake(YouTubeApiMock.listVideos);
describe('getSegmentsByHash', () => {
before(async () => {
let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "service", "shadowHidden", "hashedVideoID") VALUES';
await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0-1', 'testman', 0, 50, 'sponsor', 'PeerTube', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 20, 30, 2, 'getSegmentsByHash-0-1', 'testman', 100, 150, 'intro', 'YouTube', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
await db.prepare("run", startOfQuery + "('getSegmentsByHash-noMatchHash', 40, 50, 2, 'getSegmentsByHash-noMatchHash', 'testman', 0, 50, 'sponsor', 'YouTube', 0, 'fdaffnoMatchHash')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
await db.prepare("run", startOfQuery + "('getSegmentsByHash-1', 60, 70, 2, 'getSegmentsByHash-1', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('getSegmentsByHash-1', 1) + "')"); // hash = 3272fa85ee0927f6073ef6f07ad5f3146047c1abba794cfa364d65ab9921692b
let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "service", "hidden", "shadowHidden", "hashedVideoID") VALUES';
await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0', 'testman', 0, 50, 'sponsor', 'YouTube', 0, 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0-1', 'testman', 0, 50, 'sponsor', 'PeerTube', 0, 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 20, 30, 2, 'getSegmentsByHash-0-1', 'testman', 100, 150, 'intro', 'YouTube', 0, 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
await db.prepare("run", startOfQuery + "('getSegmentsByHash-noMatchHash', 40, 50, 2, 'getSegmentsByHash-noMatchHash', 'testman', 0, 50, 'sponsor', 'YouTube', 0, 0, 'fdaffnoMatchHash')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
await db.prepare("run", startOfQuery + "('getSegmentsByHash-1', 60, 70, 2, 'getSegmentsByHash-1', 'testman', 0, 50, 'sponsor', 'YouTube', 0, 0, '" + getHash('getSegmentsByHash-1', 1) + "')"); // hash = 3272fa85ee0927f6073ef6f07ad5f3146047c1abba794cfa364d65ab9921692b
await db.prepare("run", startOfQuery + "('onlyHidden', 60, 70, 2, 'onlyHidden', 'testman', 0, 50, 'sponsor', 'YouTube', 1, 0, '" + getHash('onlyHidden', 1) + "')"); // hash = f3a199e1af001d716cdc6599360e2b062c2d2b3fa2885f6d9d2fd741166cbbd3
});
it('Should be able to get a 200', (done: Done) => {
@ -55,6 +56,19 @@ describe('getSegmentsByHash', () => {
.catch(err => done("Couldn't call endpoint"));
});
it('Should be able to get an empty array if only hidden videos', (done: Done) => {
fetch(getbaseURL() + '/api/skipSegments/f3a1?categories=["sponsor"]')
.then(async res => {
if (res.status !== 404) done("non 404 status code, was " + res.status);
else {
const body = await res.text();
if (JSON.parse(body).length === 0 && body === '[]') done(); // pass
else done("non empty array returned");
}
})
.catch(err => done("Couldn't call endpoint"));
});
it('Should return 400 prefix too short', (done: Done) => {
fetch(getbaseURL() + '/api/skipSegments/11?categories=["shilling"]')
.then(res => {

44
test/cases/postSkipSegments.ts
View file

@ -6,6 +6,7 @@ import {db} from '../../src/databases/databases';
import {ImportMock} from 'ts-mock-imports';
import * as YouTubeAPIModule from '../../src/utils/youtubeApi';
import {YouTubeApiMock} from '../youtubeMock';
import e from 'express';
const mockManager = ImportMock.mockStaticClass(YouTubeAPIModule, 'YouTubeAPI');
const sinonStub = mockManager.mock('listVideos');
@ -193,6 +194,49 @@ describe('postSkipSegments', () => {
.catch(err => done(err));
});
it('Should be able to submit with a new duration, and hide old submissions and remove segment locks', async () => {
await db.prepare("run", `INSERT INTO "noSegments" ("userID", "videoID", "category")
VALUES ('` + getHash("VIPUser-noSegments") + "', 'noDuration', 'sponsor')");
try {
const res = await fetch(getbaseURL()
+ "/api/postVideoSponsorTimes", {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
userID: "test",
videoID: "noDuration",
videoDuration: 100,
segments: [{
segment: [1, 10],
category: "sponsor",
}],
}),
});
if (res.status === 200) {
const noSegmentsRow = await db.prepare('get', `SELECT * from "noSegments" WHERE videoID = ?`, ["noDuration"]);
const videoRows = await db.prepare('all', `SELECT "startTime", "endTime", "locked", "category", "videoDuration"
FROM "sponsorTimes" WHERE "videoID" = ? AND hidden = 0`, ["noDuration"]);
const videoRow = videoRows[0];
const hiddenVideoRows = await db.prepare('all', `SELECT "startTime", "endTime", "locked", "category", "videoDuration"
FROM "sponsorTimes" WHERE "videoID" = ? AND hidden = 1`, ["noDuration"]);
if (noSegmentsRow === undefined && videoRows.length === 1 && hiddenVideoRows.length === 1 && videoRow.startTime === 1 && videoRow.endTime === 10
&& videoRow.locked === 0 && videoRow.category === "sponsor" && videoRow.videoDuration === 100) {
return;
} else {
return "Submitted times were not saved. Actual submission: " + JSON.stringify(videoRow);
}
} else {
return "Status code was " + res.status;
}
} catch (e) {
return e;
}
});
it('Should be able to submit a single time under a different service (JSON method)', (done: Done) => {
fetch(getbaseURL()
+ "/api/postVideoSponsorTimes", {