external/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2024-11-10 01:02:30 +01:00
Code Issues Projects Releases Packages Wiki Activity

Add new "Service" option

Browse source
This commit is contained in:
Ajay Ramachandran 2021-03-19 21:31:16 -04:00
parent 8f2ea30da0
commit 29d2c9c25e
8 changed files with 155 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
databases/_upgrade_sponsorTimes_7.sql Normal file
View file

@ -0,0 +1,28 @@
BEGIN TRANSACTION;
/* Add Service field */
CREATE TABLE "sqlb_temp_table_7" (
"videoID" TEXT NOT NULL,
"startTime" REAL NOT NULL,
"endTime" REAL NOT NULL,
"votes" INTEGER NOT NULL,
"locked" INTEGER NOT NULL default '0',
"incorrectVotes" INTEGER NOT NULL default '1',
"UUID" TEXT NOT NULL UNIQUE,
"userID" TEXT NOT NULL,
"timeSubmitted" INTEGER NOT NULL,
"views" INTEGER NOT NULL,
"category" TEXT NOT NULL DEFAULT 'sponsor',
"service" TEXT NOT NULL DEFAULT 'YouTube',
"shadowHidden" INTEGER NOT NULL,
"hashedVideoID" TEXT NOT NULL default ''
);
INSERT INTO sqlb_temp_table_7 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category",'YouTube', "shadowHidden","hashedVideoID" FROM "sponsorTimes";
DROP TABLE "sponsorTimes";
ALTER TABLE sqlb_temp_table_7 RENAME TO "sponsorTimes";
UPDATE "config" SET value = 7 WHERE key = 'version';
COMMIT;

21
src/routes/getSkipSegments.ts
View file

@ -4,7 +4,7 @@ import { config } from '../config';
import { db, privateDB } from '../databases/databases';
import { skipSegmentsKey } from '../middleware/redisKeys';
import { SBRecord } from '../types/lib.model';
import { Category, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model";
import { Category, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, Service, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model";
import { getHash } from '../utils/getHash';
import { getIP } from '../utils/getIP';
import { Logger } from '../utils/logger';
@ -47,7 +47,7 @@ async function prepareCategorySegments(req: Request, videoID: VideoID, category:
}));
}
async function getSegmentsByVideoID(req: Request, videoID: string, categories: Category[]): Promise<Segment[]> {
async function getSegmentsByVideoID(req: Request, videoID: string, categories: Category[], service: Service): Promise<Segment[]> {
const cache: SegmentCache = {shadowHiddenSegmentIPs: {}};
const segments: Segment[] = [];
@ -59,8 +59,8 @@ async function getSegmentsByVideoID(req: Request, videoID: string, categories: C
.prepare(
'all',
`SELECT "startTime", "endTime", "votes", "locked", "UUID", "category", "shadowHidden" FROM "sponsorTimes"
WHERE "videoID" = ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) ORDER BY "startTime"`,
[videoID]
WHERE "videoID" = ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) AND "service" = ? ORDER BY "startTime"`,
[videoID, service]
)).reduce((acc: SBRecord<Category, DBSegment[]>, segment: DBSegment) => {
acc[segment.category] = acc[segment.category] || [];
acc[segment.category].push(segment);
@ -81,7 +81,7 @@ async function getSegmentsByVideoID(req: Request, videoID: string, categories: C
}
}
async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash, categories: Category[]): Promise<SBRecord<VideoID, VideoData>> {
async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash, categories: Category[], service: Service): Promise<SBRecord<VideoID, VideoData>> {
const cache: SegmentCache = {shadowHiddenSegmentIPs: {}};
const segments: SBRecord<VideoID, VideoData> = {};
@ -95,8 +95,8 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash,
.prepare(
'all',
`SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "category", "shadowHidden", "hashedVideoID" FROM "sponsorTimes"
WHERE "hashedVideoID" LIKE ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) ORDER BY "startTime"`,
[hashedVideoIDPrefix + '%']
WHERE "hashedVideoID" LIKE ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) AND "service" = ? ORDER BY "startTime"`,
[hashedVideoIDPrefix + '%', service]
)).reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => {
acc[segment.videoID] = acc[segment.videoID] || {
hash: segment.hashedVideoID,
@ -239,6 +239,11 @@ async function handleGetSegments(req: Request, res: Response): Promise<Segment[]
? [req.query.category]
: ['sponsor'];
let service: Service = req.query.service ?? req.body.service ?? Service.YouTube;
if (!Object.values(Service).some((val) => val == service)) {
service = Service.YouTube;
}
// Only 404s are cached at the moment
const redisResult = await redis.getAsync(skipSegmentsKey(videoID));
@ -251,7 +256,7 @@ async function handleGetSegments(req: Request, res: Response): Promise<Segment[]
}
}
const segments = await getSegmentsByVideoID(req, videoID, categories);
const segments = await getSegmentsByVideoID(req, videoID, categories, service);
if (segments === null || segments === undefined) {
res.sendStatus(500);

9
src/routes/getSkipSegmentsByHash.ts
View file

@ -1,7 +1,7 @@
import {hashPrefixTester} from '../utils/hashPrefixTester';
import {getSegmentsByHash} from './getSkipSegments';
import {Request, Response} from 'express';
import { Category, VideoIDHash } from '../types/segments.model';
import { Category, Service, VideoIDHash } from '../types/segments.model';
export async function getSkipSegmentsByHash(req: Request, res: Response) {
let hashPrefix = req.params.prefix as VideoIDHash;
@ -25,12 +25,17 @@ export async function getSkipSegmentsByHash(req: Request, res: Response) {
catch(error) {
return res.status(400).send("Bad parameter: categories (invalid JSON)");
}
let service: Service = req.query.service ?? req.body.service ?? Service.YouTube;
if (!Object.values(Service).some((val) => val == service)) {
service = Service.YouTube;
}
// filter out none string elements, only flat array with strings is valid
categories = categories.filter((item: any) => typeof item === "string");
// Get all video id's that match hash prefix
const segments = await getSegmentsByHash(req, hashPrefix, categories);
const segments = await getSegmentsByHash(req, hashPrefix, categories, service);
if (!segments) return res.status(404).json([]);

22
src/routes/postSkipSegments.ts
View file

@ -13,6 +13,7 @@ import {dispatchEvent} from '../utils/webhookUtils';
import {Request, Response} from 'express';
import { skipSegmentsKey } from '../middleware/redisKeys';
import redis from '../utils/redis';
import { Service } from '../types/segments.model';
async function sendWebhookNotification(userID: string, videoID: string, UUID: string, submissionCount: number, youtubeData: any, {submissionStart, submissionEnd}: { submissionStart: number; submissionEnd: number; }, segmentInfo: any) {
@ -45,8 +46,8 @@ async function sendWebhookNotification(userID: string, videoID: string, UUID: st
});
}
async function sendWebhooks(userID: string, videoID: string, UUID: string, segmentInfo: any) {
if (config.youtubeAPIKey !== null) {
async function sendWebhooks(userID: string, videoID: string, UUID: string, segmentInfo: any, service: Service) {
if (config.youtubeAPIKey !== null && service == Service.YouTube) {
const userSubmissionCountRow = await db.prepare('get', `SELECT count(*) as "submissionCount" FROM "sponsorTimes" WHERE "userID" = ?`, [userID]);
YouTubeAPI.listVideos(videoID, (err: any, data: any) => {
@ -267,7 +268,10 @@ export async function postSkipSegments(req: Request, res: Response) {
const videoID = req.query.videoID || req.body.videoID;
let userID = req.query.userID || req.body.userID;
let service: Service = req.query.service ?? req.body.service ?? Service.YouTube;
if (!Object.values(Service).some((val) => val == service)) {
service = Service.YouTube;
}
let segments = req.body.segments;
if (segments === undefined) {
@ -367,7 +371,7 @@ export async function postSkipSegments(req: Request, res: Response) {
//check if this info has already been submitted before
const duplicateCheck2Row = await db.prepare('get', `SELECT COUNT(*) as count FROM "sponsorTimes" WHERE "startTime" = ?
and "endTime" = ? and "category" = ? and "videoID" = ?`, [startTime, endTime, segments[i].category, videoID]);
and "endTime" = ? and "category" = ? and "videoID" = ? and "service" = ?`, [startTime, endTime, segments[i].category, videoID, service]);
if (duplicateCheck2Row.count > 0) {
res.sendStatus(409);
return;
@ -375,7 +379,7 @@ export async function postSkipSegments(req: Request, res: Response) {
}
// Auto moderator check
if (!isVIP) {
if (!isVIP && service == Service.YouTube) {
const autoModerateResult = await autoModerateSubmission({userID, videoID, segments});//startTime, endTime, category: segments[i].category});
if (autoModerateResult == "Rejected based on NeuralBlock predictions.") {
// If NB automod rejects, the submission will start with -2 votes.
@ -491,9 +495,9 @@ export async function postSkipSegments(req: Request, res: Response) {
const startingLocked = isVIP ? 1 : 0;
try {
await db.prepare('run', `INSERT INTO "sponsorTimes"
("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "shadowHidden", "hashedVideoID")
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, shadowBanned, getHash(videoID, 1),
("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "service", "shadowHidden", "hashedVideoID")
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, service, shadowBanned, getHash(videoID, 1),
],
);
@ -529,7 +533,7 @@ export async function postSkipSegments(req: Request, res: Response) {
res.json(newSegments);
for (let i = 0; i < segments.length; i++) {
sendWebhooks(userID, videoID, UUIDs[i], segments[i]);
sendWebhooks(userID, videoID, UUIDs[i], segments[i], service);
}
}

10
src/types/segments.model.ts
View file

@ -8,6 +8,16 @@ export type VideoIDHash = VideoID & HashedValue;
export type IPAddress = string & { __ipAddressBrand: unknown };
export type HashedIP = IPAddress & HashedValue;
// Uncomment as needed
export enum Service {
YouTube = 'YouTube',
// Nebula = 'Nebula',
PeerTube = 'PeerTube',
// RSS = 'RSS',
// Corridor = 'Corridor',
// Lbry = 'Lbry'
}
export interface Segment {
category: Category;
segment: number[];

38
test/cases/getSkipSegments.ts
View file

@ -5,16 +5,17 @@ import {getHash} from '../../src/utils/getHash';
describe('getSkipSegments', () => {
before(async () => {
let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "shadowHidden", "hashedVideoID") VALUES';
await db.prepare("run", startOfQuery + "('testtesttest', 1, 11, 2, 0, '1-uuid-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('testtesttest', 1) + "')");
await db.prepare("run", startOfQuery + "('testtesttest', 20, 33, 2, 0, '1-uuid-2', 'testman', 0, 50, 'intro', 0, '" + getHash('testtesttest', 1) + "')");
await db.prepare("run", startOfQuery + "('testtesttest,test', 1, 11, 2, 0, '1-uuid-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('testtesttest,test', 1) + "')");
await db.prepare("run", startOfQuery + "('test3', 1, 11, 2, 0, '1-uuid-4', 'testman', 0, 50, 'sponsor', 0, '" + getHash('test3', 1) + "')");
await db.prepare("run", startOfQuery + "('test3', 7, 22, -3, 0, '1-uuid-5', 'testman', 0, 50, 'sponsor', 0, '" + getHash('test3', 1) + "')");
await db.prepare("run", startOfQuery + "('multiple', 1, 11, 2, 0, '1-uuid-6', 'testman', 0, 50, 'intro', 0, '" + getHash('multiple', 1) + "')");
await db.prepare("run", startOfQuery + "('multiple', 20, 33, 2, 0, '1-uuid-7', 'testman', 0, 50, 'intro', 0, '" + getHash('multiple', 1) + "')");
await db.prepare("run", startOfQuery + "('locked', 20, 33, 2, 1, '1-uuid-locked-8', 'testman', 0, 50, 'intro', 0, '" + getHash('locked', 1) + "')");
await db.prepare("run", startOfQuery + "('locked', 20, 34, 100000, 0, '1-uuid-9', 'testman', 0, 50, 'intro', 0, '" + getHash('locked', 1) + "')");
let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "service", "shadowHidden", "hashedVideoID") VALUES';
await db.prepare("run", startOfQuery + "('testtesttest', 1, 11, 2, 0, '1-uuid-0', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('testtesttest', 1) + "')");
await db.prepare("run", startOfQuery + "('testtesttest2', 1, 11, 2, 0, '1-uuid-0-1', 'testman', 0, 50, 'sponsor', 'PeerTube', 0, '" + getHash('testtesttest2', 1) + "')");
await db.prepare("run", startOfQuery + "('testtesttest', 20, 33, 2, 0, '1-uuid-2', 'testman', 0, 50, 'intro', 'YouTube', 0, '" + getHash('testtesttest', 1) + "')");
await db.prepare("run", startOfQuery + "('testtesttest,test', 1, 11, 2, 0, '1-uuid-1', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('testtesttest,test', 1) + "')");
await db.prepare("run", startOfQuery + "('test3', 1, 11, 2, 0, '1-uuid-4', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('test3', 1) + "')");
await db.prepare("run", startOfQuery + "('test3', 7, 22, -3, 0, '1-uuid-5', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('test3', 1) + "')");
await db.prepare("run", startOfQuery + "('multiple', 1, 11, 2, 0, '1-uuid-6', 'testman', 0, 50, 'intro', 'YouTube', 0, '" + getHash('multiple', 1) + "')");
await db.prepare("run", startOfQuery + "('multiple', 20, 33, 2, 0, '1-uuid-7', 'testman', 0, 50, 'intro', 'YouTube', 0, '" + getHash('multiple', 1) + "')");
await db.prepare("run", startOfQuery + "('locked', 20, 33, 2, 1, '1-uuid-locked-8', 'testman', 0, 50, 'intro', 'YouTube', 0, '" + getHash('locked', 1) + "')");
await db.prepare("run", startOfQuery + "('locked', 20, 34, 100000, 0, '1-uuid-9', 'testman', 0, 50, 'intro', 'YouTube', 0, '" + getHash('locked', 1) + "')");
return;
});
@ -37,6 +38,23 @@ describe('getSkipSegments', () => {
.catch(err => "Couldn't call endpoint");
});
it('Should be able to get a time by category for a different service 1', () => {
fetch(getbaseURL() + "/api/skipSegments?videoID=testtesttest&category=sponsor&service=PeerTube")
.then(async res => {
if (res.status !== 200) return ("Status code was: " + res.status);
else {
const data = await res.json();
if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11
&& data[0].category === "sponsor" && data[0].UUID === "1-uuid-0-1") {
return;
} else {
return ("Received incorrect body: " + (await res.text()));
}
}
})
.catch(err => "Couldn't call endpoint");
});
it('Should be able to get a time by category 2', () => {
fetch(getbaseURL() + "/api/skipSegments?videoID=testtesttest&category=intro")
.then(async res => {

30
test/cases/getSegmentsByHash.ts → test/cases/getSkipSegmentsByHash.ts
View file

@ -12,11 +12,12 @@ sinonStub.callsFake(YouTubeApiMock.listVideos);
describe('getSegmentsByHash', () => {
before(async () => {
let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "shadowHidden", "hashedVideoID") VALUES';
await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 20, 30, 2, 'getSegmentsByHash-0-1', 'testman', 100, 150, 'intro', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
await db.prepare("run", startOfQuery + "('getSegmentsByHash-noMatchHash', 40, 50, 2, 'getSegmentsByHash-noMatchHash', 'testman', 0, 50, 'sponsor', 0, 'fdaffnoMatchHash')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
await db.prepare("run", startOfQuery + "('getSegmentsByHash-1', 60, 70, 2, 'getSegmentsByHash-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('getSegmentsByHash-1', 1) + "')"); // hash = 3272fa85ee0927f6073ef6f07ad5f3146047c1abba794cfa364d65ab9921692b
let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "service", "shadowHidden", "hashedVideoID") VALUES';
await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0-1', 'testman', 0, 50, 'sponsor', 'PeerTube', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 20, 30, 2, 'getSegmentsByHash-0-1', 'testman', 100, 150, 'intro', 'YouTube', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
await db.prepare("run", startOfQuery + "('getSegmentsByHash-noMatchHash', 40, 50, 2, 'getSegmentsByHash-noMatchHash', 'testman', 0, 50, 'sponsor', 'YouTube', 0, 'fdaffnoMatchHash')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
await db.prepare("run", startOfQuery + "('getSegmentsByHash-1', 60, 70, 2, 'getSegmentsByHash-1', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('getSegmentsByHash-1', 1) + "')"); // hash = 3272fa85ee0927f6073ef6f07ad5f3146047c1abba794cfa364d65ab9921692b
});
it('Should be able to get a 200', (done: Done) => {
@ -128,7 +129,24 @@ describe('getSegmentsByHash', () => {
if (body.length !== 2) done("expected 2 videos, got " + body.length);
else if (body[0].segments.length !== 1) done("expected 1 segments for first video, got " + body[0].segments.length);
else if (body[1].segments.length !== 1) done("expected 1 segments for second video, got " + body[1].segments.length);
else if (body[0].segments[0].category !== 'sponsor' || body[1].segments[0].category !== 'sponsor') done("both segments are not sponsor");
else if (body[0].segments[0].category !== 'sponsor'
|| body[0].segments[0].UUID !== 'getSegmentsByHash-0-0'
|| body[1].segments[0].category !== 'sponsor') done("both segments are not sponsor");
else done();
}
})
.catch(err => done("Couldn't call endpoint"));
});
it('Should be able to get 200 for no categories (default sponsor) for a non YouTube service', (done: Done) => {
fetch(getbaseURL() + '/api/skipSegments/fdaf?service=PeerTube')
.then(async res => {
if (res.status !== 200) done("non 200 status code, was " + res.status);
else {
const body = await res.json();
if (body.length !== 1) done("expected 2 videos, got " + body.length);
else if (body[0].segments.length !== 1) done("expected 1 segments for first video, got " + body[0].segments.length);
else if (body[0].segments[0].UUID !== 'getSegmentsByHash-0-0-1') done("both segments are not sponsor");
else done();
}
})

32
test/cases/postSkipSegments.ts
View file

@ -97,6 +97,38 @@ describe('postSkipSegments', () => {
.catch(err => done(err));
});
it('Should be able to submit a single time under a different service (JSON method)', (done: Done) => {
fetch(getbaseURL()
+ "/api/postVideoSponsorTimes", {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
userID: "test",
videoID: "dQw4w9WgXcG",
service: "PeerTube",
segments: [{
segment: [0, 10],
category: "sponsor",
}],
}),
})
.then(async res => {
if (res.status === 200) {
const row = await db.prepare('get', `SELECT "startTime", "endTime", "locked", "category", "service" FROM "sponsorTimes" WHERE "videoID" = ?`, ["dQw4w9WgXcG"]);
if (row.startTime === 0 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.service === "PeerTube") {
done();
} else {
done("Submitted times were not saved. Actual submission: " + JSON.stringify(row));
}
} else {
done("Status code was " + res.status);
}
})
.catch(err => done(err));
});
it('VIP submission should start locked', (done: Done) => {
fetch(getbaseURL()
+ "/api/postVideoSponsorTimes", {