diff --git a/databases/_upgrade_sponsorTimes_7.sql b/databases/_upgrade_sponsorTimes_7.sql new file mode 100644 index 0000000..8f6060b --- /dev/null +++ b/databases/_upgrade_sponsorTimes_7.sql @@ -0,0 +1,28 @@ +BEGIN TRANSACTION; + +/* Add Service field */ +CREATE TABLE "sqlb_temp_table_7" ( + "videoID" TEXT NOT NULL, + "startTime" REAL NOT NULL, + "endTime" REAL NOT NULL, + "votes" INTEGER NOT NULL, + "locked" INTEGER NOT NULL default '0', + "incorrectVotes" INTEGER NOT NULL default '1', + "UUID" TEXT NOT NULL UNIQUE, + "userID" TEXT NOT NULL, + "timeSubmitted" INTEGER NOT NULL, + "views" INTEGER NOT NULL, + "category" TEXT NOT NULL DEFAULT 'sponsor', + "service" TEXT NOT NULL DEFAULT 'YouTube', + "shadowHidden" INTEGER NOT NULL, + "hashedVideoID" TEXT NOT NULL default '' +); + +INSERT INTO sqlb_temp_table_7 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category",'YouTube', "shadowHidden","hashedVideoID" FROM "sponsorTimes"; + +DROP TABLE "sponsorTimes"; +ALTER TABLE sqlb_temp_table_7 RENAME TO "sponsorTimes"; + +UPDATE "config" SET value = 7 WHERE key = 'version'; + +COMMIT; \ No newline at end of file diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts index 65ab4f1..42a44c7 100644 --- a/src/routes/getSkipSegments.ts +++ b/src/routes/getSkipSegments.ts @@ -4,7 +4,7 @@ import { config } from '../config'; import { db, privateDB } from '../databases/databases'; import { skipSegmentsKey } from '../middleware/redisKeys'; import { SBRecord } from '../types/lib.model'; -import { Category, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model"; +import { Category, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, Service, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model"; import { getHash } from '../utils/getHash'; import { getIP } from '../utils/getIP'; import { Logger } from '../utils/logger'; @@ -47,7 +47,7 @@ async function prepareCategorySegments(req: Request, videoID: VideoID, category: })); } -async function getSegmentsByVideoID(req: Request, videoID: string, categories: Category[]): Promise { +async function getSegmentsByVideoID(req: Request, videoID: string, categories: Category[], service: Service): Promise { const cache: SegmentCache = {shadowHiddenSegmentIPs: {}}; const segments: Segment[] = []; @@ -59,8 +59,8 @@ async function getSegmentsByVideoID(req: Request, videoID: string, categories: C .prepare( 'all', `SELECT "startTime", "endTime", "votes", "locked", "UUID", "category", "shadowHidden" FROM "sponsorTimes" - WHERE "videoID" = ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) ORDER BY "startTime"`, - [videoID] + WHERE "videoID" = ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) AND "service" = ? ORDER BY "startTime"`, + [videoID, service] )).reduce((acc: SBRecord, segment: DBSegment) => { acc[segment.category] = acc[segment.category] || []; acc[segment.category].push(segment); @@ -81,7 +81,7 @@ async function getSegmentsByVideoID(req: Request, videoID: string, categories: C } } -async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash, categories: Category[]): Promise> { +async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash, categories: Category[], service: Service): Promise> { const cache: SegmentCache = {shadowHiddenSegmentIPs: {}}; const segments: SBRecord = {}; @@ -95,8 +95,8 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash, .prepare( 'all', `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "category", "shadowHidden", "hashedVideoID" FROM "sponsorTimes" - WHERE "hashedVideoID" LIKE ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) ORDER BY "startTime"`, - [hashedVideoIDPrefix + '%'] + WHERE "hashedVideoID" LIKE ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) AND "service" = ? ORDER BY "startTime"`, + [hashedVideoIDPrefix + '%', service] )).reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => { acc[segment.videoID] = acc[segment.videoID] || { hash: segment.hashedVideoID, @@ -239,6 +239,11 @@ async function handleGetSegments(req: Request, res: Response): Promise val == service)) { + service = Service.YouTube; + } + // Only 404s are cached at the moment const redisResult = await redis.getAsync(skipSegmentsKey(videoID)); @@ -251,7 +256,7 @@ async function handleGetSegments(req: Request, res: Response): Promise val == service)) { + service = Service.YouTube; + } // filter out none string elements, only flat array with strings is valid categories = categories.filter((item: any) => typeof item === "string"); // Get all video id's that match hash prefix - const segments = await getSegmentsByHash(req, hashPrefix, categories); + const segments = await getSegmentsByHash(req, hashPrefix, categories, service); if (!segments) return res.status(404).json([]); diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index 6a9c781..1101eac 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -13,6 +13,7 @@ import {dispatchEvent} from '../utils/webhookUtils'; import {Request, Response} from 'express'; import { skipSegmentsKey } from '../middleware/redisKeys'; import redis from '../utils/redis'; +import { Service } from '../types/segments.model'; async function sendWebhookNotification(userID: string, videoID: string, UUID: string, submissionCount: number, youtubeData: any, {submissionStart, submissionEnd}: { submissionStart: number; submissionEnd: number; }, segmentInfo: any) { @@ -45,8 +46,8 @@ async function sendWebhookNotification(userID: string, videoID: string, UUID: st }); } -async function sendWebhooks(userID: string, videoID: string, UUID: string, segmentInfo: any) { - if (config.youtubeAPIKey !== null) { +async function sendWebhooks(userID: string, videoID: string, UUID: string, segmentInfo: any, service: Service) { + if (config.youtubeAPIKey !== null && service == Service.YouTube) { const userSubmissionCountRow = await db.prepare('get', `SELECT count(*) as "submissionCount" FROM "sponsorTimes" WHERE "userID" = ?`, [userID]); YouTubeAPI.listVideos(videoID, (err: any, data: any) => { @@ -267,7 +268,10 @@ export async function postSkipSegments(req: Request, res: Response) { const videoID = req.query.videoID || req.body.videoID; let userID = req.query.userID || req.body.userID; - + let service: Service = req.query.service ?? req.body.service ?? Service.YouTube; + if (!Object.values(Service).some((val) => val == service)) { + service = Service.YouTube; + } let segments = req.body.segments; if (segments === undefined) { @@ -367,7 +371,7 @@ export async function postSkipSegments(req: Request, res: Response) { //check if this info has already been submitted before const duplicateCheck2Row = await db.prepare('get', `SELECT COUNT(*) as count FROM "sponsorTimes" WHERE "startTime" = ? - and "endTime" = ? and "category" = ? and "videoID" = ?`, [startTime, endTime, segments[i].category, videoID]); + and "endTime" = ? and "category" = ? and "videoID" = ? and "service" = ?`, [startTime, endTime, segments[i].category, videoID, service]); if (duplicateCheck2Row.count > 0) { res.sendStatus(409); return; @@ -375,7 +379,7 @@ export async function postSkipSegments(req: Request, res: Response) { } // Auto moderator check - if (!isVIP) { + if (!isVIP && service == Service.YouTube) { const autoModerateResult = await autoModerateSubmission({userID, videoID, segments});//startTime, endTime, category: segments[i].category}); if (autoModerateResult == "Rejected based on NeuralBlock predictions.") { // If NB automod rejects, the submission will start with -2 votes. @@ -491,9 +495,9 @@ export async function postSkipSegments(req: Request, res: Response) { const startingLocked = isVIP ? 1 : 0; try { await db.prepare('run', `INSERT INTO "sponsorTimes" - ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "shadowHidden", "hashedVideoID") - VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [ - videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, shadowBanned, getHash(videoID, 1), + ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "service", "shadowHidden", "hashedVideoID") + VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [ + videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, service, shadowBanned, getHash(videoID, 1), ], ); @@ -529,7 +533,7 @@ export async function postSkipSegments(req: Request, res: Response) { res.json(newSegments); for (let i = 0; i < segments.length; i++) { - sendWebhooks(userID, videoID, UUIDs[i], segments[i]); + sendWebhooks(userID, videoID, UUIDs[i], segments[i], service); } } diff --git a/src/types/segments.model.ts b/src/types/segments.model.ts index e7b6eaf..2478b70 100644 --- a/src/types/segments.model.ts +++ b/src/types/segments.model.ts @@ -8,6 +8,16 @@ export type VideoIDHash = VideoID & HashedValue; export type IPAddress = string & { __ipAddressBrand: unknown }; export type HashedIP = IPAddress & HashedValue; +// Uncomment as needed +export enum Service { + YouTube = 'YouTube', + // Nebula = 'Nebula', + PeerTube = 'PeerTube', + // RSS = 'RSS', + // Corridor = 'Corridor', + // Lbry = 'Lbry' +} + export interface Segment { category: Category; segment: number[]; diff --git a/test/cases/getSkipSegments.ts b/test/cases/getSkipSegments.ts index 85dcfbb..51f1c1b 100644 --- a/test/cases/getSkipSegments.ts +++ b/test/cases/getSkipSegments.ts @@ -5,16 +5,17 @@ import {getHash} from '../../src/utils/getHash'; describe('getSkipSegments', () => { before(async () => { - let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "shadowHidden", "hashedVideoID") VALUES'; - await db.prepare("run", startOfQuery + "('testtesttest', 1, 11, 2, 0, '1-uuid-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('testtesttest', 1) + "')"); - await db.prepare("run", startOfQuery + "('testtesttest', 20, 33, 2, 0, '1-uuid-2', 'testman', 0, 50, 'intro', 0, '" + getHash('testtesttest', 1) + "')"); - await db.prepare("run", startOfQuery + "('testtesttest,test', 1, 11, 2, 0, '1-uuid-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('testtesttest,test', 1) + "')"); - await db.prepare("run", startOfQuery + "('test3', 1, 11, 2, 0, '1-uuid-4', 'testman', 0, 50, 'sponsor', 0, '" + getHash('test3', 1) + "')"); - await db.prepare("run", startOfQuery + "('test3', 7, 22, -3, 0, '1-uuid-5', 'testman', 0, 50, 'sponsor', 0, '" + getHash('test3', 1) + "')"); - await db.prepare("run", startOfQuery + "('multiple', 1, 11, 2, 0, '1-uuid-6', 'testman', 0, 50, 'intro', 0, '" + getHash('multiple', 1) + "')"); - await db.prepare("run", startOfQuery + "('multiple', 20, 33, 2, 0, '1-uuid-7', 'testman', 0, 50, 'intro', 0, '" + getHash('multiple', 1) + "')"); - await db.prepare("run", startOfQuery + "('locked', 20, 33, 2, 1, '1-uuid-locked-8', 'testman', 0, 50, 'intro', 0, '" + getHash('locked', 1) + "')"); - await db.prepare("run", startOfQuery + "('locked', 20, 34, 100000, 0, '1-uuid-9', 'testman', 0, 50, 'intro', 0, '" + getHash('locked', 1) + "')"); + let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "service", "shadowHidden", "hashedVideoID") VALUES'; + await db.prepare("run", startOfQuery + "('testtesttest', 1, 11, 2, 0, '1-uuid-0', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('testtesttest', 1) + "')"); + await db.prepare("run", startOfQuery + "('testtesttest2', 1, 11, 2, 0, '1-uuid-0-1', 'testman', 0, 50, 'sponsor', 'PeerTube', 0, '" + getHash('testtesttest2', 1) + "')"); + await db.prepare("run", startOfQuery + "('testtesttest', 20, 33, 2, 0, '1-uuid-2', 'testman', 0, 50, 'intro', 'YouTube', 0, '" + getHash('testtesttest', 1) + "')"); + await db.prepare("run", startOfQuery + "('testtesttest,test', 1, 11, 2, 0, '1-uuid-1', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('testtesttest,test', 1) + "')"); + await db.prepare("run", startOfQuery + "('test3', 1, 11, 2, 0, '1-uuid-4', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('test3', 1) + "')"); + await db.prepare("run", startOfQuery + "('test3', 7, 22, -3, 0, '1-uuid-5', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('test3', 1) + "')"); + await db.prepare("run", startOfQuery + "('multiple', 1, 11, 2, 0, '1-uuid-6', 'testman', 0, 50, 'intro', 'YouTube', 0, '" + getHash('multiple', 1) + "')"); + await db.prepare("run", startOfQuery + "('multiple', 20, 33, 2, 0, '1-uuid-7', 'testman', 0, 50, 'intro', 'YouTube', 0, '" + getHash('multiple', 1) + "')"); + await db.prepare("run", startOfQuery + "('locked', 20, 33, 2, 1, '1-uuid-locked-8', 'testman', 0, 50, 'intro', 'YouTube', 0, '" + getHash('locked', 1) + "')"); + await db.prepare("run", startOfQuery + "('locked', 20, 34, 100000, 0, '1-uuid-9', 'testman', 0, 50, 'intro', 'YouTube', 0, '" + getHash('locked', 1) + "')"); return; }); @@ -37,6 +38,23 @@ describe('getSkipSegments', () => { .catch(err => "Couldn't call endpoint"); }); + it('Should be able to get a time by category for a different service 1', () => { + fetch(getbaseURL() + "/api/skipSegments?videoID=testtesttest&category=sponsor&service=PeerTube") + .then(async res => { + if (res.status !== 200) return ("Status code was: " + res.status); + else { + const data = await res.json(); + if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11 + && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0-1") { + return; + } else { + return ("Received incorrect body: " + (await res.text())); + } + } + }) + .catch(err => "Couldn't call endpoint"); + }); + it('Should be able to get a time by category 2', () => { fetch(getbaseURL() + "/api/skipSegments?videoID=testtesttest&category=intro") .then(async res => { diff --git a/test/cases/getSegmentsByHash.ts b/test/cases/getSkipSegmentsByHash.ts similarity index 80% rename from test/cases/getSegmentsByHash.ts rename to test/cases/getSkipSegmentsByHash.ts index d957e2d..fd45884 100644 --- a/test/cases/getSegmentsByHash.ts +++ b/test/cases/getSkipSegmentsByHash.ts @@ -12,11 +12,12 @@ sinonStub.callsFake(YouTubeApiMock.listVideos); describe('getSegmentsByHash', () => { before(async () => { - let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "shadowHidden", "hashedVideoID") VALUES'; - await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910 - await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 20, 30, 2, 'getSegmentsByHash-0-1', 'testman', 100, 150, 'intro', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910 - await db.prepare("run", startOfQuery + "('getSegmentsByHash-noMatchHash', 40, 50, 2, 'getSegmentsByHash-noMatchHash', 'testman', 0, 50, 'sponsor', 0, 'fdaffnoMatchHash')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910 - await db.prepare("run", startOfQuery + "('getSegmentsByHash-1', 60, 70, 2, 'getSegmentsByHash-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('getSegmentsByHash-1', 1) + "')"); // hash = 3272fa85ee0927f6073ef6f07ad5f3146047c1abba794cfa364d65ab9921692b + let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "service", "shadowHidden", "hashedVideoID") VALUES'; + await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910 + await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0-1', 'testman', 0, 50, 'sponsor', 'PeerTube', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910 + await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 20, 30, 2, 'getSegmentsByHash-0-1', 'testman', 100, 150, 'intro', 'YouTube', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910 + await db.prepare("run", startOfQuery + "('getSegmentsByHash-noMatchHash', 40, 50, 2, 'getSegmentsByHash-noMatchHash', 'testman', 0, 50, 'sponsor', 'YouTube', 0, 'fdaffnoMatchHash')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910 + await db.prepare("run", startOfQuery + "('getSegmentsByHash-1', 60, 70, 2, 'getSegmentsByHash-1', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('getSegmentsByHash-1', 1) + "')"); // hash = 3272fa85ee0927f6073ef6f07ad5f3146047c1abba794cfa364d65ab9921692b }); it('Should be able to get a 200', (done: Done) => { @@ -128,7 +129,24 @@ describe('getSegmentsByHash', () => { if (body.length !== 2) done("expected 2 videos, got " + body.length); else if (body[0].segments.length !== 1) done("expected 1 segments for first video, got " + body[0].segments.length); else if (body[1].segments.length !== 1) done("expected 1 segments for second video, got " + body[1].segments.length); - else if (body[0].segments[0].category !== 'sponsor' || body[1].segments[0].category !== 'sponsor') done("both segments are not sponsor"); + else if (body[0].segments[0].category !== 'sponsor' + || body[0].segments[0].UUID !== 'getSegmentsByHash-0-0' + || body[1].segments[0].category !== 'sponsor') done("both segments are not sponsor"); + else done(); + } + }) + .catch(err => done("Couldn't call endpoint")); + }); + + it('Should be able to get 200 for no categories (default sponsor) for a non YouTube service', (done: Done) => { + fetch(getbaseURL() + '/api/skipSegments/fdaf?service=PeerTube') + .then(async res => { + if (res.status !== 200) done("non 200 status code, was " + res.status); + else { + const body = await res.json(); + if (body.length !== 1) done("expected 2 videos, got " + body.length); + else if (body[0].segments.length !== 1) done("expected 1 segments for first video, got " + body[0].segments.length); + else if (body[0].segments[0].UUID !== 'getSegmentsByHash-0-0-1') done("both segments are not sponsor"); else done(); } }) diff --git a/test/cases/postSkipSegments.ts b/test/cases/postSkipSegments.ts index e31326a..6bc6179 100644 --- a/test/cases/postSkipSegments.ts +++ b/test/cases/postSkipSegments.ts @@ -97,6 +97,38 @@ describe('postSkipSegments', () => { .catch(err => done(err)); }); + it('Should be able to submit a single time under a different service (JSON method)', (done: Done) => { + fetch(getbaseURL() + + "/api/postVideoSponsorTimes", { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + userID: "test", + videoID: "dQw4w9WgXcG", + service: "PeerTube", + segments: [{ + segment: [0, 10], + category: "sponsor", + }], + }), + }) + .then(async res => { + if (res.status === 200) { + const row = await db.prepare('get', `SELECT "startTime", "endTime", "locked", "category", "service" FROM "sponsorTimes" WHERE "videoID" = ?`, ["dQw4w9WgXcG"]); + if (row.startTime === 0 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.service === "PeerTube") { + done(); + } else { + done("Submitted times were not saved. Actual submission: " + JSON.stringify(row)); + } + } else { + done("Status code was " + res.status); + } + }) + .catch(err => done(err)); + }); + it('VIP submission should start locked', (done: Done) => { fetch(getbaseURL() + "/api/postVideoSponsorTimes", {