external/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2024-11-10 01:02:30 +01:00
Code Issues Projects Releases Packages Wiki Activity

Add random timestamp generation to get branding

Browse source
This commit is contained in:
Ajay 2023-06-08 03:39:44 -04:00
parent 8e5be402e1
commit 5834643ba0
6 changed files with 166 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
package-lock.json generated
View file

@ -20,6 +20,7 @@
"pg": "^8.8.0",
"rate-limit-redis": "^3.0.1",
"redis": "^4.5.0",
"seedrandom": "^3.0.5",
"sync-mysql": "^3.0.1"
},
"devDependencies": {
@ -31,6 +32,7 @@
"@types/mocha": "^10.0.0",
"@types/node": "^18.11.9",
"@types/pg": "^8.6.5",
"@types/seedrandom": "^3.0.5",
"@types/sinon": "^10.0.13",
"@typescript-eslint/eslint-plugin": "^5.44.0",
"@typescript-eslint/parser": "^5.44.0",
@ -984,6 +986,12 @@
"integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==",
"devOptional": true
},
"node_modules/@types/seedrandom": {
"version": "3.0.5",
"resolved": "https://registry.npmjs.org/@types/seedrandom/-/seedrandom-3.0.5.tgz",
"integrity": "sha512-kopEpYpFQvQdYsZkZVwht/0THHmTFFYXDaqV/lM45eweJ8kcGVDgZHs0RVTolSq55UPZNmjhKc9r7UvLu/mQQg==",
"dev": true
},
"node_modules/@types/semver": {
"version": "7.3.13",
"resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.3.13.tgz",
@ -4903,6 +4911,11 @@
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
},
"node_modules/seedrandom": {
"version": "3.0.5",
"resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz",
"integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg=="
},
"node_modules/semver": {
"version": "7.3.7",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.3.7.tgz",
@ -6579,6 +6592,12 @@
"integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==",
"devOptional": true
},
"@types/seedrandom": {
"version": "3.0.5",
"resolved": "https://registry.npmjs.org/@types/seedrandom/-/seedrandom-3.0.5.tgz",
"integrity": "sha512-kopEpYpFQvQdYsZkZVwht/0THHmTFFYXDaqV/lM45eweJ8kcGVDgZHs0RVTolSq55UPZNmjhKc9r7UvLu/mQQg==",
"dev": true
},
"@types/semver": {
"version": "7.3.13",
"resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.3.13.tgz",
@ -9442,6 +9461,11 @@
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
},
"seedrandom": {
"version": "3.0.5",
"resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz",
"integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg=="
},
"semver": {
"version": "7.3.7",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.3.7.tgz",

2
package.json
View file

@ -30,6 +30,7 @@
"pg": "^8.8.0",
"rate-limit-redis": "^3.0.1",
"redis": "^4.5.0",
"seedrandom": "^3.0.5",
"sync-mysql": "^3.0.1"
},
"devDependencies": {
@ -41,6 +42,7 @@
"@types/mocha": "^10.0.0",
"@types/node": "^18.11.9",
"@types/pg": "^8.6.5",
"@types/seedrandom": "^3.0.5",
"@types/sinon": "^10.0.13",
"@typescript-eslint/eslint-plugin": "^5.44.0",
"@typescript-eslint/parser": "^5.44.0",

104
src/routes/getBranding.ts
View file

@ -3,7 +3,7 @@ import { isEmpty } from "lodash";
import { config } from "../config";
import { db, privateDB } from "../databases/databases";
import { Postgres } from "../databases/Postgres";
import { BrandingDBSubmission, BrandingHashDBResult, BrandingResult, ThumbnailDBResult, ThumbnailResult, TitleDBResult, TitleResult } from "../types/branding.model";
import { BrandingDBSubmission, BrandingDBSubmissionData, BrandingHashDBResult, BrandingResult, BrandingSegmentDBResult, BrandingSegmentHashDBResult, ThumbnailDBResult, ThumbnailResult, TitleDBResult, TitleResult } from "../types/branding.model";
import { HashedIP, IPAddress, Service, VideoID, VideoIDHash, Visibility } from "../types/segments.model";
import { shuffleArray } from "../utils/array";
import { getHashCache } from "../utils/getHashCache";
@ -14,6 +14,7 @@ import { Logger } from "../utils/logger";
import { promiseOrTimeout } from "../utils/promise";
import { QueryCacher } from "../utils/queryCacher";
import { brandingHashKey, brandingIPKey, brandingKey } from "../utils/redisKeys";
import * as SeedRandom from "seedrandom";
enum BrandingSubmissionType {
Title = "title",
@ -39,10 +40,25 @@ export async function getVideoBranding(res: Response, videoID: VideoID, service:
{ useReplica: true }
) as Promise<ThumbnailDBResult[]>;
const getBranding = async () => ({
titles: await getTitles(),
thumbnails: await getThumbnails()
});
const getSegments = () => db.prepare(
"all",
`SELECT "startTime", "endTime", "videoDuration" FROM "sponsorTimes"
WHERE "votes" >= 0 AND "shadowHidden" = 0 AND "hidden" = 0 AND "actionType" = 'skip' AND "videoID" = ? AND "service" = ?`,
[videoID, service],
{ useReplica: true }
) as Promise<BrandingSegmentDBResult[]>;
const getBranding = async () => {
const titles = getTitles();
const thumbnails = getThumbnails();
const segments = getSegments();
return {
titles: await titles,
thumbnails: await thumbnails,
segments: await segments
};
};
const brandingTrace = await QueryCacher.getTraced(getBranding, brandingKey(videoID, service));
const branding = brandingTrace.data;
@ -62,7 +78,7 @@ export async function getVideoBranding(res: Response, videoID: VideoID, service:
currentIP: null as Promise<HashedIP> | null
};
return filterAndSortBranding(branding.titles, branding.thumbnails, ip, cache);
return filterAndSortBranding(videoID, branding.titles, branding.thumbnails, branding.segments, ip, cache);
}
export async function getVideoBrandingByHash(videoHashPrefix: VideoIDHash, service: Service, ip: IPAddress): Promise<Record<VideoID, BrandingResult>> {
@ -84,18 +100,28 @@ export async function getVideoBrandingByHash(videoHashPrefix: VideoIDHash, servi
{ useReplica: true }
) as Promise<ThumbnailDBResult[]>;
const getSegments = () => db.prepare(
"all",
`SELECT "videoID", "startTime", "endTime", "videoDuration" FROM "sponsorTimes"
WHERE "votes" >= 0 AND "shadowHidden" = 0 AND "hidden" = 0 AND "actionType" = 'skip' AND "hashedVideoID" LIKE ? AND "service" = ?`,
[`${videoHashPrefix}%`, service],
{ useReplica: true }
) as Promise<BrandingSegmentHashDBResult[]>;
const branding = await QueryCacher.get(async () => {
// Make sure they are both called in parallel
const branding = {
titles: getTitles(),
thumbnails: getThumbnails()
thumbnails: getThumbnails(),
segments: getSegments()
};
const dbResult: Record<VideoID, BrandingHashDBResult> = {};
const initResult = (submission: BrandingDBSubmission) => {
const initResult = (submission: BrandingDBSubmissionData) => {
dbResult[submission.videoID] = dbResult[submission.videoID] || {
titles: [],
thumbnails: []
thumbnails: [],
segments: []
};
};
@ -108,6 +134,11 @@ export async function getVideoBrandingByHash(videoHashPrefix: VideoIDHash, servi
dbResult[thumbnail.videoID].thumbnails.push(thumbnail);
});
(await branding.segments).map((segment) => {
initResult(segment);
dbResult[segment.videoID].segments.push(segment);
});
return dbResult;
}, brandingHashKey(videoHashPrefix, service));
@ -119,13 +150,17 @@ export async function getVideoBrandingByHash(videoHashPrefix: VideoIDHash, servi
const processedResult: Record<VideoID, BrandingResult> = {};
await Promise.all(Object.keys(branding).map(async (key) => {
const castedKey = key as VideoID;
processedResult[castedKey] = await filterAndSortBranding(branding[castedKey].titles, branding[castedKey].thumbnails, ip, cache);
processedResult[castedKey] = await filterAndSortBranding(castedKey, branding[castedKey].titles,
branding[castedKey].thumbnails, branding[castedKey].segments, ip, cache);
}));
return processedResult;
}
async function filterAndSortBranding(dbTitles: TitleDBResult[], dbThumbnails: ThumbnailDBResult[], ip: IPAddress, cache: { currentIP: Promise<HashedIP> | null }): Promise<BrandingResult> {
async function filterAndSortBranding(videoID: VideoID, dbTitles: TitleDBResult[],
dbThumbnails: ThumbnailDBResult[], dbSegments: BrandingSegmentDBResult[],
ip: IPAddress, cache: { currentIP: Promise<HashedIP> | null }): Promise<BrandingResult> {
const shouldKeepTitles = shouldKeepSubmission(dbTitles, BrandingSubmissionType.Title, ip, cache);
const shouldKeepThumbnails = shouldKeepSubmission(dbThumbnails, BrandingSubmissionType.Thumbnail, ip, cache);
@ -153,7 +188,8 @@ async function filterAndSortBranding(dbTitles: TitleDBResult[], dbThumbnails: Th
return {
titles,
thumbnails
thumbnails,
randomTime: findRandomTime(videoID, dbSegments)
};
}
@ -180,6 +216,50 @@ async function shouldKeepSubmission(submissions: BrandingDBSubmission[], type: B
return (_, index) => shouldKeep[index];
}
export function findRandomTime(videoID: VideoID, segments: BrandingSegmentDBResult[]): number {
const randomTime = SeedRandom.alea(videoID)();
if (segments.length === 0) return randomTime;
const videoDuration = segments[0].videoDuration;
// There are segments, treat this as a relative time in the chopped up video
const sorted = segments.sort((a, b) => a.startTime - b.startTime);
const emptySegments: [number, number][] = [];
let totalTime = 0;
let nextEndTime = -1;
for (const segment of sorted) {
if (segment.startTime > nextEndTime) {
if (nextEndTime !== -1) {
emptySegments.push([nextEndTime, segment.startTime]);
totalTime += segment.startTime - nextEndTime;
}
}
nextEndTime = Math.max(segment.endTime, nextEndTime);
}
if (nextEndTime < videoDuration) {
emptySegments.push([nextEndTime, videoDuration]);
totalTime += videoDuration - nextEndTime;
}
let cursor = 0;
for (const segment of emptySegments) {
const duration = segment[1] - segment[0];
if (cursor + duration >= randomTime * totalTime) {
// Found it
return (segment[0] + (randomTime * totalTime - cursor)) / videoDuration;
}
cursor += duration;
}
// Fallback to just the random time
return randomTime;
}
export async function getBranding(req: Request, res: Response) {
const videoID: VideoID = req.query.videoID as VideoID;
const service: Service = getService(req.query.service as string);

25
src/types/branding.model.ts
View file

@ -3,10 +3,13 @@ import { UserID } from "./user.model";
export type BrandingUUID = string & { readonly __brandingUUID: unique symbol };
export interface BrandingDBSubmission {
export interface BrandingDBSubmissionData {
videoID: VideoID,
}
export interface BrandingDBSubmission extends BrandingDBSubmissionData {
shadowHidden: number,
UUID: BrandingUUID,
videoID: VideoID,
hashedVideoID: VideoIDHash
}
@ -42,12 +45,14 @@ export interface ThumbnailResult {
export interface BrandingResult {
titles: TitleResult[],
thumbnails: ThumbnailResult[]
thumbnails: ThumbnailResult[],
randomTime: number
}
export interface BrandingHashDBResult {
titles: TitleDBResult[],
thumbnails: ThumbnailDBResult[]
thumbnails: ThumbnailDBResult[],
segments: BrandingSegmentDBResult[]
}
export interface OriginalThumbnailSubmission {
@ -72,4 +77,16 @@ export interface BrandingSubmission {
videoID: VideoID;
userID: UserID;
service: Service;
}
export interface BrandingSegmentDBResult {
startTime: number;
endTime: number;
videoDuration: number;
}
export interface BrandingSegmentHashDBResult extends BrandingDBSubmissionData {
startTime: number;
endTime: number;
videoDuration: number;
}

4
src/utils/redisKeys.ts
View file

@ -18,13 +18,13 @@ export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: S
}
export const brandingKey = (videoID: VideoID, service: Service): string =>
`branding.${service}.videoID.${videoID}`;
`branding.v2.${service}.videoID.${videoID}`;
export function brandingHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string {
hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 4) as VideoIDHash;
if (hashedVideoIDPrefix.length !== 4) Logger.warn(`Redis skip segment hash-prefix key is not length 4! ${hashedVideoIDPrefix}`);
return `branding.${service}.${hashedVideoIDPrefix}`;
return `branding.v2.${service}.${hashedVideoIDPrefix}`;
}
export const brandingIPKey = (uuid: BrandingUUID): string =>

27
test/cases/getBranding.ts
View file

@ -3,7 +3,7 @@ import assert from "assert";
import { getHash } from "../../src/utils/getHash";
import { db } from "../../src/databases/databases";
import { Service } from "../../src/types/segments.model";
import { BrandingResult, BrandingUUID } from "../../src/types/branding.model";
import { BrandingUUID, ThumbnailResult, TitleResult } from "../../src/types/branding.model";
import { partialDeepEquals } from "../utils/partialDeepEquals";
describe("getBranding", () => {
@ -11,11 +11,13 @@ describe("getBranding", () => {
const videoID2Locked = "videoID2";
const videoID2ShadowHide = "videoID3";
const videoIDEmpty = "videoID4";
const videoIDRandomTime = "videoID5";
const videoID1Hash = getHash(videoID1, 1).slice(0, 4);
const videoID2LockedHash = getHash(videoID2Locked, 1).slice(0, 4);
const videoID2ShadowHideHash = getHash(videoID2ShadowHide, 1).slice(0, 4);
const videoIDEmptyHash = "aaaa";
const videoIDRandomTimeHash = getHash(videoIDRandomTime, 1).slice(0, 4);
const endpoint = "/api/branding";
const getBranding = (params: Record<string, any>) => client({
@ -97,6 +99,10 @@ describe("getBranding", () => {
db.prepare("run", thumbnailVotesQuery, ["UUID22T", 2, 0, 0]),
db.prepare("run", thumbnailVotesQuery, ["UUID32T", 1, 0, 1])
]);
const query = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "actionType", "service", "videoDuration", "hidden", "shadowHidden", "description", "hashedVideoID") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)';
await db.prepare("run", query, [videoIDRandomTime, 1, 11, 1, 0, "uuidbranding1", "testman", 0, 50, "sponsor", "skip", "YouTube", 100, 0, 0, "", videoIDRandomTimeHash]);
await db.prepare("run", query, [videoIDRandomTime, 20, 33, 2, 0, "uuidbranding2", "testman", 0, 50, "intro", "skip", "YouTube", 100, 0, 0, "", videoIDRandomTimeHash]);
});
it("should get top titles and thumbnails", async () => {
@ -221,7 +227,24 @@ describe("getBranding", () => {
assert.strictEqual(result2.status, 404);
});
async function checkVideo(videoID: string, videoIDHash: string, expected: BrandingResult) {
it("should get correct random time", async () => {
const videoDuration = 100;
const result1 = await getBranding({ videoID: videoIDRandomTime });
const result2 = await getBrandingByHash(videoIDRandomTimeHash, {});
const randomTime = result1.data.randomTime;
assert.strictEqual(randomTime, result2.data[videoIDRandomTime].randomTime);
assert.ok(randomTime > 0 && randomTime < 1);
const timeAbsolute = randomTime * videoDuration;
assert.ok(timeAbsolute < 1 || (timeAbsolute > 11 && timeAbsolute < 20) || timeAbsolute > 33);
});
async function checkVideo(videoID: string, videoIDHash: string, expected: {
titles: TitleResult[],
thumbnails: ThumbnailResult[]
}) {
const result1 = await getBranding({ videoID });
const result2 = await getBrandingByHash(videoIDHash, {});