From cc24a4902fc92c4ca2aafa41bf904389a87a23e1 Mon Sep 17 00:00:00 2001 From: Ajay Date: Fri, 23 Dec 2022 16:56:27 -0500 Subject: [PATCH 01/10] Initial get branding --- databases/_private.db.sql | 26 ++++ databases/_sponsorTimes.db.sql | 43 ++++++ databases/_sponsorTimes_indexes.sql | 34 +++++ src/routes/getBranding.ts | 209 ++++++++++++++++++++++++++++ src/types/branding.model.ts | 58 ++++++++ src/utils/array.ts | 8 ++ src/utils/redisKeys.ts | 15 ++ 7 files changed, 393 insertions(+) create mode 100644 src/routes/getBranding.ts create mode 100644 src/types/branding.model.ts create mode 100644 src/utils/array.ts diff --git a/databases/_private.db.sql b/databases/_private.db.sql index 0da7016..eb73bb2 100644 --- a/databases/_private.db.sql +++ b/databases/_private.db.sql @@ -26,4 +26,30 @@ CREATE TABLE IF NOT EXISTS "config" ( "value" TEXT NOT NULL ); +CREATE TABLE IF NOT EXISTS "titles" ( + "UUID" TEXT NOT NULL PRIMARY KEY, + "hashedIP" TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS "titleVotes" ( + "id" SERIAL PRIMARY KEY, + "UUID" TEXT NOT NULL, + "userID" TEXT NOT NULL, + "hashedIP" TEXT NOT NULL, + "type" INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS "thumbnails" ( + "UUID" TEXT NOT NULL PRIMARY KEY, + "hashedIP" TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS "thumbnailVotes" ( + "id" SERIAL PRIMARY KEY, + "UUID" TEXT NOT NULL, + "userID" TEXT NOT NULL, + "hashedIP" TEXT NOT NULL, + "type" INTEGER NOT NULL +); + COMMIT; diff --git a/databases/_sponsorTimes.db.sql b/databases/_sponsorTimes.db.sql index 1ab5210..a706283 100644 --- a/databases/_sponsorTimes.db.sql +++ b/databases/_sponsorTimes.db.sql @@ -37,6 +37,49 @@ CREATE TABLE IF NOT EXISTS "config" ( "value" TEXT NOT NULL ); +CREATE TABLE IF NOT EXISTS "titles" ( + "videoID" TEXT NOT NULL, + "title" TEXT NOT NULL, + "original" BOOLEAN NOT NULL, + "userID" TEXT NOT NULL, + "service" TEXT NOT NULL, + "hashedVideoID" TEXT NOT NULL, + "timeSubmitted" INTEGER NOT NULL, + "UUID" TEXT NOT NULL PRIMARY KEY +); + +CREATE TABLE IF NOT EXISTS "titleVotes" ( + "UUID" TEXT NOT NULL PRIMARY KEY, + "votes" INTEGER NOT NULL default 0, + "locked" INTEGER NOT NULL default 0, + "shadowHidden" INTEGER NOT NULL default 0, + FOREIGN KEY("UUID") REFERENCES "titles"("UUID") +); + +CREATE TABLE IF NOT EXISTS "thumbnails" ( + "videoID" TEXT NOT NULL, + "original" INTEGER default 0, + "userID" TEXT NOT NULL, + "service" TEXT NOT NULL, + "hashedVideoID" TEXT NOT NULL, + "timeSubmitted" INTEGER NOT NULL, + "UUID" TEXT NOT NULL PRIMARY KEY +); + +CREATE TABLE IF NOT EXISTS "thumbnailTimestamps" ( + "UUID" TEXT NOT NULL PRIMARY KEY, + "timestamp" INTEGER NOT NULL default 0 + FOREIGN KEY("UUID") REFERENCES "thumbnails"("UUID") +); + +CREATE TABLE IF NOT EXISTS "thumbnailVotes" ( + "UUID" TEXT NOT NULL PRIMARY KEY, + "votes" INTEGER NOT NULL default 0, + "locked" INTEGER NOT NULL default 0, + "shadowHidden" INTEGER NOT NULL default 0, + FOREIGN KEY("UUID") REFERENCES "thumbnails"("UUID") +); + CREATE EXTENSION IF NOT EXISTS pgcrypto; --!sqlite-ignore CREATE EXTENSION IF NOT EXISTS pg_trgm; --!sqlite-ignore diff --git a/databases/_sponsorTimes_indexes.sql b/databases/_sponsorTimes_indexes.sql index 4f5c52c..afcd16f 100644 --- a/databases/_sponsorTimes_indexes.sql +++ b/databases/_sponsorTimes_indexes.sql @@ -115,4 +115,38 @@ CREATE INDEX IF NOT EXISTS "ratings_videoID" CREATE INDEX IF NOT EXISTS "userFeatures_userID" ON public."userFeatures" USING btree ("userID" COLLATE pg_catalog."default" ASC NULLS LAST, "feature" ASC NULLS LAST) + TABLESPACE pg_default; + +-- titles + +CREATE INDEX IF NOT EXISTS "titles_timeSubmitted" + ON public."titles" USING btree + ("timeSubmitted" ASC NULLS LAST) + TABLESPACE pg_default; + +CREATE INDEX IF NOT EXISTS "titles_videoID" + ON public."titles" USING btree + ("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, "service" COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; + +CREATE INDEX IF NOT EXISTS "titles_hashedVideoID" + ON public."titles" USING btree + ("hashedVideoID" COLLATE pg_catalog."default" ASC NULLS LAST, "service" COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; + +-- thumbnails + +CREATE INDEX IF NOT EXISTS "thumbnails_timeSubmitted" + ON public."thumbnails" USING btree + ("timeSubmitted" ASC NULLS LAST) + TABLESPACE pg_default; + +CREATE INDEX IF NOT EXISTS "thumbnails_videoID" + ON public."thumbnails" USING btree + ("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, "service" COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; + +CREATE INDEX IF NOT EXISTS "thumbnails_hashedVideoID" + ON public."thumbnails" USING btree + ("hashedVideoID" COLLATE pg_catalog."default" ASC NULLS LAST, "service" COLLATE pg_catalog."default" ASC NULLS LAST) TABLESPACE pg_default; \ No newline at end of file diff --git a/src/routes/getBranding.ts b/src/routes/getBranding.ts new file mode 100644 index 0000000..eae6962 --- /dev/null +++ b/src/routes/getBranding.ts @@ -0,0 +1,209 @@ +import { Request, Response } from "express"; +import { isEmpty } from "lodash"; +import { config } from "../config"; +import { db, privateDB } from "../databases/databases"; +import { BrandingDBSubmission, BrandingHashDBResult, BrandingHashResult, BrandingResult, ThumbnailDBResult, ThumbnailResult, TitleDBResult, TitleResult } from "../types/branding.model"; +import { HashedIP, IPAddress, Service, VideoID, VideoIDHash, Visibility } from "../types/segments.model"; +import { shuffleArray } from "../utils/array"; +import { getHashCache } from "../utils/getHashCache"; +import { getIP } from "../utils/getIP"; +import { getService } from "../utils/getService"; +import { hashPrefixTester } from "../utils/hashPrefixTester"; +import { promiseOrTimeout } from "../utils/promise"; +import { QueryCacher } from "../utils/queryCacher"; +import { brandingHashKey, brandingIPKey, brandingKey } from "../utils/redisKeys"; + +enum BrandingSubmissionType { + Title = "title", + Thumbnail = "thumbnail" +} + +export async function getVideoBranding(videoID: VideoID, service: Service, ip: IPAddress): Promise { + const getTitles = () => db.prepare( + "all", + `SELECT "titles"."title", "titles"."original", "titleVotes"."votes", "titleVotes"."locked", "titleVotes"."shadowHidden", "title"."UUID", "title"."videoID", "title"."hashedVideoID + FROM "titles" JOIN "titleVotes" ON "titles"."UUID" = "titleVotes"."UUID" + WHERE "titles"."videoID" = ? AND "titles"."service" = ? AND "titleVotes"."votes" > -2`, + [videoID, service], + { useReplica: true } + ) as Promise; + + const getThumbnails = () => db.prepare( + "all", + `SELECT "thumbnailTimestamps"."timestamp", "thumbnails"."original", "thumbnailVotes"."votes", "thumbnailVotes"."locked", "thumbnailVotes"."shadowHidden", "thumbnails"."UUID", "thumbnails"."videoID", "thumbnails"."hashedVideoID" + FROM "thumbnails" LEFT JOIN "thumbnailVotes" ON "thumbnails"."UUID" = "thumbnailVotes"."UUID" LEFT JOIN "thumbnailTimestamps" ON "thumbnails"."UUID" = "thumbnailTimestamps"."UUID" + WHERE "thumbnails"."videoID" = ? AND "thumbnails"."service" = ? AND "thumbnailVotes"."votes" > -2`, + [videoID, service], + { useReplica: true } + ) as Promise; + + // eslint-disable-next-line require-await + const getBranding = async () => ({ + titles: getTitles(), + thumbnails: getThumbnails() + }); + + const branding = await QueryCacher.get(getBranding, brandingKey(videoID, service)); + + const cache = { + currentIP: null as Promise | null + }; + + return filterAndSortBranding(await branding.titles, await branding.thumbnails, ip, cache); +} + +export async function getVideoBrandingByHash(videoHashPrefix: VideoIDHash, service: Service, ip: IPAddress): Promise> { + const getTitles = () => db.prepare( + "all", + `SELECT "titles"."title", "titles"."original", "titleVotes"."votes", "titleVotes"."locked", "titleVotes"."shadowHidden", "title"."UUID", "title"."videoID", "title"."hashedVideoID + FROM "titles" JOIN "titleVotes" ON "titles"."UUID" = "titleVotes"."UUID" + WHERE "titles"."hashedVideoID" LIKE ? AND "titles"."service" = ? AND "titleVotes"."votes" > -2`, + [`${videoHashPrefix}%`, service], + { useReplica: true } + ) as Promise; + + const getThumbnails = () => db.prepare( + "all", + `SELECT "thumbnailTimestamps"."timestamp", "thumbnails"."original", "thumbnailVotes"."votes", "thumbnailVotes"."locked", "thumbnailVotes"."shadowHidden", "thumbnails"."UUID", "thumbnails"."videoID", "thumbnails"."hashedVideoID" + FROM "thumbnails" LEFT JOIN "thumbnailVotes" ON "thumbnails"."UUID" = "thumbnailVotes"."UUID" LEFT JOIN "thumbnailTimestamps" ON "thumbnails"."UUID" = "thumbnailTimestamps"."UUID" + WHERE "thumbnails"."hashedVideoID" LIKE ? AND "thumbnails"."service" = ? AND "thumbnailVotes"."votes" > -2`, + [`${videoHashPrefix}%`, service], + { useReplica: true } + ) as Promise; + + const branding = await QueryCacher.get(async () => { + // Make sure they are both called in parallel + const branding = { + titles: getTitles(), + thumbnails: getThumbnails() + }; + + const dbResult: Record = {}; + const initResult = (submission: BrandingDBSubmission) => { + dbResult[submission.videoID] = dbResult[submission.videoID] || { + hash: submission.hashedVideoID, + branding: { + titles: [], + thumbnails: [] + } + }; + }; + + (await branding.titles).map((title) => { + initResult(title); + dbResult[title.videoID].branding.titles.push(title); + }); + (await branding.thumbnails).map((thumbnail) => { + initResult(thumbnail); + dbResult[thumbnail.videoID].branding.thumbnails.push(thumbnail); + }); + + return dbResult; + }, brandingHashKey(videoHashPrefix, service)); + + + const cache = { + currentIP: null as Promise | null + }; + + const processedResult: Record = {}; + await Promise.all(Object.keys(branding).map(async (key) => { + const castedKey = key as VideoID; + processedResult[castedKey] = { + hash: branding[castedKey].hash, + branding: await filterAndSortBranding(branding[castedKey].branding.titles, branding[castedKey].branding.thumbnails, ip, cache) + }; + })); + + return processedResult; +} + +async function filterAndSortBranding(dbTitles: TitleDBResult[], dbThumbnails: ThumbnailDBResult[], ip: IPAddress, cache: { currentIP: Promise | null}): Promise { + const shouldKeepTitles = shouldKeepSubmission(dbTitles, BrandingSubmissionType.Title, ip, cache); + const shouldKeepThumbnails = shouldKeepSubmission(dbThumbnails, BrandingSubmissionType.Thumbnail, ip, cache); + + const titles = shuffleArray(dbTitles.filter(await shouldKeepTitles)) + .sort((a, b) => b.votes - a.votes) + .sort((a, b) => b.locked - a.locked) + .map((r) => ({ + title: r.title, + original: r.original === 1, + votes: r.votes, + locked: r.locked === 1, + UUID: r.UUID, + })) as TitleResult[]; + + const thumbnails = shuffleArray(dbThumbnails.filter(await shouldKeepThumbnails)) + .sort((a, b) => b.votes - a.votes) + .sort((a, b) => b.locked - a.locked) + .map((r) => ({ + timestamp: r.timestamp, + original: r.original === 1, + votes: r.votes, + locked: r.locked === 1, + UUID: r.UUID + })) as ThumbnailResult[]; + + return { + titles, + thumbnails + }; +} + +async function shouldKeepSubmission(submissions: BrandingDBSubmission[], type: BrandingSubmissionType, ip: IPAddress, + cache: { currentIP: Promise | null }): Promise<(_: unknown, index: number) => boolean> { + + const shouldKeep = await Promise.all(submissions.map(async (s) => { + if (s.shadowHidden != Visibility.HIDDEN) return true; + const table = type === BrandingSubmissionType.Title ? "titles" : "thumbnails"; + const fetchData = () => privateDB.prepare("get", `SELECT "hashedIP" FROM "${table}" WHERE "UUID" = ?`, + [s.UUID], { useReplica: true }) as Promise<{ hashedIP: HashedIP }>; + try { + const submitterIP = await promiseOrTimeout(QueryCacher.get(fetchData, brandingIPKey(s.UUID)), 150); + if (cache.currentIP === null) cache.currentIP = getHashCache((ip + config.globalSalt) as IPAddress); + const hashedIP = await cache.currentIP; + + return submitterIP.hashedIP !== hashedIP; + } catch (e) { + // give up on shadow hide for now + return true; + } + })); + + return (_, index) => shouldKeep[index]; +} + +export async function getBranding(req: Request, res: Response) { + const videoID: VideoID = req.query.videoID as VideoID; + const service: Service = getService(req.query.service, req.body.service); + + if (!videoID) { + return res.status(400).send("Missing parameter: videoID"); + } + + const ip = getIP(req); + const result = await getVideoBranding(videoID, service, ip); + + const status = result.titles.length > 0 || result.thumbnails.length > 0 ? 200 : 404; + return res.status(status).json(result); +} + +export async function getBrandingByHash(req: Request, res: Response) { + let hashPrefix = req.params.prefix as VideoIDHash; + if (!req.params.prefix || !hashPrefixTester(req.params.prefix)) { + return res.status(400).send("Hash prefix does not match format requirements."); // Exit early on faulty prefix + } + hashPrefix = hashPrefix.toLowerCase() as VideoIDHash; + + const service: Service = getService(req.query.service, req.body.service); + + if (!hashPrefix || hashPrefix.length !== 4) { + return res.status(400).send("Hash prefix does not match format requirements."); + } + + const ip = getIP(req); + const result = await getVideoBrandingByHash(hashPrefix, service, ip); + + const status = !isEmpty(result) ? 200 : 404; + return res.status(status).json(result); +} \ No newline at end of file diff --git a/src/types/branding.model.ts b/src/types/branding.model.ts new file mode 100644 index 0000000..69bf35d --- /dev/null +++ b/src/types/branding.model.ts @@ -0,0 +1,58 @@ +import { VideoID, VideoIDHash } from "./segments.model"; + +export type BrandingUUID = string & { readonly __brandingUUID: unique symbol }; + +export interface BrandingDBSubmission { + shadowHidden: number, + UUID: BrandingUUID, + videoID: VideoID, + hashedVideoID: VideoIDHash +} + +export interface TitleDBResult extends BrandingDBSubmission { + title: string, + original: number, + votes: number, + locked: number +} + +export interface TitleResult { + title: string, + original: boolean, + votes: number, + locked: boolean, + UUID: BrandingUUID +} + +export interface ThumbnailDBResult extends BrandingDBSubmission { + timestamp?: number, + original: number, + votes: number, + locked: number +} + +export interface ThumbnailResult { + timestamp?: number, + original: boolean, + votes: number, + locked: boolean, + UUID: BrandingUUID +} + +export interface BrandingResult { + titles: TitleResult[], + thumbnails: ThumbnailResult[] +} + +export interface BrandingHashDBResult { + hash: VideoIDHash; + branding: { + titles: TitleDBResult[], + thumbnails: ThumbnailDBResult[] + }; +} + +export interface BrandingHashResult { + hash: VideoIDHash; + branding: BrandingResult; +} \ No newline at end of file diff --git a/src/utils/array.ts b/src/utils/array.ts new file mode 100644 index 0000000..d7cfef6 --- /dev/null +++ b/src/utils/array.ts @@ -0,0 +1,8 @@ +export function shuffleArray(array: T[]): T[] { + for (let i = array.length - 1; i > 0; i--) { + const j = Math.floor(Math.random() * (i + 1)); + [array[i], array[j]] = [array[j], array[i]]; + } + + return array; +} \ No newline at end of file diff --git a/src/utils/redisKeys.ts b/src/utils/redisKeys.ts index e482e10..59ecf76 100644 --- a/src/utils/redisKeys.ts +++ b/src/utils/redisKeys.ts @@ -2,6 +2,7 @@ import { Service, VideoID, VideoIDHash } from "../types/segments.model"; import { Feature, HashedUserID, UserID } from "../types/user.model"; import { HashedValue } from "../types/hash.model"; import { Logger } from "./logger"; +import { BrandingUUID } from "../types/branding.model"; export const skipSegmentsKey = (videoID: VideoID, service: Service): string => `segments.v4.${service}.videoID.${videoID}`; @@ -16,6 +17,20 @@ export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: S return `segments.v4.${service}.${hashedVideoIDPrefix}`; } +export const brandingKey = (videoID: VideoID, service: Service): string => + `branding.${service}.videoID.${videoID}`; + +export function brandingHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string { + hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 4) as VideoIDHash; + if (hashedVideoIDPrefix.length !== 4) Logger.warn(`Redis skip segment hash-prefix key is not length 4! ${hashedVideoIDPrefix}`); + + return `branding.${service}.${hashedVideoIDPrefix}`; +} + +export const brandingIPKey = (uuid: BrandingUUID): string => + `branding.shadow.${uuid}`; + + export const shadowHiddenIPKey = (videoID: VideoID, timeSubmitted: number, service: Service): string => `segments.${service}.videoID.${videoID}.shadow.${timeSubmitted}`; From 2a7083b9ef5346bed88ff58ce7b75564fb6bc02b Mon Sep 17 00:00:00 2001 From: Ajay Date: Fri, 30 Dec 2022 15:07:59 -0500 Subject: [PATCH 02/10] Remove hash from result to save bandwidth --- src/routes/getBranding.ts | 2 -- src/types/branding.model.ts | 2 -- 2 files changed, 4 deletions(-) diff --git a/src/routes/getBranding.ts b/src/routes/getBranding.ts index eae6962..dff3602 100644 --- a/src/routes/getBranding.ts +++ b/src/routes/getBranding.ts @@ -81,7 +81,6 @@ export async function getVideoBrandingByHash(videoHashPrefix: VideoIDHash, servi const dbResult: Record = {}; const initResult = (submission: BrandingDBSubmission) => { dbResult[submission.videoID] = dbResult[submission.videoID] || { - hash: submission.hashedVideoID, branding: { titles: [], thumbnails: [] @@ -110,7 +109,6 @@ export async function getVideoBrandingByHash(videoHashPrefix: VideoIDHash, servi await Promise.all(Object.keys(branding).map(async (key) => { const castedKey = key as VideoID; processedResult[castedKey] = { - hash: branding[castedKey].hash, branding: await filterAndSortBranding(branding[castedKey].branding.titles, branding[castedKey].branding.thumbnails, ip, cache) }; })); diff --git a/src/types/branding.model.ts b/src/types/branding.model.ts index 69bf35d..228c93b 100644 --- a/src/types/branding.model.ts +++ b/src/types/branding.model.ts @@ -45,7 +45,6 @@ export interface BrandingResult { } export interface BrandingHashDBResult { - hash: VideoIDHash; branding: { titles: TitleDBResult[], thumbnails: ThumbnailDBResult[] @@ -53,6 +52,5 @@ export interface BrandingHashDBResult { } export interface BrandingHashResult { - hash: VideoIDHash; branding: BrandingResult; } \ No newline at end of file From 07c683e8f0a98032509112e12a5a960d3a1e7bc9 Mon Sep 17 00:00:00 2001 From: Ajay Date: Fri, 27 Jan 2023 22:36:29 -0500 Subject: [PATCH 03/10] Add vote/submission for titles and thumbnails --- databases/_private.db.sql | 12 +- databases/_sponsorTimes.db.sql | 4 +- src/app.ts | 6 + src/databases/Sqlite.ts | 15 ++ src/routes/getBranding.ts | 2 +- src/routes/postBranding.ts | 129 ++++++++++++ src/types/branding.model.ts | 27 ++- test/cases/postBranding.ts | 362 +++++++++++++++++++++++++++++++++ 8 files changed, 543 insertions(+), 14 deletions(-) create mode 100644 src/routes/postBranding.ts create mode 100644 test/cases/postBranding.ts diff --git a/databases/_private.db.sql b/databases/_private.db.sql index eb73bb2..50cf974 100644 --- a/databases/_private.db.sql +++ b/databases/_private.db.sql @@ -26,26 +26,18 @@ CREATE TABLE IF NOT EXISTS "config" ( "value" TEXT NOT NULL ); -CREATE TABLE IF NOT EXISTS "titles" ( - "UUID" TEXT NOT NULL PRIMARY KEY, - "hashedIP" TEXT NOT NULL -); - CREATE TABLE IF NOT EXISTS "titleVotes" ( "id" SERIAL PRIMARY KEY, + "videoID" TEXT NOT NULL, "UUID" TEXT NOT NULL, "userID" TEXT NOT NULL, "hashedIP" TEXT NOT NULL, "type" INTEGER NOT NULL ); -CREATE TABLE IF NOT EXISTS "thumbnails" ( - "UUID" TEXT NOT NULL PRIMARY KEY, - "hashedIP" TEXT NOT NULL -); - CREATE TABLE IF NOT EXISTS "thumbnailVotes" ( "id" SERIAL PRIMARY KEY, + "videoID" TEXT NOT NULL, "UUID" TEXT NOT NULL, "userID" TEXT NOT NULL, "hashedIP" TEXT NOT NULL, diff --git a/databases/_sponsorTimes.db.sql b/databases/_sponsorTimes.db.sql index a706283..8a5daf7 100644 --- a/databases/_sponsorTimes.db.sql +++ b/databases/_sponsorTimes.db.sql @@ -68,7 +68,7 @@ CREATE TABLE IF NOT EXISTS "thumbnails" ( CREATE TABLE IF NOT EXISTS "thumbnailTimestamps" ( "UUID" TEXT NOT NULL PRIMARY KEY, - "timestamp" INTEGER NOT NULL default 0 + "timestamp" INTEGER NOT NULL default 0, FOREIGN KEY("UUID") REFERENCES "thumbnails"("UUID") ); @@ -76,7 +76,7 @@ CREATE TABLE IF NOT EXISTS "thumbnailVotes" ( "UUID" TEXT NOT NULL PRIMARY KEY, "votes" INTEGER NOT NULL default 0, "locked" INTEGER NOT NULL default 0, - "shadowHidden" INTEGER NOT NULL default 0, + "shadowHidden" INTEGER NOT NULL default 0, FOREIGN KEY("UUID") REFERENCES "thumbnails"("UUID") ); diff --git a/src/app.ts b/src/app.ts index 61b2347..3827713 100644 --- a/src/app.ts +++ b/src/app.ts @@ -50,6 +50,8 @@ import { getVideoLabelsByHash } from "./routes/getVideoLabelByHash"; import { addFeature } from "./routes/addFeature"; import { generateTokenRequest } from "./routes/generateToken"; import { verifyTokenRequest } from "./routes/verifyToken"; +import { getBranding, getBrandingByHashEndpoint } from "./routes/getBranding"; +import { postBranding } from "./routes/postBranding"; export function createServer(callback: () => void): Server { // Create a service (the app object is just a callback). @@ -206,6 +208,10 @@ function setupRoutes(router: Router) { router.get("/api/videoLabels", getVideoLabels); router.get("/api/videoLabels/:prefix", getVideoLabelsByHash); + router.get("/api/branding", getBranding); + router.get("/api/branding/:prefix", getBrandingByHashEndpoint); + router.post("/api/branding", postBranding); + /* istanbul ignore next */ if (config.postgres?.enabled) { router.get("/database", (req, res) => dumpDatabase(req, res, true)); diff --git a/src/databases/Sqlite.ts b/src/databases/Sqlite.ts index ac03753..2e61dfa 100644 --- a/src/databases/Sqlite.ts +++ b/src/databases/Sqlite.ts @@ -14,6 +14,21 @@ export class Sqlite implements IDatabase { // eslint-disable-next-line require-await async prepare(type: QueryType, query: string, params: any[] = []): Promise { + if (query.includes(";")) { + const promises = []; + let paramsCount = 0; + for (const q of query.split(";")) { + if (q.trim() !== "") { + const currentParamCount = q.match(/\?/g)?.length ?? 0; + promises.push(this.prepare(type, q, params.slice(paramsCount, paramsCount + currentParamCount))); + + paramsCount += currentParamCount; + } + } + + return (await Promise.all(promises)).flat(); + } + // Logger.debug(`prepare (sqlite): type: ${type}, query: ${query}, params: ${params}`); const preparedQuery = this.db.prepare(Sqlite.processQuery(query)); diff --git a/src/routes/getBranding.ts b/src/routes/getBranding.ts index dff3602..3d8e686 100644 --- a/src/routes/getBranding.ts +++ b/src/routes/getBranding.ts @@ -186,7 +186,7 @@ export async function getBranding(req: Request, res: Response) { return res.status(status).json(result); } -export async function getBrandingByHash(req: Request, res: Response) { +export async function getBrandingByHashEndpoint(req: Request, res: Response) { let hashPrefix = req.params.prefix as VideoIDHash; if (!req.params.prefix || !hashPrefixTester(req.params.prefix)) { return res.status(400).send("Hash prefix does not match format requirements."); // Exit early on faulty prefix diff --git a/src/routes/postBranding.ts b/src/routes/postBranding.ts new file mode 100644 index 0000000..ddcd6ed --- /dev/null +++ b/src/routes/postBranding.ts @@ -0,0 +1,129 @@ +import { Request, Response } from "express"; +import { config } from "../config"; +import { db, privateDB } from "../databases/databases"; + +import { BrandingSubmission, BrandingUUID, TimeThumbnailSubmission } from "../types/branding.model"; +import { HashedIP, IPAddress, VideoID } from "../types/segments.model"; +import { HashedUserID } from "../types/user.model"; +import { getHashCache } from "../utils/getHashCache"; +import { getIP } from "../utils/getIP"; +import { getService } from "../utils/getService"; +import { isUserVIP } from "../utils/isUserVIP"; +import { Logger } from "../utils/logger"; + +enum BrandingType { + Title, + Thumbnail +} + +interface ExistingVote { + UUID: BrandingUUID; + type: number; + id: number; +} + +export async function postBranding(req: Request, res: Response) { + const { videoID, userID, title, thumbnail } = req.body as BrandingSubmission; + const service = getService(req.body.service); + + if (!videoID || !userID || userID.length < 30 || !service + || ((!title || !title.title) + && (!thumbnail || thumbnail.original == null + || (!thumbnail.original && !(thumbnail as TimeThumbnailSubmission).timestamp)))) { + res.status(400).send("Bad Request"); + return; + } + + try { + const hashedUserID = await getHashCache(userID); + const isVip = await isUserVIP(hashedUserID); + const hashedVideoID = await getHashCache(videoID, 1); + const hashedIP = await getHashCache(getIP(req) + config.globalSalt as IPAddress); + + const now = Date.now(); + const voteType = 1; + + await Promise.all([(async () => { + if (title) { + const existingUUID = (await db.prepare("get", `SELECT "UUID" from "titles" where "videoID" = ? AND "title" = ?`, [videoID, title.title]))?.UUID; + const UUID = existingUUID || crypto.randomUUID(); + + const existingVote = await handleExistingVotes(BrandingType.Title, videoID, hashedUserID, UUID, hashedIP, voteType); + if (existingUUID) { + await updateVoteTotals(BrandingType.Title, existingVote, UUID, isVip); + } else { + await db.prepare("run", `INSERT INTO "titles" ("videoID", "title", "original", "userID", "service", "hashedVideoID", "timeSubmitted", "UUID") VALUES (?, ?, ?, ?, ?, ?, ?, ?); + INSERT INTO "titleVotes" ("UUID", "votes", "locked", "shadowHidden") VALUES (?, 0, ?, 0);`, + [videoID, title.title, title.original ? 1 : 0, hashedUserID, service, hashedVideoID, now, UUID, UUID, isVip ? 1 : 0]); + } + } + })(), (async () => { + if (thumbnail) { + const existingUUID = thumbnail.original + ? (await db.prepare("get", `SELECT "UUID" from "thumbnails" where "videoID" = ? AND "original" = 1`, [videoID]))?.UUID + : (await db.prepare("get", `SELECT "thumbnails"."UUID" from "thumbnailTimestamps" JOIN "thumbnails" ON "thumbnails"."UUID" = "thumbnailTimestamps"."UUID" + WHERE "thumbnailTimestamps"."timestamp" = ? AND "thumbnails"."videoID" = ?`, [(thumbnail as TimeThumbnailSubmission).timestamp, videoID]))?.UUID; + const UUID = existingUUID || crypto.randomUUID(); + + const existingVote = await handleExistingVotes(BrandingType.Thumbnail, videoID, hashedUserID, UUID, hashedIP, voteType); + if (existingUUID) { + await updateVoteTotals(BrandingType.Thumbnail, existingVote, UUID, isVip); + } else { + await db.prepare("run", `INSERT INTO "thumbnails" ("videoID", "original", "userID", "service", "hashedVideoID", "timeSubmitted", "UUID") VALUES (?, ?, ?, ?, ?, ?, ?); + INSERT INTO "thumbnailVotes" ("UUID", "votes", "locked", "shadowHidden") VALUES (?, 0, ?, 0); + ${thumbnail.original ? "" : `INSERT INTO "thumbnailTimestamps" ("UUID", "timestamp") VALUES (?, ?)`}`, + [videoID, thumbnail.original ? 1 : 0, hashedUserID, service, hashedVideoID, now, UUID, UUID, + isVip ? 1 : 0, thumbnail.original ? null : UUID, thumbnail.original ? null : (thumbnail as TimeThumbnailSubmission).timestamp]); + } + + } + })()]); + + res.status(200).send("OK"); + } catch (e) { + Logger.error(e as string); + res.status(500).send("Internal Server Error"); + } +} + +/** + * Finds an existing vote, if found, and it's for a different submission, it undoes it, and points to the new submission. + * If no existing vote, it adds one. + */ +async function handleExistingVotes(type: BrandingType, videoID: VideoID, + hashedUserID: HashedUserID, UUID: BrandingUUID, hashedIP: HashedIP, voteType: number): Promise { + const table = type === BrandingType.Title ? `"titleVotes"` : `"thumbnailVotes"`; + + const existingVote = await privateDB.prepare("get", `SELECT "id", "UUID", "type" from ${table} where "videoID" = ? AND "userID" = ?`, [videoID, hashedUserID]); + if (existingVote && existingVote.UUID !== UUID) { + if (existingVote.type === 1) { + await db.prepare("run", `UPDATE ${table} SET "votes" = "votes" - 1 WHERE "UUID" = ?`, [existingVote.UUID]); + } + + await privateDB.prepare("run", `UPDATE ${table} SET "type" = ?, "UUID" = ? WHERE "id" = ?`, [voteType, UUID, existingVote.id]); + } else if (!existingVote) { + await privateDB.prepare("run", `INSERT INTO ${table} ("videoID", "UUID", "userID", "hashedIP", "type") VALUES (?, ?, ?, ?, ?)`, + [videoID, UUID, hashedUserID, hashedIP, voteType]); + } + + return existingVote; +} + +/** + * Only called if an existing vote exists. + * Will update public vote totals and locked status. + */ +async function updateVoteTotals(type: BrandingType, existingVote: ExistingVote, UUID: BrandingUUID, isVip: boolean): Promise { + if (!existingVote) return; + + const table = type === BrandingType.Title ? `"titleVotes"` : `"thumbnailVotes"`; + + // Don't upvote if we vote on the same submission + if (!existingVote || existingVote.UUID !== UUID) { + await db.prepare("run", `UPDATE ${table} SET "votes" = "votes" + 1 WHERE "UUID" = ?`, [UUID]); + } + + if (isVip) { + await db.prepare("run", `UPDATE ${table} SET "locked" = 1 WHERE "UUID" = ?`, [UUID]); + } +} \ No newline at end of file diff --git a/src/types/branding.model.ts b/src/types/branding.model.ts index 228c93b..8662ad3 100644 --- a/src/types/branding.model.ts +++ b/src/types/branding.model.ts @@ -1,4 +1,5 @@ -import { VideoID, VideoIDHash } from "./segments.model"; +import { Service, VideoID, VideoIDHash } from "./segments.model"; +import { UserID } from "./user.model"; export type BrandingUUID = string & { readonly __brandingUUID: unique symbol }; @@ -53,4 +54,28 @@ export interface BrandingHashDBResult { export interface BrandingHashResult { branding: BrandingResult; +} + +export interface OriginalThumbnailSubmission { + original: true; +} + +export interface TimeThumbnailSubmission { + timestamp: number; + original: false; +} + +export type ThumbnailSubmission = OriginalThumbnailSubmission | TimeThumbnailSubmission; + +export interface TitleSubmission { + title: string; + original: boolean; +} + +export interface BrandingSubmission { + title: TitleSubmission; + thumbnail: ThumbnailSubmission; + videoID: VideoID; + userID: UserID; + service: Service; } \ No newline at end of file diff --git a/test/cases/postBranding.ts b/test/cases/postBranding.ts new file mode 100644 index 0000000..ccc15f1 --- /dev/null +++ b/test/cases/postBranding.ts @@ -0,0 +1,362 @@ +import { db } from "../../src/databases/databases"; +import { getHash } from "../../src/utils/getHash"; +import { client } from "../utils/httpClient"; +import assert from "assert"; +import { Service } from "../../src/types/segments.model"; + +describe("postBranding", () => { + + const vipUser = `VIPPostBrandingUser${".".repeat(16)}`; + const userID1 = `PostBrandingUser1${".".repeat(16)}`; + const userID2 = `PostBrandingUser2${".".repeat(16)}`; + const userID3 = `PostBrandingUser3${".".repeat(16)}`; + const userID4 = `PostBrandingUser4${".".repeat(16)}`; + const userID5 = `PostBrandingUser4${".".repeat(16)}`; + + const endpoint = "/api/branding"; + const postBranding = (data: Record) => client({ + method: "POST", + url: endpoint, + data + }); + + const queryTitleByVideo = (videoID: string) => db.prepare("get", `SELECT * FROM "titles" WHERE "videoID" = ? ORDER BY "timeSubmitted" DESC`, [videoID]); + const queryThumbnailByVideo = (videoID: string) => db.prepare("get", `SELECT * FROM "thumbnails" WHERE "videoID" = ? ORDER BY "timeSubmitted" DESC`, [videoID]); + const queryThumbnailTimestampsByUUID = (UUID: string) => db.prepare("get", `SELECT * FROM "thumbnailTimestamps" WHERE "UUID" = ?`, [UUID]); + const queryTitleVotesByUUID = (UUID: string) => db.prepare("get", `SELECT * FROM "titleVotes" WHERE "UUID" = ?`, [UUID]); + const queryThumbnailVotesByUUID = (UUID: string) => db.prepare("get", `SELECT * FROM "thumbnailVotes" WHERE "UUID" = ?`, [UUID]); + + before(() => { + const insertVipUserQuery = 'INSERT INTO "vipUsers" ("userID") VALUES (?)'; + db.prepare("run", insertVipUserQuery, [getHash(vipUser)]); + }); + + it("Submit only title", async () => { + const videoID = "postBrand1"; + const title = { + title: "Some title", + original: false + }; + + const res = await postBranding({ + title, + userID: userID1, + service: Service.YouTube, + videoID + }); + + assert.strictEqual(res.status, 200); + const dbTitle = await queryTitleByVideo(videoID); + const dbVotes = await queryTitleVotesByUUID(dbTitle.UUID); + + assert.strictEqual(dbTitle.title, title.title); + assert.strictEqual(dbTitle.original, title.original ? 1 : 0); + + assert.strictEqual(dbVotes.votes, 0); + assert.strictEqual(dbVotes.locked, 0); + assert.strictEqual(dbVotes.shadowHidden, 0); + }); + + it("Submit only original title", async () => { + const videoID = "postBrand2"; + const title = { + title: "Some title", + original: true + }; + + const res = await postBranding({ + title, + userID: userID2, + service: Service.YouTube, + videoID + }); + + assert.strictEqual(res.status, 200); + const dbTitle = await queryTitleByVideo(videoID); + const dbVotes = await queryTitleVotesByUUID(dbTitle.UUID); + + assert.strictEqual(dbTitle.title, title.title); + assert.strictEqual(dbTitle.original, title.original ? 1 : 0); + + assert.strictEqual(dbVotes.votes, 0); + assert.strictEqual(dbVotes.locked, 0); + assert.strictEqual(dbVotes.shadowHidden, 0); + }); + + it("Submit only original thumbnail", async () => { + const videoID = "postBrand3"; + const thumbnail = { + original: true + }; + + const res = await postBranding({ + thumbnail, + userID: userID3, + service: Service.YouTube, + videoID + }); + + assert.strictEqual(res.status, 200); + const dbThumbnail = await queryThumbnailByVideo(videoID); + const dbVotes = await queryThumbnailVotesByUUID(dbThumbnail.UUID); + + assert.strictEqual(dbThumbnail.original, thumbnail.original ? 1 : 0); + + assert.strictEqual(dbVotes.votes, 0); + assert.strictEqual(dbVotes.locked, 0); + assert.strictEqual(dbVotes.shadowHidden, 0); + }); + + it("Submit only custom thumbnail", async () => { + const videoID = "postBrand4"; + const thumbnail = { + timestamp: 12.42, + original: false + }; + + const res = await postBranding({ + thumbnail, + userID: userID4, + service: Service.YouTube, + videoID + }); + + assert.strictEqual(res.status, 200); + const dbThumbnail = await queryThumbnailByVideo(videoID); + const dbThumbnailTimestamps = await queryThumbnailTimestampsByUUID(dbThumbnail.UUID); + const dbVotes = await queryThumbnailVotesByUUID(dbThumbnail.UUID); + + assert.strictEqual(dbThumbnailTimestamps.timestamp, thumbnail.timestamp); + assert.strictEqual(dbThumbnail.original, thumbnail.original ? 1 : 0); + + assert.strictEqual(dbVotes.votes, 0); + assert.strictEqual(dbVotes.locked, 0); + assert.strictEqual(dbVotes.shadowHidden, 0); + }); + + it("Submit title and thumbnail", async () => { + const videoID = "postBrand5"; + const title = { + title: "Some title", + original: false + }; + const thumbnail = { + timestamp: 12.42, + original: false + }; + + const res = await postBranding({ + title, + thumbnail, + userID: userID5, + service: Service.YouTube, + videoID + }); + + assert.strictEqual(res.status, 200); + const dbTitle = await queryTitleByVideo(videoID); + const dbTitleVotes = await queryTitleVotesByUUID(dbTitle.UUID); + const dbThumbnail = await queryThumbnailByVideo(videoID); + const dbThumbnailTimestamps = await queryThumbnailTimestampsByUUID(dbThumbnail.UUID); + const dbThumbnailVotes = await queryThumbnailVotesByUUID(dbThumbnail.UUID); + + assert.strictEqual(dbTitle.title, title.title); + assert.strictEqual(dbTitle.original, title.original ? 1 : 0); + + assert.strictEqual(dbTitleVotes.votes, 0); + assert.strictEqual(dbTitleVotes.locked, 0); + assert.strictEqual(dbTitleVotes.shadowHidden, 0); + + assert.strictEqual(dbThumbnailTimestamps.timestamp, thumbnail.timestamp); + assert.strictEqual(dbThumbnail.original, thumbnail.original ? 1 : 0); + + assert.strictEqual(dbThumbnailVotes.votes, 0); + assert.strictEqual(dbThumbnailVotes.locked, 0); + assert.strictEqual(dbThumbnailVotes.shadowHidden, 0); + }); + + it("Submit title and thumbnail as VIP", async () => { + const videoID = "postBrand6"; + const title = { + title: "Some title", + original: false + }; + const thumbnail = { + timestamp: 12.42, + original: false + }; + + const res = await postBranding({ + title, + thumbnail, + userID: vipUser, + service: Service.YouTube, + videoID + }); + + assert.strictEqual(res.status, 200); + const dbTitle = await queryTitleByVideo(videoID); + const dbTitleVotes = await queryTitleVotesByUUID(dbTitle.UUID); + const dbThumbnail = await queryThumbnailByVideo(videoID); + const dbThumbnailTimestamps = await queryThumbnailTimestampsByUUID(dbThumbnail.UUID); + const dbThumbnailVotes = await queryThumbnailVotesByUUID(dbThumbnail.UUID); + + assert.strictEqual(dbTitle.title, title.title); + assert.strictEqual(dbTitle.original, title.original ? 1 : 0); + + assert.strictEqual(dbTitleVotes.votes, 0); + assert.strictEqual(dbTitleVotes.locked, 1); + assert.strictEqual(dbTitleVotes.shadowHidden, 0); + + assert.strictEqual(dbThumbnailTimestamps.timestamp, thumbnail.timestamp); + assert.strictEqual(dbThumbnail.original, thumbnail.original ? 1 : 0); + + assert.strictEqual(dbThumbnailVotes.votes, 0); + assert.strictEqual(dbThumbnailVotes.locked, 1); + assert.strictEqual(dbThumbnailVotes.shadowHidden, 0); + }); + + it("Vote the same title again", async () => { + const videoID = "postBrand1"; + const title = { + title: "Some title", + original: false + }; + + const res = await postBranding({ + title, + userID: userID1, + service: Service.YouTube, + videoID + }); + + assert.strictEqual(res.status, 200); + const dbTitle = await queryTitleByVideo(videoID); + const dbVotes = await queryTitleVotesByUUID(dbTitle.UUID); + + assert.strictEqual(dbTitle.title, title.title); + assert.strictEqual(dbTitle.original, title.original ? 1 : 0); + + assert.strictEqual(dbVotes.votes, 0); + assert.strictEqual(dbVotes.locked, 0); + assert.strictEqual(dbVotes.shadowHidden, 0); + }); + + it("Vote for a different title", async () => { + const videoID = "postBrand1"; + const title = { + title: "Some other title", + original: false + }; + + const oldDbTitle = await queryTitleByVideo(videoID); + + const res = await postBranding({ + title, + userID: userID1, + service: Service.YouTube, + videoID + }); + + assert.strictEqual(res.status, 200); + const dbTitle = await queryTitleByVideo(videoID); + const dbVotes = await queryTitleVotesByUUID(dbTitle.UUID); + const oldDBVotes = await queryTitleVotesByUUID(oldDbTitle.UUID); + + assert.strictEqual(dbTitle.title, title.title); + assert.strictEqual(dbTitle.original, title.original ? 1 : 0); + + assert.strictEqual(dbVotes.votes, 0); + assert.strictEqual(dbVotes.locked, 0); + assert.strictEqual(dbVotes.shadowHidden, 0); + + assert.strictEqual(oldDBVotes.votes, -1); + assert.strictEqual(oldDBVotes.locked, 0); + assert.strictEqual(oldDBVotes.shadowHidden, 0); + }); + + it("Vote for the same thumbnail again", async () => { + const videoID = "postBrand4"; + const thumbnail = { + timestamp: 12.42, + original: false + }; + + const res = await postBranding({ + thumbnail, + userID: userID4, + service: Service.YouTube, + videoID + }); + + assert.strictEqual(res.status, 200); + const dbThumbnail = await queryThumbnailByVideo(videoID); + const dbThumbnailTimestamps = await queryThumbnailTimestampsByUUID(dbThumbnail.UUID); + const dbVotes = await queryThumbnailVotesByUUID(dbThumbnail.UUID); + + assert.strictEqual(dbThumbnailTimestamps.timestamp, thumbnail.timestamp); + assert.strictEqual(dbThumbnail.original, thumbnail.original ? 1 : 0); + + assert.strictEqual(dbVotes.votes, 0); + assert.strictEqual(dbVotes.locked, 0); + assert.strictEqual(dbVotes.shadowHidden, 0); + }); + + it("Vote for the same thumbnail again original", async () => { + const videoID = "postBrand3"; + const thumbnail = { + original: true + }; + + const res = await postBranding({ + thumbnail, + userID: userID3, + service: Service.YouTube, + videoID + }); + + assert.strictEqual(res.status, 200); + const dbThumbnail = await queryThumbnailByVideo(videoID); + const dbVotes = await queryThumbnailVotesByUUID(dbThumbnail.UUID); + + assert.strictEqual(dbThumbnail.original, thumbnail.original ? 1 : 0); + + assert.strictEqual(dbVotes.votes, 0); + assert.strictEqual(dbVotes.locked, 0); + assert.strictEqual(dbVotes.shadowHidden, 0); + }); + + it("Vote for a different thumbnail", async () => { + const videoID = "postBrand4"; + const thumbnail = { + timestamp: 15.34, + original: false + }; + + const oldDbThumbnail = await queryThumbnailByVideo(videoID); + + const res = await postBranding({ + thumbnail, + userID: userID4, + service: Service.YouTube, + videoID + }); + + assert.strictEqual(res.status, 200); + const dbThumbnail = await queryThumbnailByVideo(videoID); + const dbThumbnailTimestamps = await queryThumbnailTimestampsByUUID(dbThumbnail.UUID); + const dbVotes = await queryThumbnailVotesByUUID(dbThumbnail.UUID); + const oldDBVotes = await queryThumbnailVotesByUUID(oldDbThumbnail.UUID); + + assert.strictEqual(dbThumbnailTimestamps.timestamp, thumbnail.timestamp); + assert.strictEqual(dbThumbnail.original, thumbnail.original ? 1 : 0); + + assert.strictEqual(dbVotes.votes, 0); + assert.strictEqual(dbVotes.locked, 0); + assert.strictEqual(dbVotes.shadowHidden, 0); + + assert.strictEqual(oldDBVotes.votes, -1); + assert.strictEqual(oldDBVotes.locked, 0); + assert.strictEqual(oldDBVotes.shadowHidden, 0); + }); +}); \ No newline at end of file From 4d8ce40ef4e25a063d71f55831157ad50ff6293a Mon Sep 17 00:00:00 2001 From: Ajay Date: Fri, 27 Jan 2023 22:42:46 -0500 Subject: [PATCH 04/10] Add missing import --- src/routes/postBranding.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/routes/postBranding.ts b/src/routes/postBranding.ts index ddcd6ed..166dfd4 100644 --- a/src/routes/postBranding.ts +++ b/src/routes/postBranding.ts @@ -10,6 +10,7 @@ import { getIP } from "../utils/getIP"; import { getService } from "../utils/getService"; import { isUserVIP } from "../utils/isUserVIP"; import { Logger } from "../utils/logger"; +import crypto from "crypto"; enum BrandingType { Title, From 36f1d156054a002aa2dd7395091d98a711962ef8 Mon Sep 17 00:00:00 2001 From: Ajay Date: Sat, 28 Jan 2023 01:53:59 -0500 Subject: [PATCH 05/10] Add tests for get branding and fix issues Also improve deep partial equals --- src/routes/getBranding.ts | 63 +++++----- src/types/branding.model.ts | 10 +- test/cases/getBranding.ts | 217 ++++++++++++++++++++++++++++++++ test/utils/partialDeepEquals.ts | 32 +++-- 4 files changed, 274 insertions(+), 48 deletions(-) create mode 100644 test/cases/getBranding.ts diff --git a/src/routes/getBranding.ts b/src/routes/getBranding.ts index 3d8e686..0a8decf 100644 --- a/src/routes/getBranding.ts +++ b/src/routes/getBranding.ts @@ -2,13 +2,14 @@ import { Request, Response } from "express"; import { isEmpty } from "lodash"; import { config } from "../config"; import { db, privateDB } from "../databases/databases"; -import { BrandingDBSubmission, BrandingHashDBResult, BrandingHashResult, BrandingResult, ThumbnailDBResult, ThumbnailResult, TitleDBResult, TitleResult } from "../types/branding.model"; +import { BrandingDBSubmission, BrandingHashDBResult, BrandingResult, ThumbnailDBResult, ThumbnailResult, TitleDBResult, TitleResult } from "../types/branding.model"; import { HashedIP, IPAddress, Service, VideoID, VideoIDHash, Visibility } from "../types/segments.model"; import { shuffleArray } from "../utils/array"; import { getHashCache } from "../utils/getHashCache"; import { getIP } from "../utils/getIP"; import { getService } from "../utils/getService"; import { hashPrefixTester } from "../utils/hashPrefixTester"; +import { Logger } from "../utils/logger"; import { promiseOrTimeout } from "../utils/promise"; import { QueryCacher } from "../utils/queryCacher"; import { brandingHashKey, brandingIPKey, brandingKey } from "../utils/redisKeys"; @@ -21,7 +22,7 @@ enum BrandingSubmissionType { export async function getVideoBranding(videoID: VideoID, service: Service, ip: IPAddress): Promise { const getTitles = () => db.prepare( "all", - `SELECT "titles"."title", "titles"."original", "titleVotes"."votes", "titleVotes"."locked", "titleVotes"."shadowHidden", "title"."UUID", "title"."videoID", "title"."hashedVideoID + `SELECT "titles"."title", "titles"."original", "titleVotes"."votes", "titleVotes"."locked", "titleVotes"."shadowHidden", "titles"."UUID", "titles"."videoID", "titles"."hashedVideoID" FROM "titles" JOIN "titleVotes" ON "titles"."UUID" = "titleVotes"."UUID" WHERE "titles"."videoID" = ? AND "titles"."service" = ? AND "titleVotes"."votes" > -2`, [videoID, service], @@ -52,10 +53,10 @@ export async function getVideoBranding(videoID: VideoID, service: Service, ip: I return filterAndSortBranding(await branding.titles, await branding.thumbnails, ip, cache); } -export async function getVideoBrandingByHash(videoHashPrefix: VideoIDHash, service: Service, ip: IPAddress): Promise> { +export async function getVideoBrandingByHash(videoHashPrefix: VideoIDHash, service: Service, ip: IPAddress): Promise> { const getTitles = () => db.prepare( "all", - `SELECT "titles"."title", "titles"."original", "titleVotes"."votes", "titleVotes"."locked", "titleVotes"."shadowHidden", "title"."UUID", "title"."videoID", "title"."hashedVideoID + `SELECT "titles"."title", "titles"."original", "titleVotes"."votes", "titleVotes"."locked", "titleVotes"."shadowHidden", "titles"."UUID", "titles"."videoID", "titles"."hashedVideoID" FROM "titles" JOIN "titleVotes" ON "titles"."UUID" = "titleVotes"."UUID" WHERE "titles"."hashedVideoID" LIKE ? AND "titles"."service" = ? AND "titleVotes"."votes" > -2`, [`${videoHashPrefix}%`, service], @@ -81,20 +82,18 @@ export async function getVideoBrandingByHash(videoHashPrefix: VideoIDHash, servi const dbResult: Record = {}; const initResult = (submission: BrandingDBSubmission) => { dbResult[submission.videoID] = dbResult[submission.videoID] || { - branding: { - titles: [], - thumbnails: [] - } + titles: [], + thumbnails: [] }; }; (await branding.titles).map((title) => { initResult(title); - dbResult[title.videoID].branding.titles.push(title); + dbResult[title.videoID].titles.push(title); }); (await branding.thumbnails).map((thumbnail) => { initResult(thumbnail); - dbResult[thumbnail.videoID].branding.thumbnails.push(thumbnail); + dbResult[thumbnail.videoID].thumbnails.push(thumbnail); }); return dbResult; @@ -105,18 +104,16 @@ export async function getVideoBrandingByHash(videoHashPrefix: VideoIDHash, servi currentIP: null as Promise | null }; - const processedResult: Record = {}; + const processedResult: Record = {}; await Promise.all(Object.keys(branding).map(async (key) => { const castedKey = key as VideoID; - processedResult[castedKey] = { - branding: await filterAndSortBranding(branding[castedKey].branding.titles, branding[castedKey].branding.thumbnails, ip, cache) - }; + processedResult[castedKey] = await filterAndSortBranding(branding[castedKey].titles, branding[castedKey].thumbnails, ip, cache); })); return processedResult; } -async function filterAndSortBranding(dbTitles: TitleDBResult[], dbThumbnails: ThumbnailDBResult[], ip: IPAddress, cache: { currentIP: Promise | null}): Promise { +async function filterAndSortBranding(dbTitles: TitleDBResult[], dbThumbnails: ThumbnailDBResult[], ip: IPAddress, cache: { currentIP: Promise | null }): Promise { const shouldKeepTitles = shouldKeepSubmission(dbTitles, BrandingSubmissionType.Title, ip, cache); const shouldKeepThumbnails = shouldKeepSubmission(dbThumbnails, BrandingSubmissionType.Thumbnail, ip, cache); @@ -164,7 +161,7 @@ async function shouldKeepSubmission(submissions: BrandingDBSubmission[], type: B return submitterIP.hashedIP !== hashedIP; } catch (e) { // give up on shadow hide for now - return true; + return false; } })); @@ -173,35 +170,41 @@ async function shouldKeepSubmission(submissions: BrandingDBSubmission[], type: B export async function getBranding(req: Request, res: Response) { const videoID: VideoID = req.query.videoID as VideoID; - const service: Service = getService(req.query.service, req.body.service); + const service: Service = getService(req.query.service as string); if (!videoID) { return res.status(400).send("Missing parameter: videoID"); } const ip = getIP(req); - const result = await getVideoBranding(videoID, service, ip); + try { + const result = await getVideoBranding(videoID, service, ip); - const status = result.titles.length > 0 || result.thumbnails.length > 0 ? 200 : 404; - return res.status(status).json(result); + const status = result.titles.length > 0 || result.thumbnails.length > 0 ? 200 : 404; + return res.status(status).json(result); + } catch (e) { + Logger.error(e as string); + return res.status(500).send("Internal server error"); + } } export async function getBrandingByHashEndpoint(req: Request, res: Response) { let hashPrefix = req.params.prefix as VideoIDHash; - if (!req.params.prefix || !hashPrefixTester(req.params.prefix)) { + if (!hashPrefix || !hashPrefixTester(hashPrefix) || hashPrefix.length !== 4) { return res.status(400).send("Hash prefix does not match format requirements."); // Exit early on faulty prefix } hashPrefix = hashPrefix.toLowerCase() as VideoIDHash; - const service: Service = getService(req.query.service, req.body.service); - - if (!hashPrefix || hashPrefix.length !== 4) { - return res.status(400).send("Hash prefix does not match format requirements."); - } - + const service: Service = getService(req.query.service as string); const ip = getIP(req); - const result = await getVideoBrandingByHash(hashPrefix, service, ip); - const status = !isEmpty(result) ? 200 : 404; - return res.status(status).json(result); + try { + const result = await getVideoBrandingByHash(hashPrefix, service, ip); + + const status = !isEmpty(result) ? 200 : 404; + return res.status(status).json(result); + } catch (e) { + Logger.error(e as string); + return res.status(500).send([]); + } } \ No newline at end of file diff --git a/src/types/branding.model.ts b/src/types/branding.model.ts index 8662ad3..f5ac991 100644 --- a/src/types/branding.model.ts +++ b/src/types/branding.model.ts @@ -46,14 +46,8 @@ export interface BrandingResult { } export interface BrandingHashDBResult { - branding: { - titles: TitleDBResult[], - thumbnails: ThumbnailDBResult[] - }; -} - -export interface BrandingHashResult { - branding: BrandingResult; + titles: TitleDBResult[], + thumbnails: ThumbnailDBResult[] } export interface OriginalThumbnailSubmission { diff --git a/test/cases/getBranding.ts b/test/cases/getBranding.ts new file mode 100644 index 0000000..0cca74e --- /dev/null +++ b/test/cases/getBranding.ts @@ -0,0 +1,217 @@ +import { client } from "../utils/httpClient"; +import assert from "assert"; +import { getHash } from "../../src/utils/getHash"; +import { db } from "../../src/databases/databases"; +import { Service } from "../../src/types/segments.model"; +import { BrandingResult, BrandingUUID } from "../../src/types/branding.model"; +import { partialDeepEquals } from "../utils/partialDeepEquals"; + +describe("getBranding", () => { + const videoID1 = "videoID1"; + const videoID2Locked = "videoID2"; + const videoID2ShadowHide = "videoID3"; + const videoIDEmpty = "videoID4"; + + const videoID1Hash = getHash(videoID1, 1).slice(0, 4); + const videoID2LockedHash = getHash(videoID2Locked, 1).slice(0, 4); + const videoID2ShadowHideHash = getHash(videoID2ShadowHide, 1).slice(0, 4); + const videoIDEmptyHash = "aaaa"; + + const endpoint = "/api/branding"; + const getBranding = (params: Record) => client({ + method: "GET", + url: endpoint, + params + }); + + const getBrandingByHash = (hash: string, params: Record) => client({ + method: "GET", + url: `${endpoint}/${hash}`, + params + }); + + before(() => { + const titleQuery = `INSERT INTO "titles" ("videoID", "title", "original", "userID", "service", "hashedVideoID", "timeSubmitted", "UUID") VALUES (?, ?, ?, ?, ?, ?, ?, ?)`; + const titleVotesQuery = `INSERT INTO "titleVotes" ("UUID", "votes", "locked", "shadowHidden") VALUES (?, ?, ?, ?)`; + const thumbnailQuery = `INSERT INTO "thumbnails" ("videoID", "original", "userID", "service", "hashedVideoID", "timeSubmitted", "UUID") VALUES (?, ?, ?, ?, ?, ?, ?)`; + const thumbnailTimestampsQuery = `INSERT INTO "thumbnailTimestamps" ("UUID", "timestamp") VALUES (?, ?)`; + const thumbnailVotesQuery = `INSERT INTO "thumbnailVotes" ("UUID", "votes", "locked", "shadowHidden") VALUES (?, ?, ?, ?)`; + + db.prepare("run", titleQuery, [videoID1, "title1", 0, "userID1", Service.YouTube, videoID1Hash, 1, "UUID1"]); + db.prepare("run", titleQuery, [videoID1, "title2", 0, "userID2", Service.YouTube, videoID1Hash, 1, "UUID2"]); + db.prepare("run", titleQuery, [videoID1, "title3", 1, "userID3", Service.YouTube, videoID1Hash, 1, "UUID3"]); + db.prepare("run", titleVotesQuery, ["UUID1", 3, 0, 0]); + db.prepare("run", titleVotesQuery, ["UUID2", 2, 0, 0]); + db.prepare("run", titleVotesQuery, ["UUID3", 1, 0, 0]); + db.prepare("run", thumbnailQuery, [videoID1, 0, "userID1", Service.YouTube, videoID1Hash, 1, "UUID1T"]); + db.prepare("run", thumbnailQuery, [videoID1, 1, "userID2", Service.YouTube, videoID1Hash, 1, "UUID2T"]); + db.prepare("run", thumbnailQuery, [videoID1, 0, "userID3", Service.YouTube, videoID1Hash, 1, "UUID3T"]); + db.prepare("run", thumbnailTimestampsQuery, ["UUID1T", 1]); + db.prepare("run", thumbnailTimestampsQuery, ["UUID3T", 3]); + db.prepare("run", thumbnailVotesQuery, ["UUID1T", 3, 0, 0]); + db.prepare("run", thumbnailVotesQuery, ["UUID2T", 2, 0, 0]); + db.prepare("run", thumbnailVotesQuery, ["UUID3T", 1, 0, 0]); + + db.prepare("run", titleQuery, [videoID2Locked, "title1", 0, "userID1", Service.YouTube, videoID2LockedHash, 1, "UUID11"]); + db.prepare("run", titleQuery, [videoID2Locked, "title2", 0, "userID2", Service.YouTube, videoID2LockedHash, 1, "UUID21"]); + db.prepare("run", titleQuery, [videoID2Locked, "title3", 1, "userID3", Service.YouTube, videoID2LockedHash, 1, "UUID31"]); + db.prepare("run", titleVotesQuery, ["UUID11", 3, 0, 0]); + db.prepare("run", titleVotesQuery, ["UUID21", 2, 0, 0]); + db.prepare("run", titleVotesQuery, ["UUID31", 1, 1, 0]); + db.prepare("run", thumbnailQuery, [videoID2Locked, 0, "userID1", Service.YouTube, videoID2LockedHash, 1, "UUID11T"]); + db.prepare("run", thumbnailQuery, [videoID2Locked, 1, "userID2", Service.YouTube, videoID2LockedHash, 1, "UUID21T"]); + db.prepare("run", thumbnailQuery, [videoID2Locked, 0, "userID3", Service.YouTube, videoID2LockedHash, 1, "UUID31T"]); + db.prepare("run", thumbnailTimestampsQuery, ["UUID11T", 1]); + db.prepare("run", thumbnailTimestampsQuery, ["UUID31T", 3]); + db.prepare("run", thumbnailVotesQuery, ["UUID11T", 3, 0, 0]); + db.prepare("run", thumbnailVotesQuery, ["UUID21T", 2, 0, 0]); + db.prepare("run", thumbnailVotesQuery, ["UUID31T", 1, 1, 0]); + + db.prepare("run", titleQuery, [videoID2ShadowHide, "title1", 0, "userID1", Service.YouTube, videoID2ShadowHideHash, 1, "UUID12"]); + db.prepare("run", titleQuery, [videoID2ShadowHide, "title2", 0, "userID2", Service.YouTube, videoID2ShadowHideHash, 1, "UUID22"]); + db.prepare("run", titleQuery, [videoID2ShadowHide, "title3", 1, "userID3", Service.YouTube, videoID2ShadowHideHash, 1, "UUID32"]); + db.prepare("run", titleVotesQuery, ["UUID12", 3, 0, 0]); + db.prepare("run", titleVotesQuery, ["UUID22", 2, 0, 0]); + db.prepare("run", titleVotesQuery, ["UUID32", 1, 0, 1]); + db.prepare("run", thumbnailQuery, [videoID2ShadowHide, 0, "userID1", Service.YouTube, videoID2ShadowHideHash, 1, "UUID12T"]); + db.prepare("run", thumbnailQuery, [videoID2ShadowHide, 1, "userID2", Service.YouTube, videoID2ShadowHideHash, 1, "UUID22T"]); + db.prepare("run", thumbnailQuery, [videoID2ShadowHide, 0, "userID3", Service.YouTube, videoID2ShadowHideHash, 1, "UUID32T"]); + db.prepare("run", thumbnailTimestampsQuery, ["UUID12T", 1]); + db.prepare("run", thumbnailTimestampsQuery, ["UUID32T", 3]); + db.prepare("run", thumbnailVotesQuery, ["UUID12T", 3, 0, 0]); + db.prepare("run", thumbnailVotesQuery, ["UUID22T", 2, 0, 0]); + db.prepare("run", thumbnailVotesQuery, ["UUID32T", 1, 0, 1]); + }); + + it("should get top titles and thumbnails", async () => { + await checkVideo(videoID1, videoID1Hash, { + titles: [{ + title: "title1", + original: false, + votes: 3, + locked: false, + UUID: "UUID1" as BrandingUUID + }, { + title: "title2", + original: false, + votes: 2, + locked: false, + UUID: "UUID2" as BrandingUUID + }, { + title: "title3", + original: true, + votes: 1, + locked: false, + UUID: "UUID3" as BrandingUUID + }], + thumbnails: [{ + timestamp: 1, + original: false, + votes: 3, + locked: false, + UUID: "UUID1T" as BrandingUUID + }, { + original: true, + votes: 2, + locked: false, + UUID: "UUID2T" as BrandingUUID + }, { + timestamp: 3, + original: false, + votes: 1, + locked: false, + UUID: "UUID3T" as BrandingUUID + }] + }); + }); + + it("should get top titles and thumbnails prioritizing locks", async () => { + await checkVideo(videoID2Locked, videoID2LockedHash, { + titles: [{ + title: "title3", + original: true, + votes: 1, + locked: true, + UUID: "UUID31" as BrandingUUID + }, { + title: "title1", + original: false, + votes: 3, + locked: false, + UUID: "UUID11" as BrandingUUID + }, { + title: "title2", + original: false, + votes: 2, + locked: false, + UUID: "UUID21" as BrandingUUID + }], + thumbnails: [{ + timestamp: 3, + original: false, + votes: 1, + locked: true, + UUID: "UUID31T" as BrandingUUID + }, { + timestamp: 1, + original: false, + votes: 3, + locked: false, + UUID: "UUID11T" as BrandingUUID + }, { + original: true, + votes: 2, + locked: false, + UUID: "UUID21T" as BrandingUUID + }] + }); + }); + + it("should get top titles and hide shadow hidden", async () => { + await checkVideo(videoID2ShadowHide, videoID2ShadowHideHash, { + titles: [{ + title: "title1", + original: false, + votes: 3, + locked: false, + UUID: "UUID12" as BrandingUUID + }, { + title: "title2", + original: false, + votes: 2, + locked: false, + UUID: "UUID22" as BrandingUUID + }], + thumbnails: [{ + timestamp: 1, + original: false, + votes: 3, + locked: false, + UUID: "UUID12T" as BrandingUUID + }, { + original: true, + votes: 2, + locked: false, + UUID: "UUID22T" as BrandingUUID + }] + }); + }); + + it("should get 404 when nothing", async () => { + const result1 = await getBranding({ videoID: videoIDEmpty }); + const result2 = await getBrandingByHash(videoIDEmptyHash, {}); + + assert.strictEqual(result1.status, 404); + assert.strictEqual(result2.status, 404); + }); + + async function checkVideo(videoID: string, videoIDHash: string, expected: BrandingResult) { + const result1 = await getBranding({ videoID }); + const result2 = await getBrandingByHash(videoIDHash, {}); + + assert.strictEqual(result1.status, 200); + assert.strictEqual(result2.status, 200); + assert.deepEqual(result1.data, result2.data[videoID]); + assert.ok(partialDeepEquals(result1.data, expected)); + } +}); diff --git a/test/utils/partialDeepEquals.ts b/test/utils/partialDeepEquals.ts index b2832f6..1077da5 100644 --- a/test/utils/partialDeepEquals.ts +++ b/test/utils/partialDeepEquals.ts @@ -1,21 +1,22 @@ import { Logger } from "../../src/utils/logger"; -function printActualExpected(actual: Record, expected: Record): void { +function printActualExpected(actual: Record, expected: Record, failedKey: string): void { Logger.error(`Actual: ${JSON.stringify(actual)}`); Logger.error(`Expected: ${JSON.stringify(expected)}`); + Logger.error(`Failed on key: ${failedKey}`); } export const partialDeepEquals = (actual: Record, expected: Record, print = true): boolean => { // loop over key, value of expected - for (const [ key, value ] of Object.entries(expected)) { + for (const [key, value] of Object.entries(expected)) { // if value is object or array, recurse if (Array.isArray(value) || typeof value === "object") { if (!partialDeepEquals(actual?.[key], value, false)) { - if (print) printActualExpected(actual, expected); + if (print) printActualExpected(actual, expected, key); return false; } } else if (actual?.[key] !== value) { - if (print) printActualExpected(actual, expected); + if (print) printActualExpected(actual, expected, key); return false; } } @@ -31,28 +32,39 @@ export const arrayPartialDeepEquals = (actual: Array, expected: Array) export const arrayDeepEquals = (actual: Record, expected: Record, print = true): boolean => { if (actual.length !== expected.length) return false; let flag = true; + let failedKey = ""; const actualString = JSON.stringify(actual); const expectedString = JSON.stringify(expected); // check every value in arr1 for match in arr2 - actual.every((value: any) => { if (flag && !expectedString.includes(JSON.stringify(value))) flag = false; }); + actual.every((value: any) => { + if (flag && !expectedString.includes(JSON.stringify(value))) { + flag = false; + failedKey = value; + } + }); // check arr2 for match in arr1 - expected.every((value: any) => { if (flag && !actualString.includes(JSON.stringify(value))) flag = false; }); + expected.every((value: any) => { + if (flag && !actualString.includes(JSON.stringify(value))) { + flag = false; + failedKey = value; + } + }); - if (!flag && print) printActualExpected(actual, expected); + if (!flag && print) printActualExpected(actual, expected, failedKey); return flag; }; export const mixedDeepEquals = (actual: Record, expected: Record, print = true): boolean => { - for (const [ key, value ] of Object.entries(expected)) { + for (const [key, value] of Object.entries(expected)) { // if value is object or array, recurse if (Array.isArray(value)) { if (!arrayDeepEquals(actual?.[key], value, false)) { - if (print) printActualExpected(actual, expected); + if (print) printActualExpected(actual, expected, key); return false; } } else if (actual?.[key] !== value) { - if (print) printActualExpected(actual, expected); + if (print) printActualExpected(actual, expected, key); return false; } } From b91114165e1028ef8a32fe0fb0dc5e373889a7de Mon Sep 17 00:00:00 2001 From: Ajay Date: Sat, 28 Jan 2023 02:12:33 -0500 Subject: [PATCH 06/10] use seperate queries --- src/databases/Sqlite.ts | 15 --------------- src/routes/postBranding.ts | 28 +++++++++++++++++----------- 2 files changed, 17 insertions(+), 26 deletions(-) diff --git a/src/databases/Sqlite.ts b/src/databases/Sqlite.ts index 2e61dfa..ac03753 100644 --- a/src/databases/Sqlite.ts +++ b/src/databases/Sqlite.ts @@ -14,21 +14,6 @@ export class Sqlite implements IDatabase { // eslint-disable-next-line require-await async prepare(type: QueryType, query: string, params: any[] = []): Promise { - if (query.includes(";")) { - const promises = []; - let paramsCount = 0; - for (const q of query.split(";")) { - if (q.trim() !== "") { - const currentParamCount = q.match(/\?/g)?.length ?? 0; - promises.push(this.prepare(type, q, params.slice(paramsCount, paramsCount + currentParamCount))); - - paramsCount += currentParamCount; - } - } - - return (await Promise.all(promises)).flat(); - } - // Logger.debug(`prepare (sqlite): type: ${type}, query: ${query}, params: ${params}`); const preparedQuery = this.db.prepare(Sqlite.processQuery(query)); diff --git a/src/routes/postBranding.ts b/src/routes/postBranding.ts index 166dfd4..648cb43 100644 --- a/src/routes/postBranding.ts +++ b/src/routes/postBranding.ts @@ -29,8 +29,8 @@ export async function postBranding(req: Request, res: Response) { if (!videoID || !userID || userID.length < 30 || !service || ((!title || !title.title) - && (!thumbnail || thumbnail.original == null - || (!thumbnail.original && !(thumbnail as TimeThumbnailSubmission).timestamp)))) { + && (!thumbnail || thumbnail.original == null + || (!thumbnail.original && !(thumbnail as TimeThumbnailSubmission).timestamp)))) { res.status(400).send("Bad Request"); return; } @@ -53,9 +53,11 @@ export async function postBranding(req: Request, res: Response) { if (existingUUID) { await updateVoteTotals(BrandingType.Title, existingVote, UUID, isVip); } else { - await db.prepare("run", `INSERT INTO "titles" ("videoID", "title", "original", "userID", "service", "hashedVideoID", "timeSubmitted", "UUID") VALUES (?, ?, ?, ?, ?, ?, ?, ?); - INSERT INTO "titleVotes" ("UUID", "votes", "locked", "shadowHidden") VALUES (?, 0, ?, 0);`, - [videoID, title.title, title.original ? 1 : 0, hashedUserID, service, hashedVideoID, now, UUID, UUID, isVip ? 1 : 0]); + await db.prepare("run", `INSERT INTO "titles" ("videoID", "title", "original", "userID", "service", "hashedVideoID", "timeSubmitted", "UUID") VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + [videoID, title.title, title.original ? 1 : 0, hashedUserID, service, hashedVideoID, now, UUID]); + + await db.prepare("run", `INSERT INTO "titleVotes" ("UUID", "votes", "locked", "shadowHidden") VALUES (?, 0, ?, 0);`, + [UUID, isVip ? 1 : 0]); } } })(), (async () => { @@ -70,13 +72,17 @@ export async function postBranding(req: Request, res: Response) { if (existingUUID) { await updateVoteTotals(BrandingType.Thumbnail, existingVote, UUID, isVip); } else { - await db.prepare("run", `INSERT INTO "thumbnails" ("videoID", "original", "userID", "service", "hashedVideoID", "timeSubmitted", "UUID") VALUES (?, ?, ?, ?, ?, ?, ?); - INSERT INTO "thumbnailVotes" ("UUID", "votes", "locked", "shadowHidden") VALUES (?, 0, ?, 0); - ${thumbnail.original ? "" : `INSERT INTO "thumbnailTimestamps" ("UUID", "timestamp") VALUES (?, ?)`}`, - [videoID, thumbnail.original ? 1 : 0, hashedUserID, service, hashedVideoID, now, UUID, UUID, - isVip ? 1 : 0, thumbnail.original ? null : UUID, thumbnail.original ? null : (thumbnail as TimeThumbnailSubmission).timestamp]); - } + await db.prepare("run", `INSERT INTO "thumbnails" ("videoID", "original", "userID", "service", "hashedVideoID", "timeSubmitted", "UUID") VALUES (?, ?, ?, ?, ?, ?, ?)`, + [videoID, thumbnail.original ? 1 : 0, hashedUserID, service, hashedVideoID, now, UUID]); + await db.prepare("run", `INSERT INTO "thumbnailVotes" ("UUID", "votes", "locked", "shadowHidden") VALUES (?, 0, ?, 0)`, + [UUID, isVip ? 1 : 0]); + + if (!thumbnail.original) { + await db.prepare("run", `INSERT INTO "thumbnailTimestamps" ("UUID", "timestamp") VALUES (?, ?)`, + [UUID, (thumbnail as TimeThumbnailSubmission).timestamp]); + } + } } })()]); From 10e9aef8cc547e632f3064d35667f0eb1e1f3897 Mon Sep 17 00:00:00 2001 From: Ajay Date: Sat, 28 Jan 2023 02:20:16 -0500 Subject: [PATCH 07/10] use await in get branding test --- test/cases/getBranding.ts | 102 ++++++++++++++++++++++---------------- 1 file changed, 59 insertions(+), 43 deletions(-) diff --git a/test/cases/getBranding.ts b/test/cases/getBranding.ts index 0cca74e..dbd2894 100644 --- a/test/cases/getBranding.ts +++ b/test/cases/getBranding.ts @@ -30,57 +30,73 @@ describe("getBranding", () => { params }); - before(() => { + before(async () => { const titleQuery = `INSERT INTO "titles" ("videoID", "title", "original", "userID", "service", "hashedVideoID", "timeSubmitted", "UUID") VALUES (?, ?, ?, ?, ?, ?, ?, ?)`; const titleVotesQuery = `INSERT INTO "titleVotes" ("UUID", "votes", "locked", "shadowHidden") VALUES (?, ?, ?, ?)`; const thumbnailQuery = `INSERT INTO "thumbnails" ("videoID", "original", "userID", "service", "hashedVideoID", "timeSubmitted", "UUID") VALUES (?, ?, ?, ?, ?, ?, ?)`; const thumbnailTimestampsQuery = `INSERT INTO "thumbnailTimestamps" ("UUID", "timestamp") VALUES (?, ?)`; const thumbnailVotesQuery = `INSERT INTO "thumbnailVotes" ("UUID", "votes", "locked", "shadowHidden") VALUES (?, ?, ?, ?)`; - db.prepare("run", titleQuery, [videoID1, "title1", 0, "userID1", Service.YouTube, videoID1Hash, 1, "UUID1"]); - db.prepare("run", titleQuery, [videoID1, "title2", 0, "userID2", Service.YouTube, videoID1Hash, 1, "UUID2"]); - db.prepare("run", titleQuery, [videoID1, "title3", 1, "userID3", Service.YouTube, videoID1Hash, 1, "UUID3"]); - db.prepare("run", titleVotesQuery, ["UUID1", 3, 0, 0]); - db.prepare("run", titleVotesQuery, ["UUID2", 2, 0, 0]); - db.prepare("run", titleVotesQuery, ["UUID3", 1, 0, 0]); - db.prepare("run", thumbnailQuery, [videoID1, 0, "userID1", Service.YouTube, videoID1Hash, 1, "UUID1T"]); - db.prepare("run", thumbnailQuery, [videoID1, 1, "userID2", Service.YouTube, videoID1Hash, 1, "UUID2T"]); - db.prepare("run", thumbnailQuery, [videoID1, 0, "userID3", Service.YouTube, videoID1Hash, 1, "UUID3T"]); - db.prepare("run", thumbnailTimestampsQuery, ["UUID1T", 1]); - db.prepare("run", thumbnailTimestampsQuery, ["UUID3T", 3]); - db.prepare("run", thumbnailVotesQuery, ["UUID1T", 3, 0, 0]); - db.prepare("run", thumbnailVotesQuery, ["UUID2T", 2, 0, 0]); - db.prepare("run", thumbnailVotesQuery, ["UUID3T", 1, 0, 0]); + await Promise.all([ + db.prepare("run", titleQuery, [videoID1, "title1", 0, "userID1", Service.YouTube, videoID1Hash, 1, "UUID1"]), + db.prepare("run", titleQuery, [videoID1, "title2", 0, "userID2", Service.YouTube, videoID1Hash, 1, "UUID2"]), + db.prepare("run", titleQuery, [videoID1, "title3", 1, "userID3", Service.YouTube, videoID1Hash, 1, "UUID3"]), + db.prepare("run", thumbnailQuery, [videoID1, 0, "userID1", Service.YouTube, videoID1Hash, 1, "UUID1T"]), + db.prepare("run", thumbnailQuery, [videoID1, 1, "userID2", Service.YouTube, videoID1Hash, 1, "UUID2T"]), + db.prepare("run", thumbnailQuery, [videoID1, 0, "userID3", Service.YouTube, videoID1Hash, 1, "UUID3T"]), + ]); - db.prepare("run", titleQuery, [videoID2Locked, "title1", 0, "userID1", Service.YouTube, videoID2LockedHash, 1, "UUID11"]); - db.prepare("run", titleQuery, [videoID2Locked, "title2", 0, "userID2", Service.YouTube, videoID2LockedHash, 1, "UUID21"]); - db.prepare("run", titleQuery, [videoID2Locked, "title3", 1, "userID3", Service.YouTube, videoID2LockedHash, 1, "UUID31"]); - db.prepare("run", titleVotesQuery, ["UUID11", 3, 0, 0]); - db.prepare("run", titleVotesQuery, ["UUID21", 2, 0, 0]); - db.prepare("run", titleVotesQuery, ["UUID31", 1, 1, 0]); - db.prepare("run", thumbnailQuery, [videoID2Locked, 0, "userID1", Service.YouTube, videoID2LockedHash, 1, "UUID11T"]); - db.prepare("run", thumbnailQuery, [videoID2Locked, 1, "userID2", Service.YouTube, videoID2LockedHash, 1, "UUID21T"]); - db.prepare("run", thumbnailQuery, [videoID2Locked, 0, "userID3", Service.YouTube, videoID2LockedHash, 1, "UUID31T"]); - db.prepare("run", thumbnailTimestampsQuery, ["UUID11T", 1]); - db.prepare("run", thumbnailTimestampsQuery, ["UUID31T", 3]); - db.prepare("run", thumbnailVotesQuery, ["UUID11T", 3, 0, 0]); - db.prepare("run", thumbnailVotesQuery, ["UUID21T", 2, 0, 0]); - db.prepare("run", thumbnailVotesQuery, ["UUID31T", 1, 1, 0]); + await Promise.all([ + db.prepare("run", titleVotesQuery, ["UUID1", 3, 0, 0]), + db.prepare("run", titleVotesQuery, ["UUID2", 2, 0, 0]), + db.prepare("run", titleVotesQuery, ["UUID3", 1, 0, 0]), + db.prepare("run", thumbnailTimestampsQuery, ["UUID1T", 1]), + db.prepare("run", thumbnailTimestampsQuery, ["UUID3T", 3]), + db.prepare("run", thumbnailVotesQuery, ["UUID1T", 3, 0, 0]), + db.prepare("run", thumbnailVotesQuery, ["UUID2T", 2, 0, 0]), + db.prepare("run", thumbnailVotesQuery, ["UUID3T", 1, 0, 0]) + ]); - db.prepare("run", titleQuery, [videoID2ShadowHide, "title1", 0, "userID1", Service.YouTube, videoID2ShadowHideHash, 1, "UUID12"]); - db.prepare("run", titleQuery, [videoID2ShadowHide, "title2", 0, "userID2", Service.YouTube, videoID2ShadowHideHash, 1, "UUID22"]); - db.prepare("run", titleQuery, [videoID2ShadowHide, "title3", 1, "userID3", Service.YouTube, videoID2ShadowHideHash, 1, "UUID32"]); - db.prepare("run", titleVotesQuery, ["UUID12", 3, 0, 0]); - db.prepare("run", titleVotesQuery, ["UUID22", 2, 0, 0]); - db.prepare("run", titleVotesQuery, ["UUID32", 1, 0, 1]); - db.prepare("run", thumbnailQuery, [videoID2ShadowHide, 0, "userID1", Service.YouTube, videoID2ShadowHideHash, 1, "UUID12T"]); - db.prepare("run", thumbnailQuery, [videoID2ShadowHide, 1, "userID2", Service.YouTube, videoID2ShadowHideHash, 1, "UUID22T"]); - db.prepare("run", thumbnailQuery, [videoID2ShadowHide, 0, "userID3", Service.YouTube, videoID2ShadowHideHash, 1, "UUID32T"]); - db.prepare("run", thumbnailTimestampsQuery, ["UUID12T", 1]); - db.prepare("run", thumbnailTimestampsQuery, ["UUID32T", 3]); - db.prepare("run", thumbnailVotesQuery, ["UUID12T", 3, 0, 0]); - db.prepare("run", thumbnailVotesQuery, ["UUID22T", 2, 0, 0]); - db.prepare("run", thumbnailVotesQuery, ["UUID32T", 1, 0, 1]); + await Promise.all([ + db.prepare("run", titleQuery, [videoID2Locked, "title1", 0, "userID1", Service.YouTube, videoID2LockedHash, 1, "UUID11"]), + db.prepare("run", titleQuery, [videoID2Locked, "title2", 0, "userID2", Service.YouTube, videoID2LockedHash, 1, "UUID21"]), + db.prepare("run", titleQuery, [videoID2Locked, "title3", 1, "userID3", Service.YouTube, videoID2LockedHash, 1, "UUID31"]), + db.prepare("run", thumbnailQuery, [videoID2Locked, 0, "userID1", Service.YouTube, videoID2LockedHash, 1, "UUID11T"]), + db.prepare("run", thumbnailQuery, [videoID2Locked, 1, "userID2", Service.YouTube, videoID2LockedHash, 1, "UUID21T"]), + db.prepare("run", thumbnailQuery, [videoID2Locked, 0, "userID3", Service.YouTube, videoID2LockedHash, 1, "UUID31T"]) + ]); + + await Promise.all([ + db.prepare("run", titleVotesQuery, ["UUID11", 3, 0, 0]), + db.prepare("run", titleVotesQuery, ["UUID21", 2, 0, 0]), + db.prepare("run", titleVotesQuery, ["UUID31", 1, 1, 0]), + + db.prepare("run", thumbnailTimestampsQuery, ["UUID11T", 1]), + db.prepare("run", thumbnailTimestampsQuery, ["UUID31T", 3]), + db.prepare("run", thumbnailVotesQuery, ["UUID11T", 3, 0, 0]), + db.prepare("run", thumbnailVotesQuery, ["UUID21T", 2, 0, 0]), + db.prepare("run", thumbnailVotesQuery, ["UUID31T", 1, 1, 0]), + ]); + + await Promise.all([ + db.prepare("run", titleQuery, [videoID2ShadowHide, "title1", 0, "userID1", Service.YouTube, videoID2ShadowHideHash, 1, "UUID12"]), + db.prepare("run", titleQuery, [videoID2ShadowHide, "title2", 0, "userID2", Service.YouTube, videoID2ShadowHideHash, 1, "UUID22"]), + db.prepare("run", titleQuery, [videoID2ShadowHide, "title3", 1, "userID3", Service.YouTube, videoID2ShadowHideHash, 1, "UUID32"]), + db.prepare("run", thumbnailQuery, [videoID2ShadowHide, 0, "userID1", Service.YouTube, videoID2ShadowHideHash, 1, "UUID12T"]), + db.prepare("run", thumbnailQuery, [videoID2ShadowHide, 1, "userID2", Service.YouTube, videoID2ShadowHideHash, 1, "UUID22T"]), + db.prepare("run", thumbnailQuery, [videoID2ShadowHide, 0, "userID3", Service.YouTube, videoID2ShadowHideHash, 1, "UUID32T"]) + ]); + + await Promise.all([ + db.prepare("run", titleVotesQuery, ["UUID12", 3, 0, 0]), + db.prepare("run", titleVotesQuery, ["UUID22", 2, 0, 0]), + db.prepare("run", titleVotesQuery, ["UUID32", 1, 0, 1]), + db.prepare("run", thumbnailTimestampsQuery, ["UUID12T", 1]), + db.prepare("run", thumbnailTimestampsQuery, ["UUID32T", 3]), + db.prepare("run", thumbnailVotesQuery, ["UUID12T", 3, 0, 0]), + db.prepare("run", thumbnailVotesQuery, ["UUID22T", 2, 0, 0]), + db.prepare("run", thumbnailVotesQuery, ["UUID32T", 1, 0, 1]) + ]); }); it("should get top titles and thumbnails", async () => { From 9cf2e1f0e9529e043b2f3a65971e5d88cde8f8ab Mon Sep 17 00:00:00 2001 From: Ajay Date: Sat, 28 Jan 2023 02:31:49 -0500 Subject: [PATCH 08/10] Fix private db table removal and original title type --- databases/_sponsorTimes.db.sql | 2 +- src/routes/getBranding.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/databases/_sponsorTimes.db.sql b/databases/_sponsorTimes.db.sql index 8a5daf7..8a2818d 100644 --- a/databases/_sponsorTimes.db.sql +++ b/databases/_sponsorTimes.db.sql @@ -40,7 +40,7 @@ CREATE TABLE IF NOT EXISTS "config" ( CREATE TABLE IF NOT EXISTS "titles" ( "videoID" TEXT NOT NULL, "title" TEXT NOT NULL, - "original" BOOLEAN NOT NULL, + "original" INTEGER default 0, "userID" TEXT NOT NULL, "service" TEXT NOT NULL, "hashedVideoID" TEXT NOT NULL, diff --git a/src/routes/getBranding.ts b/src/routes/getBranding.ts index 0a8decf..54fc557 100644 --- a/src/routes/getBranding.ts +++ b/src/routes/getBranding.ts @@ -150,7 +150,7 @@ async function shouldKeepSubmission(submissions: BrandingDBSubmission[], type: B const shouldKeep = await Promise.all(submissions.map(async (s) => { if (s.shadowHidden != Visibility.HIDDEN) return true; - const table = type === BrandingSubmissionType.Title ? "titles" : "thumbnails"; + const table = type === BrandingSubmissionType.Title ? "titleVotes" : "thumbnailVotes"; const fetchData = () => privateDB.prepare("get", `SELECT "hashedIP" FROM "${table}" WHERE "UUID" = ?`, [s.UUID], { useReplica: true }) as Promise<{ hashedIP: HashedIP }>; try { From 46c372a76431ec19a18f5852469fbebd7bf7d8e1 Mon Sep 17 00:00:00 2001 From: Ajay Date: Tue, 14 Feb 2023 22:10:46 -0500 Subject: [PATCH 09/10] Unlock other segments when vip votes for new segment --- src/routes/postBranding.ts | 10 +++++++ test/cases/postBranding.ts | 57 ++++++++++++++++++++++++++++++++++---- 2 files changed, 62 insertions(+), 5 deletions(-) diff --git a/src/routes/postBranding.ts b/src/routes/postBranding.ts index 648cb43..ba5a159 100644 --- a/src/routes/postBranding.ts +++ b/src/routes/postBranding.ts @@ -59,6 +59,11 @@ export async function postBranding(req: Request, res: Response) { await db.prepare("run", `INSERT INTO "titleVotes" ("UUID", "votes", "locked", "shadowHidden") VALUES (?, 0, ?, 0);`, [UUID, isVip ? 1 : 0]); } + + if (isVip) { + // unlock all other titles + await db.prepare("run", `UPDATE "titleVotes" SET "locked" = 0 WHERE "UUID" != ?`, [UUID]); + } } })(), (async () => { if (thumbnail) { @@ -82,6 +87,11 @@ export async function postBranding(req: Request, res: Response) { await db.prepare("run", `INSERT INTO "thumbnailTimestamps" ("UUID", "timestamp") VALUES (?, ?)`, [UUID, (thumbnail as TimeThumbnailSubmission).timestamp]); } + + if (isVip) { + // unlock all other titles + await db.prepare("run", `UPDATE "thumbnailVotes" SET "locked" = 0 WHERE "UUID" != ?`, [UUID]); + } } } })()]); diff --git a/test/cases/postBranding.ts b/test/cases/postBranding.ts index ccc15f1..36b878a 100644 --- a/test/cases/postBranding.ts +++ b/test/cases/postBranding.ts @@ -7,6 +7,7 @@ import { Service } from "../../src/types/segments.model"; describe("postBranding", () => { const vipUser = `VIPPostBrandingUser${".".repeat(16)}`; + const vipUser2 = `VIPPostBrandingUser2${".".repeat(16)}`; const userID1 = `PostBrandingUser1${".".repeat(16)}`; const userID2 = `PostBrandingUser2${".".repeat(16)}`; const userID3 = `PostBrandingUser3${".".repeat(16)}`; @@ -20,15 +21,16 @@ describe("postBranding", () => { data }); - const queryTitleByVideo = (videoID: string) => db.prepare("get", `SELECT * FROM "titles" WHERE "videoID" = ? ORDER BY "timeSubmitted" DESC`, [videoID]); - const queryThumbnailByVideo = (videoID: string) => db.prepare("get", `SELECT * FROM "thumbnails" WHERE "videoID" = ? ORDER BY "timeSubmitted" DESC`, [videoID]); - const queryThumbnailTimestampsByUUID = (UUID: string) => db.prepare("get", `SELECT * FROM "thumbnailTimestamps" WHERE "UUID" = ?`, [UUID]); - const queryTitleVotesByUUID = (UUID: string) => db.prepare("get", `SELECT * FROM "titleVotes" WHERE "UUID" = ?`, [UUID]); - const queryThumbnailVotesByUUID = (UUID: string) => db.prepare("get", `SELECT * FROM "thumbnailVotes" WHERE "UUID" = ?`, [UUID]); + const queryTitleByVideo = (videoID: string, all = false) => db.prepare(all ? "all" : "get", `SELECT * FROM "titles" WHERE "videoID" = ? ORDER BY "timeSubmitted" DESC`, [videoID]); + const queryThumbnailByVideo = (videoID: string, all = false) => db.prepare(all ? "all" : "get", `SELECT * FROM "thumbnails" WHERE "videoID" = ? ORDER BY "timeSubmitted" DESC`, [videoID]); + const queryThumbnailTimestampsByUUID = (UUID: string, all = false) => db.prepare(all ? "all" : "get", `SELECT * FROM "thumbnailTimestamps" WHERE "UUID" = ?`, [UUID]); + const queryTitleVotesByUUID = (UUID: string, all = false) => db.prepare(all ? "all" : "get", `SELECT * FROM "titleVotes" WHERE "UUID" = ?`, [UUID]); + const queryThumbnailVotesByUUID = (UUID: string, all = false) => db.prepare(all ? "all" : "get", `SELECT * FROM "thumbnailVotes" WHERE "UUID" = ?`, [UUID]); before(() => { const insertVipUserQuery = 'INSERT INTO "vipUsers" ("userID") VALUES (?)'; db.prepare("run", insertVipUserQuery, [getHash(vipUser)]); + db.prepare("run", insertVipUserQuery, [getHash(vipUser2)]); }); it("Submit only title", async () => { @@ -216,6 +218,51 @@ describe("postBranding", () => { assert.strictEqual(dbThumbnailVotes.shadowHidden, 0); }); + it("Submit another title and thumbnail as VIP unlocks others", async () => { + const videoID = "postBrand6"; + const title = { + title: "Some other title", + original: false + }; + const thumbnail = { + timestamp: 15.42, + original: false + }; + + const res = await postBranding({ + title, + thumbnail, + userID: vipUser2, + service: Service.YouTube, + videoID + }); + + assert.strictEqual(res.status, 200); + const dbTitles = await queryTitleByVideo(videoID, true); + const dbTitleVotes = await queryTitleVotesByUUID(dbTitles[0].UUID); + const dbTitleVotesOld = await queryTitleVotesByUUID(dbTitles[1].UUID); + const dbThumbnails = await queryThumbnailByVideo(videoID, true); + const dbThumbnailTimestamps = await queryThumbnailTimestampsByUUID(dbThumbnails[0].UUID); + const dbThumbnailVotes = await queryThumbnailVotesByUUID(dbThumbnails[0].UUID); + const dbThumbnailVotesOld = await queryThumbnailVotesByUUID(dbThumbnails[1].UUID); + + assert.strictEqual(dbTitles[0].title, title.title); + assert.strictEqual(dbTitles[0].original, title.original ? 1 : 0); + + assert.strictEqual(dbTitleVotes.votes, 0); + assert.strictEqual(dbTitleVotes.locked, 1); + assert.strictEqual(dbTitleVotes.shadowHidden, 0); + assert.strictEqual(dbTitleVotesOld.locked, 0); + + assert.strictEqual(dbThumbnailTimestamps.timestamp, thumbnail.timestamp); + assert.strictEqual(dbThumbnails[0].original, thumbnail.original ? 1 : 0); + + assert.strictEqual(dbThumbnailVotes.votes, 0); + assert.strictEqual(dbThumbnailVotes.locked, 1); + assert.strictEqual(dbThumbnailVotes.shadowHidden, 0); + assert.strictEqual(dbThumbnailVotesOld.locked, 0); + }); + it("Vote the same title again", async () => { const videoID = "postBrand1"; const title = { From 19ebca86c9191d5496b4cbc5c26fbc4347f20f87 Mon Sep 17 00:00:00 2001 From: Ajay Date: Sat, 18 Feb 2023 02:33:22 -0500 Subject: [PATCH 10/10] Fix video branding not being awaited --- src/routes/getBranding.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/routes/getBranding.ts b/src/routes/getBranding.ts index 54fc557..fff37dd 100644 --- a/src/routes/getBranding.ts +++ b/src/routes/getBranding.ts @@ -38,10 +38,9 @@ export async function getVideoBranding(videoID: VideoID, service: Service, ip: I { useReplica: true } ) as Promise; - // eslint-disable-next-line require-await const getBranding = async () => ({ - titles: getTitles(), - thumbnails: getThumbnails() + titles: await getTitles(), + thumbnails: await getThumbnails() }); const branding = await QueryCacher.get(getBranding, brandingKey(videoID, service)); @@ -50,7 +49,7 @@ export async function getVideoBranding(videoID: VideoID, service: Service, ip: I currentIP: null as Promise | null }; - return filterAndSortBranding(await branding.titles, await branding.thumbnails, ip, cache); + return filterAndSortBranding(branding.titles, branding.thumbnails, ip, cache); } export async function getVideoBrandingByHash(videoHashPrefix: VideoIDHash, service: Service, ip: IPAddress): Promise> {