From bc6db0d1090b642f557ac3a7413a39c4fa68b2a4 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 14 Nov 2021 23:09:12 -0500 Subject: [PATCH] Add endpoints for rating endpoint (dislikes) https://github.com/ajayyy/SponsorBlock/issues/1039 --- .eslintrc.js | 2 +- DatabaseSchema.md | 33 ++++++++- databases/_private_indexes.sql | 7 ++ databases/_sponsorTimes_indexes.sql | 17 +++++ databases/_upgrade_private_4.sql | 14 ++++ databases/_upgrade_sponsorTimes_28.sql | 13 ++++ src/app.ts | 8 +++ src/config.ts | 6 ++ src/routes/ratings/getRating.ts | 82 ++++++++++++++++++++++ src/routes/ratings/postRating.ts | 62 +++++++++++++++++ src/types/config.model.ts | 1 + src/types/ratings.model.ts | 6 ++ src/utils/getIP.ts | 2 +- src/utils/redisKeys.ts | 7 ++ test/cases/ratings/getRating.ts | 49 +++++++++++++ test/cases/ratings/postRating.ts | 96 ++++++++++++++++++++++++++ test/test.ts | 24 ++++--- 17 files changed, 415 insertions(+), 14 deletions(-) create mode 100644 databases/_upgrade_private_4.sql create mode 100644 databases/_upgrade_sponsorTimes_28.sql create mode 100644 src/routes/ratings/getRating.ts create mode 100644 src/routes/ratings/postRating.ts create mode 100644 src/types/ratings.model.ts create mode 100644 test/cases/ratings/getRating.ts create mode 100644 test/cases/ratings/postRating.ts diff --git a/.eslintrc.js b/.eslintrc.js index e96d7da..c082e3d 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -27,6 +27,6 @@ module.exports = { "indent": ["warn", 4, { "SwitchCase": 1 }], "object-curly-spacing": ["warn", "always"], "require-await": "warn", - "no-console": "error" + "no-console": "warn" }, }; diff --git a/DatabaseSchema.md b/DatabaseSchema.md index a9f4344..bcd9949 100644 --- a/DatabaseSchema.md +++ b/DatabaseSchema.md @@ -185,6 +185,22 @@ | hashedVideoID | TEXT | not null, default '', sha256 | | userAgent | TEXT | not null, default '' | +### ratings + +| Name | Type | | +| -- | :--: | -- | +| videoID | TEXT | not null | +| service | TEXT | not null, default 'YouTube' | +| type | INTEGER | not null | +| count | INTEGER | not null | +| hashedVideoID | TEXT | not null | + +| index | field | +| -- | :--: | +| ratings_hashedVideoID_gin | hashedVideoID | +| ratings_hashedVideoID | hashedVideoID, service | +| ratings_videoID | videoID, service | + # Private [vote](#vote) @@ -238,4 +254,19 @@ | Name | Type | | | -- | :--: | -- | | key | TEXT | not null | -| value | TEXT | not null | \ No newline at end of file +| value | TEXT | not null | + +### ratings + +| Name | Type | | +| -- | :--: | -- | +| videoID | TEXT | not null | +| service | TEXT | not null, default 'YouTube' | +| userID | TEXT | not null | +| type | INTEGER | not null | +| timeSubmitted | INTEGER | not null | +| hashedIP | TEXT | not null | + +| index | field | +| -- | :--: | +| ratings_videoID | videoID, service, userID, timeSubmitted | diff --git a/databases/_private_indexes.sql b/databases/_private_indexes.sql index 48ed490..7ecb3a0 100644 --- a/databases/_private_indexes.sql +++ b/databases/_private_indexes.sql @@ -22,4 +22,11 @@ CREATE INDEX IF NOT EXISTS "votes_userID" CREATE INDEX IF NOT EXISTS "categoryVotes_UUID" ON public."categoryVotes" USING btree ("UUID" COLLATE pg_catalog."default" ASC NULLS LAST, "userID" COLLATE pg_catalog."default" ASC NULLS LAST, "hashedIP" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; + +-- ratings + +CREATE INDEX IF NOT EXISTS "ratings_videoID" + ON public."ratings" USING btree + ("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST, "userID" COLLATE pg_catalog."default" ASC NULLS LAST, "timeSubmitted" ASC NULLS LAST) TABLESPACE pg_default; \ No newline at end of file diff --git a/databases/_sponsorTimes_indexes.sql b/databases/_sponsorTimes_indexes.sql index 4a353d0..89dd91b 100644 --- a/databases/_sponsorTimes_indexes.sql +++ b/databases/_sponsorTimes_indexes.sql @@ -86,4 +86,21 @@ CREATE INDEX IF NOT EXISTS "videoInfo_videoID" CREATE INDEX IF NOT EXISTS "videoInfo_channelID" ON public."videoInfo" USING btree ("channelID" COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; + +-- ratings + +CREATE INDEX IF NOT EXISTS "ratings_hashedVideoID_gin" + ON public."ratings" USING gin + ("hashedVideoID" COLLATE pg_catalog."default" gin_trgm_ops, category COLLATE pg_catalog."default" gin_trgm_ops) + TABLESPACE pg_default; + +CREATE INDEX IF NOT EXISTS "ratings_hashedVideoID" + ON public."ratings" USING btree + ("hashedVideoID" COLLATE pg_catalog."default" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; + +CREATE INDEX IF NOT EXISTS "ratings_videoID" + ON public."ratings" USING btree + ("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST) TABLESPACE pg_default; \ No newline at end of file diff --git a/databases/_upgrade_private_4.sql b/databases/_upgrade_private_4.sql new file mode 100644 index 0000000..1bc137c --- /dev/null +++ b/databases/_upgrade_private_4.sql @@ -0,0 +1,14 @@ +BEGIN TRANSACTION; + +CREATE TABLE IF NOT EXISTS "ratings" ( + "videoID" TEXT NOT NULL, + "service" TEXT NOT NULL default 'YouTube', + "type" INTEGER NOT NULL, + "userID" TEXT NOT NULL, + "timeSubmitted" INTEGER NOT NULL, + "hashedIP" TEXT NOT NULL +); + +UPDATE "config" SET value = 4 WHERE key = 'version'; + +COMMIT; \ No newline at end of file diff --git a/databases/_upgrade_sponsorTimes_28.sql b/databases/_upgrade_sponsorTimes_28.sql new file mode 100644 index 0000000..031436a --- /dev/null +++ b/databases/_upgrade_sponsorTimes_28.sql @@ -0,0 +1,13 @@ +BEGIN TRANSACTION; + +CREATE TABLE IF NOT EXISTS "ratings" ( + "videoID" TEXT NOT NULL, + "service" TEXT NOT NULL default 'YouTube', + "type" INTEGER NOT NULL, + "count" INTEGER NOT NULL, + "hashedVideoID" TEXT NOT NULL +); + +UPDATE "config" SET value = 28 WHERE key = 'version'; + +COMMIT; \ No newline at end of file diff --git a/src/app.ts b/src/app.ts index fbb7d2c..3f2b993 100644 --- a/src/app.ts +++ b/src/app.ts @@ -43,6 +43,8 @@ import ExpressPromiseRouter from "express-promise-router"; import { Server } from "http"; import { youtubeApiProxy } from "./routes/youtubeApiProxy"; import { getChapterNames } from "./routes/getChapterNames"; +import { postRating } from "./routes/ratings/postRating"; +import { getRating } from "./routes/ratings/getRating"; export function createServer(callback: () => void): Server { // Create a service (the app object is just a callback). @@ -74,9 +76,11 @@ function setupRoutes(router: Router) { // Rate limit endpoint lists const voteEndpoints: RequestHandler[] = [voteOnSponsorTime]; const viewEndpoints: RequestHandler[] = [viewedVideoSponsorTime]; + const postRateEndpoints: RequestHandler[] = [postRating]; if (config.rateLimit) { if (config.rateLimit.vote) voteEndpoints.unshift(rateLimitMiddleware(config.rateLimit.vote, voteGetUserID)); if (config.rateLimit.view) viewEndpoints.unshift(rateLimitMiddleware(config.rateLimit.view)); + if (config.rateLimit.rate) postRateEndpoints.unshift(rateLimitMiddleware(config.rateLimit.rate)); } //add the get function @@ -186,6 +190,10 @@ function setupRoutes(router: Router) { router.get("/api/lockReason", getLockReason); + // ratings + router.get("/api/ratings/rate/:prefix", getRating); + router.post("/api/ratings/rate", postRateEndpoints); + if (config.postgres) { router.get("/database", (req, res) => dumpDatabase(req, res, true)); router.get("/database.json", (req, res) => dumpDatabase(req, res, false)); diff --git a/src/config.ts b/src/config.ts index b60aa21..c666a90 100644 --- a/src/config.ts +++ b/src/config.ts @@ -58,6 +58,12 @@ addDefaults(config, { statusCode: 200, message: "Too many views, please try again later", }, + rate: { + windowMs: 900000, + max: 20, + statusCode: 200, + message: "Success", + } }, userCounterURL: null, newLeafURLs: null, diff --git a/src/routes/ratings/getRating.ts b/src/routes/ratings/getRating.ts new file mode 100644 index 0000000..fe5211f --- /dev/null +++ b/src/routes/ratings/getRating.ts @@ -0,0 +1,82 @@ +import { Request, Response } from "express"; +import { db } from "../../databases/databases"; +import { RatingType } from "../../types/ratings.model"; +import { Service, VideoID, VideoIDHash } from "../../types/segments.model"; +import { getService } from "../../utils/getService"; +import { hashPrefixTester } from "../../utils/hashPrefixTester"; +import { Logger } from "../../utils/logger"; +import { QueryCacher } from "../../utils/queryCacher"; +import { ratingHashKey } from "../../utils/redisKeys"; + +interface DBRating { + videoID: VideoID, + hashedVideoID: VideoIDHash, + service: Service, + type: RatingType, + count: number +} + +export async function getRating(req: Request, res: Response): Promise { + let hashPrefix = req.params.prefix as VideoIDHash; + if (!hashPrefix || !hashPrefixTester(hashPrefix)) { + return res.status(400).send("Hash prefix does not match format requirements."); // Exit early on faulty prefix + } + hashPrefix = hashPrefix.toLowerCase() as VideoIDHash; + + let types: RatingType[] = []; + try { + types = req.query.types + ? JSON.parse(req.query.types as string) + : req.query.type + ? Array.isArray(req.query.type) + ? req.query.type + : [req.query.type] + : [RatingType.Upvote, RatingType.Downvote]; + if (!Array.isArray(types)) { + return res.status(400).send("Categories parameter does not match format requirements."); + } + + types = types.map((type) => parseInt(type as unknown as string, 10)); + } catch(error) { + return res.status(400).send("Bad parameter: categories (invalid JSON)"); + } + + const service: Service = getService(req.query.service, req.body.service); + + try { + const ratings = (await getRatings(hashPrefix, service)) + .filter((rating) => types.includes(rating.type)) + .map((rating) => ({ + videoID: rating.videoID, + hash: rating.hashedVideoID, + service: rating.service, + type: rating.type, + count: rating.count + })); + + if (ratings) { + res.status(200); + } else { + res.status(404); + } + return res.send(ratings ?? []); + } catch (err) { + Logger.error(err as string); + return res.sendStatus(500); + } +} + +async function getRatings(hashPrefix: VideoIDHash, service: Service): Promise { + const fetchFromDB = () => db + .prepare( + "all", + `SELECT "videoID", "hashedVideoID", "type", "count" FROM "ratings" WHERE "hashedVideoID" LIKE ? AND "service" = ? ORDER BY "hashedVideoID"`, + [`${hashPrefix}%`, service] + ) as Promise; + + if (hashPrefix.length === 4) { + return await QueryCacher.get(fetchFromDB, ratingHashKey(hashPrefix, service)); + } + + return fetchFromDB(); +} \ No newline at end of file diff --git a/src/routes/ratings/postRating.ts b/src/routes/ratings/postRating.ts new file mode 100644 index 0000000..7e73903 --- /dev/null +++ b/src/routes/ratings/postRating.ts @@ -0,0 +1,62 @@ +import { db, privateDB } from "../../databases/databases"; +import { getHash } from "../../utils/getHash"; +import { Logger } from "../../utils/logger"; +import { Request, Response } from "express"; +import { HashedUserID, UserID } from "../../types/user.model"; +import { HashedIP, IPAddress, VideoID } from "../../types/segments.model"; +import { getIP } from "../../utils/getIP"; +import { getService } from "../../utils/getService"; +import { RatingType, RatingTypes } from "../../types/ratings.model"; +import { config } from "../../config"; + +export async function postRating(req: Request, res: Response): Promise { + const privateUserID = req.body.userID as UserID; + const videoID = req.body.videoID as VideoID; + const service = getService(req.query.service, req.body.service); + const type = req.body.type as RatingType; + const enabled = req.body.enabled ?? true; + + if (privateUserID == undefined || videoID == undefined || service == undefined || type == undefined + || (typeof privateUserID !== "string") || (typeof videoID !== "string") || (typeof service !== "string") + || (typeof type !== "number") || (enabled && (typeof enabled !== "boolean")) || !RatingTypes.includes(type)) { + //invalid request + return res.sendStatus(400); + } + + const hashedIP: HashedIP = getHash(getIP(req) + config.globalSalt as IPAddress, 1); + const hashedUserID: HashedUserID = getHash(privateUserID); + const hashedVideoID = getHash(videoID, 1); + + try { + // Check if this user has voted before + const existingVote = await privateDB.prepare("get", `SELECT count(*) as "count" FROM "ratings" WHERE "videoID" = ? AND "service" = ? AND "type" = ? AND "userID" = ?`, [videoID, service, type, hashedUserID]); + if (existingVote.count > 0 && !enabled) { + // Undo the vote + await db.prepare("run", `UPDATE "ratings" SET "count" = "count" - 1 WHERE "videoID" = ? AND "service" = ? AND type = ?`, [videoID, service, type]); + await privateDB.prepare("run", `DELETE FROM "ratings" WHERE "videoID" = ? AND "service" = ? AND "type" = ? AND "userID" = ?`, [videoID, service, type, hashedUserID]); + + return res.sendStatus(200); + } else if (existingVote.count === 0 && enabled) { + // Make sure there hasn't been another vote from this IP + const existingIPVote = (await privateDB.prepare("get", `SELECT count(*) as "count" FROM "ratings" WHERE "videoID" = ? AND "service" = ? AND "type" = ? AND "hashedIP" = ?`, [videoID, service, type, hashedIP])) + .count > 0; + if (!existingIPVote) { + // Check if general rating already exists, if so increase it + const rating = await db.prepare("get", `SELECT count(*) as "count" FROM "ratings" WHERE "videoID" = ? AND "service" = ? AND type = ?`, [videoID, service, type]); + if (rating.count > 0) { + await db.prepare("run", `UPDATE "ratings" SET "count" = "count" + 1 WHERE "videoID" = ? AND "service" = ? AND type = ?`, [videoID, service, type]); + } else { + await db.prepare("run", `INSERT INTO "ratings" ("videoID", "service", "type", "count", "hashedVideoID") VALUES (?, ?, ?, 1, ?)`, [videoID, service, type, hashedVideoID]); + } + + // Create entry in privateDB + await privateDB.prepare("run", `INSERT INTO "ratings" ("videoID", "service", "type", "userID", "timeSubmitted", "hashedIP") VALUES (?, ?, ?, ?, ?, ?)`, [videoID, service, type, hashedUserID, Date.now(), hashedIP]); + } + } + + return res.sendStatus(200); + } catch (err) { + Logger.error(err as string); + return res.sendStatus(500); + } +} \ No newline at end of file diff --git a/src/types/config.model.ts b/src/types/config.model.ts index 1614c85..bc74a27 100644 --- a/src/types/config.model.ts +++ b/src/types/config.model.ts @@ -34,6 +34,7 @@ export interface SBSConfig { rateLimit: { vote: RateLimitConfig; view: RateLimitConfig; + rate: RateLimitConfig; }; mysql?: any; privateMysql?: any; diff --git a/src/types/ratings.model.ts b/src/types/ratings.model.ts new file mode 100644 index 0000000..28b3fbb --- /dev/null +++ b/src/types/ratings.model.ts @@ -0,0 +1,6 @@ +export enum RatingType { + Downvote = 0, + Upvote = 1 +} + +export const RatingTypes = [RatingType.Downvote, RatingType.Upvote]; \ No newline at end of file diff --git a/src/utils/getIP.ts b/src/utils/getIP.ts index 668b9a7..0d2dd90 100644 --- a/src/utils/getIP.ts +++ b/src/utils/getIP.ts @@ -15,6 +15,6 @@ export function getIP(req: Request): IPAddress { case "X-Real-IP": return req.headers["x-real-ip"] as IPAddress; default: - return req.connection.remoteAddress as IPAddress; + return (req.connection?.remoteAddress || req.socket?.remoteAddress) as IPAddress; } } \ No newline at end of file diff --git a/src/utils/redisKeys.ts b/src/utils/redisKeys.ts index c93e5c1..4d9cd9e 100644 --- a/src/utils/redisKeys.ts +++ b/src/utils/redisKeys.ts @@ -16,3 +16,10 @@ export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: S export function reputationKey(userID: UserID): string { return `reputation.user.${userID}`; } + +export function ratingHashKey(hashPrefix: VideoIDHash, service: Service): string { + hashPrefix = hashPrefix.substring(0, 4) as VideoIDHash; + if (hashPrefix.length !== 4) Logger.warn(`Redis rating hash-prefix key is not length 4! ${hashPrefix}`); + + return `rating.${service}.${hashPrefix}`; +} \ No newline at end of file diff --git a/test/cases/ratings/getRating.ts b/test/cases/ratings/getRating.ts new file mode 100644 index 0000000..876ec47 --- /dev/null +++ b/test/cases/ratings/getRating.ts @@ -0,0 +1,49 @@ +import { db } from "../../../src/databases/databases"; +import { getHash } from "../../../src/utils/getHash"; +import assert from "assert"; +import { client } from "../../utils/httpClient"; +import { AxiosResponse } from "axios"; +import { partialDeepEquals } from "../../utils/partialDeepEquals"; + +const endpoint = "/api/ratings/rate/"; +const getRating = (hash: string, params?: unknown): Promise => client.get(endpoint + hash, { params }); + +describe("getRating", () => { + before(async () => { + const insertUserNameQuery = 'INSERT INTO "ratings" ("videoID", "service", "type", "count", "hashedVideoID") VALUES (?, ?, ?, ?, ?)'; + await db.prepare("run", insertUserNameQuery, ["some-likes-and-dislikes", "YouTube", 0, 5, getHash("some-likes-and-dislikes", 1)]); //b3f0 + await db.prepare("run", insertUserNameQuery, ["some-likes-and-dislikes", "YouTube", 1, 10, getHash("some-likes-and-dislikes", 1)]); + }); + + it("Should be able to get dislikes and likes by default", (done) => { + getRating("b3f0") + .then(res => { + assert.strictEqual(res.status, 200); + const expected = [{ + type: 0, + count: 5, + }, { + type: 1, + count: 10, + }]; + assert.ok(partialDeepEquals(res.data, expected)); + done(); + }) + .catch(err => done(err)); + }); + + it("Should be able to filter for only dislikes", (done) => { + getRating("b3f0", { type: 0 }) + .then(res => { + assert.strictEqual(res.status, 200); + const expected = [{ + type: 0, + count: 5, + }]; + assert.ok(partialDeepEquals(res.data, expected)); + + done(); + }) + .catch(err => done(err)); + }); +}); \ No newline at end of file diff --git a/test/cases/ratings/postRating.ts b/test/cases/ratings/postRating.ts new file mode 100644 index 0000000..9ff7ebd --- /dev/null +++ b/test/cases/ratings/postRating.ts @@ -0,0 +1,96 @@ +import { db } from "../../../src/databases/databases"; +import { getHash } from "../../../src/utils/getHash"; +import assert from "assert"; +import { client } from "../../utils/httpClient"; +import { AxiosResponse } from "axios"; +import { partialDeepEquals } from "../../utils/partialDeepEquals"; + +const endpoint = "/api/ratings/rate/"; +const postRating = (body: unknown): Promise => client.post(endpoint, body); +const queryDatabase = (videoID: string) => db.prepare("all", `SELECT * FROM "ratings" WHERE "videoID" = ?`, [videoID]); + +describe("postRating", () => { + before(async () => { + const insertUserNameQuery = 'INSERT INTO "ratings" ("videoID", "service", "type", "count", "hashedVideoID") VALUES (?, ?, ?, ?, ?)'; + await db.prepare("run", insertUserNameQuery, ["multiple-rates", "YouTube", 0, 3, getHash("multiple-rates", 1)]); + }); + + it("Should be able to vote on a video", (done) => { + postRating({ + userID: "rating-testman", + videoID: "normal-video", + type: 0 + }) + .then(async res => { + assert.strictEqual(res.status, 200); + const expected = [{ + hashedVideoID: getHash("normal-video", 1), + videoID: "normal-video", + type: 0, + count: 1, + service: "YouTube" + }]; + assert.ok(partialDeepEquals(await queryDatabase("normal-video"), expected)); + done(); + }) + .catch(err => done(err)); + }); + + it("Should be able to undo a vote on a video", (done) => { + postRating({ + userID: "rating-testman", + videoID: "normal-video", + type: 0, + enabled: false + }) + .then(async res => { + assert.strictEqual(res.status, 200); + const expected = [{ + type: 0, + count: 0 + }]; + assert.ok(partialDeepEquals(await queryDatabase("normal-video"), expected)); + done(); + }) + .catch(err => done(err)); + }); + + it("Should be able to vote after someone else on a video", (done) => { + postRating({ + userID: "rating-testman", + videoID: "multiple-rates", + type: 0 + }) + .then(async res => { + assert.strictEqual(res.status, 200); + const expected = [{ + type: 0, + count: 4 + }]; + assert.ok(partialDeepEquals(await queryDatabase("multiple-rates"), expected)); + done(); + }) + .catch(err => done(err)); + }); + + it("Should be able to vote a different type than existing votes on a video", (done) => { + postRating({ + userID: "rating-testman", + videoID: "multiple-rates", + type: 1 + }) + .then(async res => { + assert.strictEqual(res.status, 200); + const expected = [{ + type: 0, + count: 4 + }, { + type: 1, + count: 1 + }]; + assert.ok(partialDeepEquals(await queryDatabase("multiple-rates"), expected)); + done(); + }) + .catch(err => done(err)); + }); +}); \ No newline at end of file diff --git a/test/test.ts b/test/test.ts index 904ec1d..3484add 100644 --- a/test/test.ts +++ b/test/test.ts @@ -32,19 +32,21 @@ async function init() { // Instantiate a Mocha instance. const mocha = new Mocha(); - const testDir = "./test/cases"; + const testDirs = ["./test/cases", "./test/cases/ratings"]; // Add each .ts file to the mocha instance - fs.readdirSync(testDir) - .filter((file) => - // Only keep the .ts files - file.substr(-3) === ".ts" - ) - .forEach(function(file) { - mocha.addFile( - path.join(testDir, file) - ); - }); + testDirs.forEach(testDir => { + fs.readdirSync(testDir) + .filter((file) => + // Only keep the .ts files + file.substr(-3) === ".ts" + ) + .forEach(function(file) { + mocha.addFile( + path.join(testDir, file) + ); + }); + }); const mockServer = createMockServer(() => { Logger.info("Started mock HTTP Server");