Add endpoints for rating endpoint (dislikes)

https://github.com/ajayyy/SponsorBlock/issues/1039

.eslintrc.js

@@ -27,6 +27,6 @@ module.exports = {
         "indent": ["warn", 4, { "SwitchCase": 1 }],
         "object-curly-spacing": ["warn", "always"],
         "require-await": "warn",
-        "no-console": "error"
+        "no-console": "warn"
     },
 };

DatabaseSchema.md

@@ -185,6 +185,22 @@
 | hashedVideoID | TEXT | not null, default '', sha256 |
 | userAgent | TEXT | not null, default '' |
 
+### ratings  
+
+| Name | Type | |
+| -- | :--: | -- |
+| videoID | TEXT | not null |
+| service | TEXT | not null, default 'YouTube' |
+| type | INTEGER | not null |
+| count | INTEGER | not null |
+| hashedVideoID | TEXT | not null |
+
+| index | field |
+| -- | :--: |
+| ratings_hashedVideoID_gin | hashedVideoID |
+| ratings_hashedVideoID | hashedVideoID, service |
+| ratings_videoID | videoID, service |
+
 # Private 
 
 [vote](#vote)  
@@ -239,3 +255,18 @@
 | -- | :--: | -- |
 | key | TEXT | not null |
 | value | TEXT | not null |
+
+### ratings  
+
+| Name | Type | |
+| -- | :--: | -- |
+| videoID | TEXT | not null |
+| service | TEXT | not null, default 'YouTube' |
+| userID | TEXT | not null |
+| type | INTEGER | not null |
+| timeSubmitted | INTEGER | not null |
+| hashedIP | TEXT | not null |
+
+| index | field |
+| -- | :--: |
+| ratings_videoID | videoID, service, userID, timeSubmitted |

databases/_private_indexes.sql

@@ -23,3 +23,10 @@ CREATE INDEX IF NOT EXISTS "categoryVotes_UUID"
     ON public."categoryVotes" USING btree
     ("UUID" COLLATE pg_catalog."default" ASC NULLS LAST, "userID" COLLATE pg_catalog."default" ASC NULLS LAST, "hashedIP" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST)
     TABLESPACE pg_default;
+
+-- ratings
+
+CREATE INDEX IF NOT EXISTS "ratings_videoID"
+    ON public."ratings" USING btree
+    ("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST, "userID" COLLATE pg_catalog."default" ASC NULLS LAST, "timeSubmitted" ASC NULLS LAST)
+    TABLESPACE pg_default;

databases/_sponsorTimes_indexes.sql

@@ -87,3 +87,20 @@ CREATE INDEX IF NOT EXISTS "videoInfo_channelID"
     ON public."videoInfo" USING btree
     ("channelID" COLLATE pg_catalog."default" ASC NULLS LAST)
     TABLESPACE pg_default;
+
+-- ratings
+
+CREATE INDEX IF NOT EXISTS "ratings_hashedVideoID_gin"
+    ON public."ratings" USING gin
+    ("hashedVideoID" COLLATE pg_catalog."default" gin_trgm_ops, category COLLATE pg_catalog."default" gin_trgm_ops)
+    TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "ratings_hashedVideoID"
+    ON public."ratings" USING btree
+    ("hashedVideoID" COLLATE pg_catalog."default" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+CREATE INDEX IF NOT EXISTS "ratings_videoID"
+    ON public."ratings" USING btree
+    ("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;

databases/_upgrade_private_4.sql

@@ -0,0 +1,14 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE IF NOT EXISTS "ratings" (
+	"videoID"		TEXT NOT NULL,
+	"service"		TEXT NOT NULL default 'YouTube',
+	"type"			INTEGER NOT NULL,
+	"userID"		TEXT NOT NULL,
+    "timeSubmitted"	INTEGER NOT NULL,
+	"hashedIP"		TEXT NOT NULL
+);
+
+UPDATE "config" SET value = 4 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_28.sql

@@ -0,0 +1,13 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE IF NOT EXISTS "ratings" (
+	"videoID"	    TEXT NOT NULL,
+	"service"	    TEXT NOT NULL default 'YouTube',
+	"type"	        INTEGER NOT NULL,
+	"count"	        INTEGER NOT NULL,
+	"hashedVideoID"	TEXT NOT NULL
+);
+
+UPDATE "config" SET value = 28 WHERE key = 'version';
+
+COMMIT;

src/app.ts

@@ -43,6 +43,8 @@ import ExpressPromiseRouter from "express-promise-router";
 import { Server } from "http";
 import { youtubeApiProxy } from "./routes/youtubeApiProxy";
 import { getChapterNames } from "./routes/getChapterNames";
+import { postRating } from "./routes/ratings/postRating";
+import { getRating } from "./routes/ratings/getRating";
 
 export function createServer(callback: () => void): Server {
     // Create a service (the app object is just a callback).
@@ -74,9 +76,11 @@ function setupRoutes(router: Router) {
     // Rate limit endpoint lists
     const voteEndpoints: RequestHandler[] = [voteOnSponsorTime];
     const viewEndpoints: RequestHandler[] = [viewedVideoSponsorTime];
+    const postRateEndpoints: RequestHandler[] = [postRating];
     if (config.rateLimit) {
         if (config.rateLimit.vote) voteEndpoints.unshift(rateLimitMiddleware(config.rateLimit.vote, voteGetUserID));
         if (config.rateLimit.view) viewEndpoints.unshift(rateLimitMiddleware(config.rateLimit.view));
+        if (config.rateLimit.rate) postRateEndpoints.unshift(rateLimitMiddleware(config.rateLimit.rate));
     }
 
     //add the get function
@@ -186,6 +190,10 @@ function setupRoutes(router: Router) {
 
     router.get("/api/lockReason", getLockReason);
 
+    // ratings
+    router.get("/api/ratings/rate/:prefix", getRating);
+    router.post("/api/ratings/rate", postRateEndpoints);
+
     if (config.postgres) {
         router.get("/database", (req, res) => dumpDatabase(req, res, true));
         router.get("/database.json", (req, res) => dumpDatabase(req, res, false));

src/config.ts

@@ -58,6 +58,12 @@ addDefaults(config, {
             statusCode: 200,
             message: "Too many views, please try again later",
         },
+        rate: {
+            windowMs: 900000,
+            max: 20,
+            statusCode: 200,
+            message: "Success",
+        }
     },
     userCounterURL: null,
     newLeafURLs: null,

src/routes/ratings/getRating.ts

@@ -0,0 +1,82 @@
+import { Request, Response } from "express";
+import { db } from "../../databases/databases";
+import { RatingType } from "../../types/ratings.model";
+import { Service, VideoID, VideoIDHash } from "../../types/segments.model";
+import { getService } from "../../utils/getService";
+import { hashPrefixTester } from "../../utils/hashPrefixTester";
+import { Logger } from "../../utils/logger";
+import { QueryCacher } from "../../utils/queryCacher";
+import { ratingHashKey } from "../../utils/redisKeys";
+
+interface DBRating {
+    videoID: VideoID,
+    hashedVideoID: VideoIDHash,
+    service: Service,
+    type: RatingType,
+    count: number
+}
+
+export async function getRating(req: Request, res: Response): Promise<Response> {
+    let hashPrefix = req.params.prefix as VideoIDHash;
+    if (!hashPrefix || !hashPrefixTester(hashPrefix)) {
+        return res.status(400).send("Hash prefix does not match format requirements."); // Exit early on faulty prefix
+    }
+    hashPrefix = hashPrefix.toLowerCase() as VideoIDHash;
+
+    let types: RatingType[] = [];
+    try {
+        types = req.query.types
+            ? JSON.parse(req.query.types as string)
+            : req.query.type
+                ? Array.isArray(req.query.type)
+                    ? req.query.type
+                    : [req.query.type]
+                : [RatingType.Upvote, RatingType.Downvote];
+        if (!Array.isArray(types)) {
+            return res.status(400).send("Categories parameter does not match format requirements.");
+        }
+
+        types = types.map((type) => parseInt(type as unknown as string, 10));
+    } catch(error) {
+        return res.status(400).send("Bad parameter: categories (invalid JSON)");
+    }
+
+    const service: Service = getService(req.query.service, req.body.service);
+
+    try {
+        const ratings = (await getRatings(hashPrefix, service))
+            .filter((rating) => types.includes(rating.type))
+            .map((rating) => ({
+                videoID: rating.videoID,
+                hash: rating.hashedVideoID,
+                service: rating.service,
+                type: rating.type,
+                count: rating.count
+            }));
+
+        if (ratings) {
+            res.status(200);
+        } else {
+            res.status(404);
+        }
+        return res.send(ratings ?? []);
+    } catch (err) {
+        Logger.error(err as string);
+        return res.sendStatus(500);
+    }
+}
+
+async function getRatings(hashPrefix: VideoIDHash, service: Service): Promise<DBRating[]> {
+    const fetchFromDB = () => db
+        .prepare(
+            "all",
+            `SELECT "videoID", "hashedVideoID", "type", "count" FROM "ratings" WHERE "hashedVideoID" LIKE ? AND "service" = ? ORDER BY "hashedVideoID"`,
+            [`${hashPrefix}%`, service]
+        ) as Promise<DBRating[]>;
+
+    if (hashPrefix.length === 4) {
+        return await QueryCacher.get(fetchFromDB, ratingHashKey(hashPrefix, service));
+    }
+
+    return fetchFromDB();
+}

src/routes/ratings/postRating.ts

@@ -0,0 +1,62 @@
+import { db, privateDB } from "../../databases/databases";
+import { getHash } from "../../utils/getHash";
+import { Logger } from "../../utils/logger";
+import { Request, Response } from "express";
+import { HashedUserID, UserID } from "../../types/user.model";
+import { HashedIP, IPAddress, VideoID } from "../../types/segments.model";
+import { getIP } from "../../utils/getIP";
+import { getService } from "../../utils/getService";
+import { RatingType, RatingTypes } from "../../types/ratings.model";
+import { config } from "../../config";
+
+export async function postRating(req: Request, res: Response): Promise<Response> {
+    const privateUserID = req.body.userID as UserID;
+    const videoID = req.body.videoID as VideoID;
+    const service = getService(req.query.service, req.body.service);
+    const type = req.body.type as RatingType;
+    const enabled = req.body.enabled ?? true;
+
+    if (privateUserID == undefined || videoID == undefined || service == undefined || type == undefined
+          || (typeof privateUserID !== "string") || (typeof videoID !== "string") || (typeof service !== "string")
+          || (typeof type !== "number") || (enabled && (typeof enabled !== "boolean")) || !RatingTypes.includes(type)) {
+        //invalid request
+        return res.sendStatus(400);
+    }
+
+    const hashedIP: HashedIP = getHash(getIP(req) + config.globalSalt as IPAddress, 1);
+    const hashedUserID: HashedUserID = getHash(privateUserID);
+    const hashedVideoID = getHash(videoID, 1);
+
+    try {
+        // Check if this user has voted before
+        const existingVote = await privateDB.prepare("get", `SELECT count(*) as "count" FROM "ratings" WHERE "videoID" = ? AND "service" = ? AND "type" = ? AND "userID" = ?`, [videoID, service, type, hashedUserID]);
+        if (existingVote.count > 0 && !enabled) {
+            // Undo the vote
+            await db.prepare("run", `UPDATE "ratings" SET "count" = "count" - 1 WHERE "videoID" = ? AND "service" = ? AND type = ?`, [videoID, service, type]);
+            await privateDB.prepare("run", `DELETE FROM "ratings" WHERE "videoID" = ? AND "service" = ? AND "type" = ? AND "userID" = ?`, [videoID, service, type, hashedUserID]);
+
+            return res.sendStatus(200);
+        } else if (existingVote.count === 0 && enabled) {
+            // Make sure there hasn't been another vote from this IP
+            const existingIPVote = (await privateDB.prepare("get", `SELECT count(*) as "count" FROM "ratings" WHERE "videoID" = ? AND "service" = ? AND "type" = ? AND "hashedIP" = ?`, [videoID, service, type, hashedIP]))
+                .count > 0;
+            if (!existingIPVote) {
+                // Check if general rating already exists, if so increase it
+                const rating = await db.prepare("get", `SELECT count(*) as "count" FROM "ratings" WHERE "videoID" = ? AND "service" = ? AND type = ?`, [videoID, service, type]);
+                if (rating.count > 0) {
+                    await db.prepare("run", `UPDATE "ratings" SET "count" = "count" + 1 WHERE "videoID" = ? AND "service" = ? AND type = ?`, [videoID, service, type]);
+                } else {
+                    await db.prepare("run", `INSERT INTO "ratings" ("videoID", "service", "type", "count", "hashedVideoID") VALUES (?, ?, ?, 1, ?)`, [videoID, service, type, hashedVideoID]);
+                }
+
+                // Create entry in privateDB
+                await privateDB.prepare("run", `INSERT INTO "ratings" ("videoID", "service", "type", "userID", "timeSubmitted", "hashedIP") VALUES (?, ?, ?, ?, ?, ?)`, [videoID, service, type, hashedUserID, Date.now(), hashedIP]);
+            }
+        }
+
+        return res.sendStatus(200);
+    } catch (err) {
+        Logger.error(err as string);
+        return res.sendStatus(500);
+    }
+}

src/types/config.model.ts

@@ -34,6 +34,7 @@ export interface SBSConfig {
     rateLimit: {
         vote: RateLimitConfig;
         view: RateLimitConfig;
+        rate: RateLimitConfig;
     };
     mysql?: any;
     privateMysql?: any;

src/types/ratings.model.ts

@@ -0,0 +1,6 @@
+export enum RatingType {
+    Downvote = 0,
+    Upvote = 1
+}
+
+export const RatingTypes = [RatingType.Downvote, RatingType.Upvote];

src/utils/getIP.ts

@@ -15,6 +15,6 @@ export function getIP(req: Request): IPAddress {
         case "X-Real-IP":
             return req.headers["x-real-ip"] as IPAddress;
         default:
-            return req.connection.remoteAddress as IPAddress;
+            return (req.connection?.remoteAddress || req.socket?.remoteAddress) as IPAddress;
     }
 }

src/utils/redisKeys.ts

@@ -16,3 +16,10 @@ export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: S
 export function reputationKey(userID: UserID): string {
     return `reputation.user.${userID}`;
 }
+
+export function ratingHashKey(hashPrefix: VideoIDHash, service: Service): string {
+    hashPrefix = hashPrefix.substring(0, 4) as VideoIDHash;
+    if (hashPrefix.length !== 4) Logger.warn(`Redis rating hash-prefix key is not length 4! ${hashPrefix}`);
+
+    return `rating.${service}.${hashPrefix}`;
+}

test/cases/ratings/getRating.ts

@@ -0,0 +1,49 @@
+import { db } from "../../../src/databases/databases";
+import { getHash } from "../../../src/utils/getHash";
+import assert from "assert";
+import { client } from "../../utils/httpClient";
+import { AxiosResponse } from "axios";
+import { partialDeepEquals } from "../../utils/partialDeepEquals";
+
+const endpoint = "/api/ratings/rate/";
+const getRating = (hash: string, params?: unknown): Promise<AxiosResponse> => client.get(endpoint + hash, { params });
+
+describe("getRating", () => {
+    before(async () => {
+        const insertUserNameQuery = 'INSERT INTO "ratings" ("videoID", "service", "type", "count", "hashedVideoID") VALUES (?, ?, ?, ?, ?)';
+        await db.prepare("run", insertUserNameQuery, ["some-likes-and-dislikes", "YouTube", 0, 5, getHash("some-likes-and-dislikes", 1)]); //b3f0
+        await db.prepare("run", insertUserNameQuery, ["some-likes-and-dislikes", "YouTube", 1, 10, getHash("some-likes-and-dislikes", 1)]);
+    });
+
+    it("Should be able to get dislikes and likes by default", (done) => {
+        getRating("b3f0")
+            .then(res => {
+                assert.strictEqual(res.status, 200);
+                const expected = [{
+                    type: 0,
+                    count: 5,
+                }, {
+                    type: 1,
+                    count: 10,
+                }];
+                assert.ok(partialDeepEquals(res.data, expected));
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should be able to filter for only dislikes", (done) => {
+        getRating("b3f0", { type: 0 })
+            .then(res => {
+                assert.strictEqual(res.status, 200);
+                const expected = [{
+                    type: 0,
+                    count: 5,
+                }];
+                assert.ok(partialDeepEquals(res.data, expected));
+
+                done();
+            })
+            .catch(err => done(err));
+    });
+});

test/cases/ratings/postRating.ts

@@ -0,0 +1,96 @@
+import { db } from "../../../src/databases/databases";
+import { getHash } from "../../../src/utils/getHash";
+import assert from "assert";
+import { client } from "../../utils/httpClient";
+import { AxiosResponse } from "axios";
+import { partialDeepEquals } from "../../utils/partialDeepEquals";
+
+const endpoint = "/api/ratings/rate/";
+const postRating = (body: unknown): Promise<AxiosResponse> => client.post(endpoint, body);
+const queryDatabase = (videoID: string) => db.prepare("all", `SELECT * FROM "ratings" WHERE "videoID" = ?`, [videoID]);
+
+describe("postRating", () => {
+    before(async () => {
+        const insertUserNameQuery = 'INSERT INTO "ratings" ("videoID", "service", "type", "count", "hashedVideoID") VALUES (?, ?, ?, ?, ?)';
+        await db.prepare("run", insertUserNameQuery, ["multiple-rates", "YouTube", 0, 3, getHash("multiple-rates", 1)]);
+    });
+
+    it("Should be able to vote on a video", (done) => {
+        postRating({
+            userID: "rating-testman",
+            videoID: "normal-video",
+            type: 0
+        })
+            .then(async res => {
+                assert.strictEqual(res.status, 200);
+                const expected = [{
+                    hashedVideoID: getHash("normal-video", 1),
+                    videoID: "normal-video",
+                    type: 0,
+                    count: 1,
+                    service: "YouTube"
+                }];
+                assert.ok(partialDeepEquals(await queryDatabase("normal-video"), expected));
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should be able to undo a vote on a video", (done) => {
+        postRating({
+            userID: "rating-testman",
+            videoID: "normal-video",
+            type: 0,
+            enabled: false
+        })
+            .then(async res => {
+                assert.strictEqual(res.status, 200);
+                const expected = [{
+                    type: 0,
+                    count: 0
+                }];
+                assert.ok(partialDeepEquals(await queryDatabase("normal-video"), expected));
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should be able to vote after someone else on a video", (done) => {
+        postRating({
+            userID: "rating-testman",
+            videoID: "multiple-rates",
+            type: 0
+        })
+            .then(async res => {
+                assert.strictEqual(res.status, 200);
+                const expected = [{
+                    type: 0,
+                    count: 4
+                }];
+                assert.ok(partialDeepEquals(await queryDatabase("multiple-rates"), expected));
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should be able to vote a different type than existing votes on a video", (done) => {
+        postRating({
+            userID: "rating-testman",
+            videoID: "multiple-rates",
+            type: 1
+        })
+            .then(async res => {
+                assert.strictEqual(res.status, 200);
+                const expected = [{
+                    type: 0,
+                    count: 4
+                }, {
+                    type: 1,
+                    count: 1
+                }];
+                assert.ok(partialDeepEquals(await queryDatabase("multiple-rates"), expected));
+                done();
+            })
+            .catch(err => done(err));
+    });
+});

test/test.ts

@@ -32,9 +32,10 @@ async function init() {
     // Instantiate a Mocha instance.
     const mocha = new Mocha();
 
-    const testDir = "./test/cases";
+    const testDirs = ["./test/cases", "./test/cases/ratings"];
 
     // Add each .ts file to the mocha instance
+    testDirs.forEach(testDir => {
         fs.readdirSync(testDir)
             .filter((file) =>
                 // Only keep the .ts files
@@ -45,6 +46,7 @@ async function init() {
                     path.join(testDir, file)
                 );
             });
+    });
 
     const mockServer = createMockServer(() => {
         Logger.info("Started mock HTTP Server");