Add ability to add manually choose who can submit chapters

databases/_sponsorTimes_indexes.sql

@@ -108,4 +108,11 @@ CREATE INDEX IF NOT EXISTS "ratings_hashedVideoID"
 CREATE INDEX IF NOT EXISTS "ratings_videoID"
     ON public."ratings" USING btree
     ("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+--- userFeatures
+
+CREATE INDEX IF NOT EXISTS "userFeatures_userID"
+    ON public."userFeatures" USING btree
+    ("userID" COLLATE pg_catalog."default" ASC NULLS LAST, "feature" ASC NULLS LAST)
     TABLESPACE pg_default;

databases/_upgrade_sponsorTimes_33.sql

@@ -0,0 +1,13 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE IF NOT EXISTS "userFeatures" (
+	"userID" TEXT NOT NULL,
+	"feature" INTEGER NOT NULL,
+    "issuerUserID" TEXT NOT NULL,
+    "timeSubmitted" INTEGER NOT NULL,
+    PRIMARY KEY ("userID", "feature")
+);
+
+UPDATE "config" SET value = 33 WHERE key = 'version';
+
+COMMIT;

src/app.ts

@@ -48,6 +48,7 @@ import { getRating } from "./routes/ratings/getRating";
 import { postClearCache as ratingPostClearCache } from "./routes/ratings/postClearCache";
 import { getTopCategoryUsers } from "./routes/getTopCategoryUsers";
 import { addUserAsTempVIP } from "./routes/addUserAsTempVIP";
+import { addFeature } from "./routes/addFeature";
 
 export function createServer(callback: () => void): Server {
     // Create a service (the app object is just a callback).
@@ -196,6 +197,8 @@ function setupRoutes(router: Router) {
 
     router.get("/api/lockReason", getLockReason);
 
+    router.post("/api/feature", addFeature)
+
     // ratings
     router.get("/api/ratings/rate/:prefix", getRating);
     router.get("/api/ratings/rate", getRating);

src/routes/addFeature.ts

@@ -0,0 +1,72 @@
+import { getHashCache } from "../utils/getHashCache";
+import { db } from "../databases/databases";
+import { config } from "../config";
+import { Request, Response } from "express";
+import { isUserVIP } from "../utils/isUserVIP";
+import { Feature, HashedUserID } from "../types/user.model";
+import { Logger } from "../utils/logger";
+import { QueryCacher } from "../utils/queryCacher";
+
+interface AddFeatureRequest extends Request {
+    body: {
+        userID: HashedUserID;
+        adminUserID: string;
+        feature: string;
+        enabled: string;
+    }
+}
+
+const allowedFeatures = {
+    vip: [
+        Feature.ChapterSubmitter
+    ],
+    admin: [
+        Feature.ChapterSubmitter
+    ]
+}
+
+export async function addFeature(req: AddFeatureRequest, res: Response): Promise<Response> {
+    const { body: { userID, adminUserID } } = req;
+    const feature = parseInt(req.body.feature) as Feature;
+    const enabled = req.body?.enabled !== "false";
+
+    if (!userID || !adminUserID) {
+        // invalid request
+        return res.sendStatus(400);
+    }
+
+    // hash the userID
+    const adminUserIDInput = await getHashCache(adminUserID);
+    const isAdmin = adminUserIDInput !== config.adminUserID;
+    const isVIP = (await isUserVIP(userID)) || isAdmin;
+
+    if (!isAdmin && !isVIP) {
+        // not authorized
+        return res.sendStatus(403);
+    }
+
+    try {
+        const currentAllowedFeatures = isAdmin ? allowedFeatures.admin : allowedFeatures.vip;
+        if (currentAllowedFeatures.includes(feature)) {
+            if (enabled) {
+                const featureAdded = await db.prepare("get", 'SELECT "feature" from "userFeatures" WHERE "userID" = ? AND "feature" = ?', [userID, feature]);
+                if (!featureAdded) {
+                    await db.prepare("run", 'INSERT INTO "userFeatures" ("userID", "feature", "issuerUserID", "timeSubmitted") VALUES(?, ?, ?, ?)'
+                        , [userID, feature, adminUserID, Date.now()]);
+                }
+            } else {
+                await db.prepare("run", 'DELETE FROM "userFeatures" WHERE "userID" = ? AND "feature" = ?', [userID, feature]);
+            }
+
+            QueryCacher.clearFeatureCache(userID, feature);
+        } else {
+            return res.status(400).send("Invalid feature");
+        }
+
+        return res.sendStatus(200);
+    } catch (e) {
+        Logger.error(e as string);
+
+        return res.sendStatus(500);
+    }
+}

src/routes/getUserInfo.ts

@@ -7,6 +7,7 @@ import { HashedUserID, UserID } from "../types/user.model";
 import { getReputation } from "../utils/reputation";
 import { SegmentUUID } from "../types/segments.model";
 import { config } from "../config";
+import { canSubmitChapter } from "../utils/permissions";
 const maxRewardTime = config.maxRewardTimePerSegmentInSeconds;
 
 async function dbGetSubmittedSegmentSummary(userID: HashedUserID): Promise<{ minutesSaved: number, segmentCount: number }> {
@@ -105,10 +106,6 @@ async function dbGetBanned(userID: HashedUserID): Promise<boolean> {
     }
 }
 
-async function dbCanSubmitChapter(userID: HashedUserID): Promise<boolean> {
-    return (await isUserVIP(userID)) || (await getReputation(userID)) > config.minReputationToSubmitChapter;
-}
-
 type cases = Record<string, any>
 
 const executeIfFunction = (f: any) =>
@@ -133,7 +130,7 @@ const dbGetValue = (userID: HashedUserID, property: string): Promise<string|Segm
         reputation: () => getReputation(userID),
         vip: () => isUserVIP(userID),
         lastSegmentID: () => dbGetLastSegmentForUser(userID),
-        canSubmitChapter: () => dbCanSubmitChapter(userID)
+        canSubmitChapter: () => canSubmitChapter(userID)
     })("")(property);
 };
 

src/routes/postSkipSegments.ts

@@ -21,6 +21,7 @@ import { parseUserAgent } from "../utils/userAgent";
 import { getService } from "../utils/getService";
 import axios from "axios";
 import { vote } from "./voteOnSponsorTime";
+import { canSubmitChapter } from "../utils/permissions";
 
 type CheckResult = {
     pass: boolean,
@@ -200,7 +201,8 @@ async function checkUserActiveWarning(userID: string): Promise<CheckResult> {
     return CHECK_PASS;
 }
 
-function checkInvalidFields(videoID: VideoID, userID: UserID, segments: IncomingSegment[]): CheckResult {
+async function checkInvalidFields(videoID: VideoID, userID: UserID, hashedUserID: HashedUserID
+    , segments: IncomingSegment[]): Promise<CheckResult> {
     const invalidFields = [];
     const errors = [];
     if (typeof videoID !== "string" || videoID?.length == 0) {
@@ -227,6 +229,10 @@ function checkInvalidFields(videoID: VideoID, userID: UserID, segments: Incoming
                 || (segmentPair.description.length !== 0 && segmentPair.actionType !== ActionType.Chapter)) {
             invalidFields.push("segment description");
         }
+
+        if (segmentPair.actionType === ActionType.Chapter && !(await canSubmitChapter(hashedUserID))) {
+            invalidFields.push("permission to submit chapters");
+        }
     }
 
     if (invalidFields.length !== 0) {
@@ -478,14 +484,14 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
     // eslint-disable-next-line prefer-const
     let { videoID, userID: paramUserID, service, videoDuration, videoDurationParam, segments, userAgent } = preprocessInput(req);
 
-    const invalidCheckResult = checkInvalidFields(videoID, paramUserID, segments);
+    //hash the userID
+    const userID = await getHashCache(paramUserID || "");
+
+    const invalidCheckResult = await checkInvalidFields(videoID, paramUserID, userID, segments);
     if (!invalidCheckResult.pass) {
         return res.status(invalidCheckResult.errorCode).send(invalidCheckResult.errorMessage);
     }
 
-    //hash the userID
-    const userID = await getHashCache(paramUserID);
-
     const userWarningCheckResult = await checkUserActiveWarning(userID);
     if (!userWarningCheckResult.pass) {
         Logger.warn(`Caught a submission for a warned user. userID: '${userID}', videoID: '${videoID}', category: '${segments.reduce<string>((prev, val) => `${prev} ${val.category}`, "")}', times: ${segments.reduce<string>((prev, val) => `${prev} ${val.segment}`, "")}`);

src/types/user.model.ts

@@ -1,4 +1,8 @@
 import { HashedValue } from "./hash.model";
 
 export type UserID = string & { __userIDBrand: unknown };
-export type HashedUserID = UserID & HashedValue;
+export type HashedUserID = UserID & HashedValue;
+
+export enum Feature {
+    ChapterSubmitter = 0
+}

src/utils/features.ts

@@ -0,0 +1,11 @@
+import { db } from "../databases/databases";
+import { Feature, HashedUserID } from "../types/user.model";
+import { QueryCacher } from "./queryCacher";
+import { userFeatureKey } from "./redisKeys";
+
+export async function hasFeature(userID: HashedUserID, feature: Feature): Promise<boolean> {
+    return await QueryCacher.get(async () => {
+        const result = await db.prepare("get", 'SELECT "feature" from "userFeatures" WHERE "userID" = ? AND "feature" = ?', [userID, feature]);
+        return !!result;
+    }, userFeatureKey(userID, feature));
+}

src/utils/permissions.ts

@@ -0,0 +1,11 @@
+import { config } from "../config";
+import { Feature, HashedUserID } from "../types/user.model";
+import { hasFeature } from "./features";
+import { isUserVIP } from "./isUserVIP";
+import { getReputation } from "./reputation";
+
+export async function canSubmitChapter(userID: HashedUserID): Promise<boolean> {
+    return (await isUserVIP(userID))
+        || (await getReputation(userID)) > config.minReputationToSubmitChapter
+        || (await hasFeature(userID, Feature.ChapterSubmitter));
+}

src/utils/queryCacher.ts

@@ -1,8 +1,8 @@
 import redis from "../utils/redis";
 import { Logger } from "../utils/logger";
-import { skipSegmentsHashKey, skipSegmentsKey, reputationKey, ratingHashKey, skipSegmentGroupsKey } from "./redisKeys";
+import { skipSegmentsHashKey, skipSegmentsKey, reputationKey, ratingHashKey, skipSegmentGroupsKey, userFeatureKey } from "./redisKeys";
 import { Service, VideoID, VideoIDHash } from "../types/segments.model";
-import { UserID } from "../types/user.model";
+import { Feature, HashedUserID, UserID } from "../types/user.model";
 
 async function get<T>(fetchFromDB: () => Promise<T>, key: string): Promise<T> {
     try {
@@ -90,9 +90,14 @@ function clearRatingCache(videoInfo: { hashedVideoID: VideoIDHash; service: Serv
     }
 }
 
+function clearFeatureCache(userID: HashedUserID, feature: Feature): void {
+    redis.del(userFeatureKey(userID, feature)).catch((err) => Logger.error(err));
+}
+
 export const QueryCacher = {
     get,
     getAndSplit,
     clearSegmentCache,
-    clearRatingCache
+    clearRatingCache,
+    clearFeatureCache
 };

src/utils/redisKeys.ts

@@ -1,5 +1,5 @@
 import { Service, VideoID, VideoIDHash } from "../types/segments.model";
-import { HashedUserID, UserID } from "../types/user.model";
+import { Feature, HashedUserID, UserID } from "../types/user.model";
 import { HashedValue } from "../types/hash.model";
 import { Logger } from "./logger";
 
@@ -36,4 +36,8 @@ export function shaHashKey(singleIter: HashedValue): string {
 }
 
 export const tempVIPKey = (userID: HashedUserID): string =>
-    `vip.temp.${userID}`;
+    `vip.temp.${userID}`;
+
+export function userFeatureKey (userID: HashedUserID, feature: Feature): string {
+    return `user.${userID}.feature.${feature}`;
+}

test/cases/addFeatures.ts

@@ -0,0 +1,68 @@
+import assert from "assert";
+import { db } from "../../src/databases/databases";
+import { Feature, HashedUserID } from "../../src/types/user.model";
+import { hasFeature } from "../../src/utils/features";
+import { getHash } from "../../src/utils/getHash";
+import { client } from "../utils/httpClient";
+
+const endpoint = "/api/feature";
+
+const postAddFeatures = (userID: string, adminUserID: string, feature: Feature, enabled: string) => client({
+    method: "POST",
+    url: endpoint,
+    data: {
+        userID,
+        feature,
+        enabled,
+        adminUserID
+    }
+});
+
+const privateVipUserID = "VIPUser-addFeatures";
+const vipUserID = getHash(privateVipUserID);
+
+const hashedUserID1 = "user1-addFeatures" as HashedUserID;
+const hashedUserID2 = "user2-addFeatures" as HashedUserID;
+const hashedUserID3 = "user3-addFeatures" as HashedUserID;
+
+const validFeatures = [Feature.ChapterSubmitter];
+
+describe("addFeatures", () => {
+    before(() => {
+        const userFeatureQuery = `INSERT INTO "userFeatures" ("userID", "feature", "issuerUserID", "timeSubmitted") VALUES(?, ?, ?, ?)`;
+
+        return Promise.all([
+            db.prepare("run", `INSERT INTO "vipUsers" ("userID") VALUES (?)`, [vipUserID]),
+
+            db.prepare("run", userFeatureQuery, [hashedUserID2, Feature.ChapterSubmitter, "some-user", 0]),
+            db.prepare("run", userFeatureQuery, [hashedUserID3, Feature.ChapterSubmitter, "some-user", 0])
+        ]);
+    });
+
+    it("can add features", async () => {
+        for (const feature of validFeatures) {
+            const result = await postAddFeatures(hashedUserID1, vipUserID, feature, "true");
+            assert.strictEqual(result.status, 200);
+
+            assert.strictEqual(await hasFeature(hashedUserID1, feature), true);
+        }
+    });
+
+    it("can remove features", async () => {
+        const feature = Feature.ChapterSubmitter;
+
+        const result = await postAddFeatures(hashedUserID2, vipUserID, feature, "false");
+        assert.strictEqual(result.status, 200);
+
+        assert.strictEqual(await hasFeature(hashedUserID2, feature), false);
+    });
+
+    it("can update features", async () => {
+        const feature = Feature.ChapterSubmitter;
+
+        const result = await postAddFeatures(hashedUserID3, vipUserID, feature, "true");
+        assert.strictEqual(result.status, 200);
+
+        assert.strictEqual(await hasFeature(hashedUserID3, feature), true);
+    });
+});

test/cases/oldSubmitSponsorTimes.ts

@@ -44,8 +44,17 @@ describe("postVideoSponsorTime (Old submission method)", () => {
             .catch(err => done(err));
     });
 
-    it("Should return 400 for missing params", (done) => {
-        client.post(endpoint, { params: { startTime: 1, endTime: 10, userID } })
+    it("Should return 400 for missing video", (done) => {
+        client.get(endpoint, { params: { startTime: 1, endTime: 10, userID } })
+            .then(res => {
+                assert.strictEqual(res.status, 400);
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should return 400 for missing userID", (done) => {
+        client.get(endpoint, { params: { videoID: videoID1, startTime: 1, endTime: 10 } })
             .then(res => {
                 assert.strictEqual(res.status, 400);
                 done();

test/cases/postSkipSegments.ts

@@ -7,6 +7,7 @@ import * as YouTubeAPIModule from "../../src/utils/youtubeApi";
 import { YouTubeApiMock } from "../youtubeMock";
 import assert from "assert";
 import { client } from "../utils/httpClient";
+import { Feature } from "../../src/types/user.model";
 
 const mockManager = ImportMock.mockStaticClass(YouTubeAPIModule, "YouTubeAPI");
 const sinonStub = mockManager.mock("listVideos");
@@ -15,6 +16,7 @@ sinonStub.callsFake(YouTubeApiMock.listVideos);
 describe("postSkipSegments", () => {
 // Constant and helpers
     const submitUserOne = `PostSkipUser1${".".repeat(18)}`;
+    const submitUserOneHash = getHash(submitUserOne);
     const submitUserTwo = `PostSkipUser2${".".repeat(18)}`;
     const submitUserTwoHash = getHash(submitUserTwo);
     const submitUserThree = `PostSkipUser3${".".repeat(18)}`;
@@ -30,7 +32,6 @@ describe("postSkipSegments", () => {
     const banUser01 = "ban-user01-loremipsumdolorsitametconsectetur";
     const banUser01Hash = getHash(banUser01);
 
-    const submitUserOneHash = getHash(submitUserOne);
     const submitVIPuser = `VIPPostSkipUser${".".repeat(16)}`;
     const warnVideoID = "postSkip2";
     const badInputVideoID = "dQw4w9WgXcQ";
@@ -66,6 +67,15 @@ describe("postSkipSegments", () => {
         db.prepare("run", insertSponsorTimeQuery, ["full_video_duration_segment", 0, 0, 0, "full-video-duration-uuid-0", submitUserTwoHash, 0, 0, "sponsor", "full", 123, 0, "full_video_duration_segment"]);
         db.prepare("run", insertSponsorTimeQuery, ["full_video_duration_segment", 25, 30, 0, "full-video-duration-uuid-1", submitUserTwoHash, 0, 0, "sponsor", "skip", 123, 0, "full_video_duration_segment"]);
 
+        const reputationVideoID = "post_reputation_video";
+        db.prepare("run", insertSponsorTimeQuery, [reputationVideoID, 1, 11, 2,"post_reputation-5-uuid-0", submitUserOneHash, 1606240000000, 50, "sponsor", "skip", 0, 0, reputationVideoID]);
+        db.prepare("run", insertSponsorTimeQuery, [reputationVideoID, 1, 11, 2,"post_reputation-5-uuid-1", submitUserOneHash, 1606240000000, 50, "sponsor", "skip", 0, 0, reputationVideoID]);
+        db.prepare("run", insertSponsorTimeQuery, [reputationVideoID, 1, 11, 2,"post_reputation-5-uuid-2", submitUserOneHash, 1606240000000, 50, "sponsor", "skip", 0, 0, reputationVideoID]);
+        db.prepare("run", insertSponsorTimeQuery, [reputationVideoID, 1, 11, 2,"post_reputation-5-uuid-3", submitUserOneHash, 1606240000000, 50, "sponsor", "skip", 0, 0, reputationVideoID]);
+        db.prepare("run", insertSponsorTimeQuery, [reputationVideoID, 1, 11, 2,"post_reputation-5-uuid-4", submitUserOneHash, 1606240000000, 50, "sponsor", "skip", 0, 0, reputationVideoID]);
+        db.prepare("run", insertSponsorTimeQuery, [reputationVideoID, 1, 11, 0,"post_reputation-5-uuid-6", submitUserOneHash, 1606240000000, 50, "sponsor", "skip", 0, 0, reputationVideoID]);
+        db.prepare("run", insertSponsorTimeQuery, [reputationVideoID, 1, 11, 0,"post_reputation-5-uuid-7", submitUserOneHash, 1606240000000, 50, "sponsor", "skip", 0, 0, reputationVideoID]);
+
         const now = Date.now();
         const warnVip01Hash = getHash("warn-vip01-qwertyuiopasdfghjklzxcvbnm");
         const reason01 = "Reason01";
@@ -102,6 +112,9 @@ describe("postSkipSegments", () => {
 
         // ban user
         db.prepare("run", `INSERT INTO "shadowBannedUsers" ("userID") VALUES(?)`, [banUser01Hash]);
+
+        // user feature
+        db.prepare("run", `INSERT INTO "userFeatures" ("userID", "feature", "issuerUserID", "timeSubmitted") VALUES(?, ?, ?, ?)`, [submitUserTwoHash, Feature.ChapterSubmitter, "some-user", 0]);
     });
 
     it("Should be able to submit a single time (Params method)", (done) => {
@@ -189,7 +202,7 @@ describe("postSkipSegments", () => {
             .catch(err => done(err));
     });
 
-    it("Should be able to submit a single chapter (JSON method)", (done) => {
+    it("Should be able to submit a single chapter due to reputation (JSON method)", (done) => {
         const videoID = "postSkipChapter1";
         postSkipSegmentJSON({
             userID: submitUserOne,
@@ -217,6 +230,34 @@ describe("postSkipSegments", () => {
             .catch(err => done(err));
     });
 
+    it("Should be able to submit a single chapter due to user feature (JSON method)", (done) => {
+        const videoID = "postSkipChapter2";
+        postSkipSegmentJSON({
+            userID: submitUserTwo,
+            videoID,
+            segments: [{
+                segment: [0, 10],
+                category: "chapter",
+                actionType: "chapter",
+                description: "This is a chapter"
+            }],
+        })
+            .then(async res => {
+                assert.strictEqual(res.status, 200);
+                const row = await queryDatabaseChapter(videoID);
+                const expected = {
+                    startTime: 0,
+                    endTime: 10,
+                    category: "chapter",
+                    actionType: "chapter",
+                    description: "This is a chapter"
+                };
+                assert.ok(partialDeepEquals(row, expected));
+                done();
+            })
+            .catch(err => done(err));
+    });
+
     it("Should not be able to submit an music_offtopic with mute action type (JSON method)", (done) => {
         const videoID = "postSkip4";
         postSkipSegmentJSON({
@@ -237,8 +278,27 @@ describe("postSkipSegments", () => {
             .catch(err => done(err));
     });
 
+    it("Should not be able to submit a chapter without permission (JSON method)", (done) => {
+        const videoID = "postSkipChapter3";
+        postSkipSegmentJSON({
+            userID: submitUserThree,
+            videoID,
+            segments: [{
+                segment: [0, 10],
+                category: "chapter",
+                actionType: "chapter",
+                description: "This is a chapter"
+            }],
+        })
+            .then(res => {
+                assert.strictEqual(res.status, 400);
+                done();
+            })
+            .catch(err => done(err));
+    });
+
     it("Should not be able to submit a chapter with skip action type (JSON method)", (done) => {
-        const videoID = "postSkipChapter2";
+        const videoID = "postSkipChapter4";
         postSkipSegmentJSON({
             userID: submitUserOne,
             videoID,
@@ -258,7 +318,7 @@ describe("postSkipSegments", () => {
     });
 
     it("Should not be able to submit a sponsor with a description (JSON method)", (done) => {
-        const videoID = "postSkipChapter3";
+        const videoID = "postSkipChapter5";
         postSkipSegmentJSON({
             userID: submitUserOne,
             videoID,