Merge pull request #541 from mchangrh/etagTest

add etag and other tests

ci.json

@@ -56,7 +56,6 @@
             ]
         }
     ],
-    "maxNumberOfActiveWarnings": 3,
     "hoursAfterWarningExpires": 24,
     "rateLimit": {
         "vote": {

package.json

@@ -9,8 +9,8 @@
     "cover:report": "nyc report",
     "dev": "nodemon",
     "dev:bash": "nodemon -x 'npm test ; npm start'",
-    "postgres:docker": "docker run --rm -p 5432:5432 -e POSTGRES_USER=ci_db_user -e POSTGRES_PASSWORD=ci_db_pass postgres:14-alpine",
+    "postgres:docker": "docker run --init -it --rm -p 5432:5432 -e POSTGRES_USER=ci_db_user -e POSTGRES_PASSWORD=ci_db_pass postgres:14-alpine",
-    "redis:docker": "docker run --rm -p 6379:6379 redis:7-alpine --save '' --appendonly no",
+    "redis:docker": "docker run --init -it --rm -p 6379:6379 redis:7-alpine --save '' --appendonly no",
     "start": "ts-node src/index.ts",
     "tsc": "tsc -p tsconfig.json",
     "lint": "eslint src test",

src/middleware/userCounter.ts

@@ -14,7 +14,7 @@ export function userCounter(req: Request, res: Response, next: NextFunction): vo
                 method: "post",
                 url: `${config.userCounterURL}/api/v1/addIP?hashedIP=${getIP(req)}`,
                 httpAgent
-            }).catch(() => Logger.debug(`Failing to connect to user counter at: ${config.userCounterURL}`));
+            }).catch(() => /* instanbul skip next */ Logger.debug(`Failing to connect to user counter at: ${config.userCounterURL}`));
         }
     }
 

src/routes/getSegmentInfo.ts

@@ -34,11 +34,11 @@ async function handleGetSegmentInfo(req: Request, res: Response): Promise<DBSegm
     // deduplicate with set
     UUIDs = [ ...new Set(UUIDs)];
     // if more than 10 entries, slice
-    if (UUIDs.length > 10) UUIDs = UUIDs.slice(0, 10);
+    if (!Array.isArray(UUIDs) || !UUIDs?.length) {
-    if (!Array.isArray(UUIDs) || !UUIDs) {
         res.status(400).send("UUIDs parameter does not match format requirements.");
         return;
     }
+    if (UUIDs.length > 10) UUIDs = UUIDs.slice(0, 10);
     const DBSegments = await getSegmentsByUUID(UUIDs);
     // all uuids failed lookup
     if (!DBSegments?.length) {

src/routes/getSkipSegmentsByHash.ts

@@ -25,13 +25,13 @@ export async function getSkipSegmentsByHash(req: Request, res: Response): Promis
     try {
         await getEtag("skipSegmentsHash", hashPrefix, service)
             .then(etag => res.set("ETag", etag))
-            .catch(() => null);
+            .catch(/* istanbul ignore next */ () => null);
         const output = Object.entries(segments).map(([videoID, data]) => ({
             videoID,
             segments: data.segments,
         }));
         return res.status(output.length === 0 ? 404 : 200).json(output);
-    } catch(e) {
+    } catch (e) /* istanbul ignore next */ {
         Logger.error(`skip segments by hash error: ${e}`);
 
         return res.status(500).send("Internal server error");

src/routes/getTopCategoryUsers.ts

@@ -2,10 +2,12 @@ import { db } from "../databases/databases";
 import { createMemoryCache } from "../utils/createMemoryCache";
 import { config } from "../config";
 import { Request, Response } from "express";
+import { validateCategories } from "../utils/parseParams";
 
 const MILLISECONDS_IN_MINUTE = 60000;
 // eslint-disable-next-line @typescript-eslint/no-misused-promises
 const getTopCategoryUsersWithCache = createMemoryCache(generateTopCategoryUsersStats, config.getTopUsersCacheTimeMinutes * MILLISECONDS_IN_MINUTE);
+/* istanbul ignore next */
 const maxRewardTimePerSegmentInSeconds = config.maxRewardTimePerSegmentInSeconds ?? 86400;
 
 interface DBSegment {
@@ -38,7 +40,6 @@ async function generateTopCategoryUsersStats(sortBy: string, category: string) {
         }
     }
 
-
     return {
         userNames,
         viewCounts,
@@ -51,7 +52,7 @@ export async function getTopCategoryUsers(req: Request, res: Response): Promise<
     const sortType = parseInt(req.query.sortType as string);
     const category = req.query.category as string;
 
-    if (sortType == undefined || !config.categoryList.includes(category) ) {
+    if (sortType == undefined || !validateCategories([category]) ) {
         //invalid request
         return res.sendStatus(400);
     }

src/routes/getTotalStats.ts

@@ -3,6 +3,7 @@ import { config } from "../config";
 import { Request, Response } from "express";
 import axios from "axios";
 import { Logger } from "../utils/logger";
+import { getCWSUsers } from "../utils/getCWSUsers";
 
 // A cache of the number of chrome web store users
 let chromeUsersCache = 0;
@@ -29,30 +30,30 @@ let lastFetch: DBStatsData = {
 updateExtensionUsers();
 
 export async function getTotalStats(req: Request, res: Response): Promise<void> {
-
+    const countContributingUsers = Boolean(req.query?.countContributingUsers == "true");
-    const row = await getStats(!!req.query.countContributingUsers);
+    const row = await getStats(countContributingUsers);
     lastFetch = row;
 
-    if (row !== undefined) {
+    /* istanbul ignore if */
-        const extensionUsers = chromeUsersCache + firefoxUsersCache;
+    if (!row) res.sendStatus(500);
+    const extensionUsers = chromeUsersCache + firefoxUsersCache;
 
-        //send this result
+    //send this result
-        res.send({
+    res.send({
-            userCount: row.userCount,
+        userCount: row.userCount ?? 0,
-            activeUsers: extensionUsers,
+        activeUsers: extensionUsers,
-            apiUsers: Math.max(apiUsersCache, extensionUsers),
+        apiUsers: Math.max(apiUsersCache, extensionUsers),
-            viewCount: row.viewCount,
+        viewCount: row.viewCount,
-            totalSubmissions: row.totalSubmissions,
+        totalSubmissions: row.totalSubmissions,
-            minutesSaved: row.minutesSaved,
+        minutesSaved: row.minutesSaved,
-        });
+    });
 
-        // Check if the cache should be updated (every ~14 hours)
+    // Check if the cache should be updated (every ~14 hours)
-        const now = Date.now();
+    const now = Date.now();
-        if (now - lastUserCountCheck > 5000000) {
+    if (now - lastUserCountCheck > 5000000) {
-            lastUserCountCheck = now;
+        lastUserCountCheck = now;
 
-            updateExtensionUsers();
+        updateExtensionUsers();
-        }
     }
 }
 
@@ -67,42 +68,53 @@ function getStats(countContributingUsers: boolean): Promise<DBStatsData> {
     }
 }
 
-
 function updateExtensionUsers() {
+    /* istanbul ignore else */
     if (config.userCounterURL) {
         axios.get(`${config.userCounterURL}/api/v1/userCount`)
-            .then(res => {
+            .then(res => apiUsersCache = Math.max(apiUsersCache, res.data.userCount))
-                apiUsersCache = Math.max(apiUsersCache, res.data.userCount);
+            .catch( /* istanbul ignore next */ () => Logger.debug(`Failing to connect to user counter at: ${config.userCounterURL}`));
-            })
-            .catch(() => Logger.debug(`Failing to connect to user counter at: ${config.userCounterURL}`));
     }
 
     const mozillaAddonsUrl = "https://addons.mozilla.org/api/v3/addons/addon/sponsorblock/";
     const chromeExtensionUrl = "https://chrome.google.com/webstore/detail/sponsorblock-for-youtube/mnjggcdmjocbbbhaepdhchncahnbgone";
+    const chromeExtId = "mnjggcdmjocbbbhaepdhchncahnbgone";
 
     axios.get(mozillaAddonsUrl)
-        .then(res => {
+        .then(res => firefoxUsersCache = res.data.average_daily_users )
-            firefoxUsersCache = res.data.average_daily_users;
+        .catch( /* istanbul ignore next */ () => {
-            axios.get(chromeExtensionUrl)
-                .then(res => {
-                    const body = res.data;
-                    // 2021-01-05
-                    // [...]<span><meta itemprop="interactionCount" content="UserDownloads:100.000+"/><meta itemprop="opera[...]
-                    const matchingString = '"UserDownloads:';
-                    const matchingStringLen = matchingString.length;
-                    const userDownloadsStartIndex = body.indexOf(matchingString);
-                    if (userDownloadsStartIndex >= 0) {
-                        const closingQuoteIndex = body.indexOf('"', userDownloadsStartIndex + matchingStringLen);
-                        const userDownloadsStr = body.substr(userDownloadsStartIndex + matchingStringLen, closingQuoteIndex - userDownloadsStartIndex).replace(",", "").replace(".", "");
-                        chromeUsersCache = parseInt(userDownloadsStr);
-                    }
-                    else {
-                        lastUserCountCheck = 0;
-                    }
-                })
-                .catch(() => Logger.debug(`Failing to connect to ${chromeExtensionUrl}`));
-        })
-        .catch(() => {
             Logger.debug(`Failing to connect to ${mozillaAddonsUrl}`);
+            return 0;
         });
+    getCWSUsers(chromeExtId)
+        .then(res => chromeUsersCache = res)
+        .catch(/* istanbul ignore next */ () =>
+            getChromeUsers(chromeExtensionUrl)
+                .then(res => chromeUsersCache = res)
+        );
 }
+
+/* istanbul ignore next */
+function getChromeUsers(chromeExtensionUrl: string): Promise<number> {
+    return axios.get(chromeExtensionUrl)
+        .then(res => {
+            const body = res.data;
+            // 2021-01-05
+            // [...]<span><meta itemprop="interactionCount" content="UserDownloads:100.000+"/><meta itemprop="opera[...]
+            const matchingString = '"UserDownloads:';
+            const matchingStringLen = matchingString.length;
+            const userDownloadsStartIndex = body.indexOf(matchingString);
+            /* istanbul ignore else */
+            if (userDownloadsStartIndex >= 0) {
+                const closingQuoteIndex = body.indexOf('"', userDownloadsStartIndex + matchingStringLen);
+                const userDownloadsStr = body.substr(userDownloadsStartIndex + matchingStringLen, closingQuoteIndex - userDownloadsStartIndex).replace(",", "").replace(".", "");
+                return parseInt(userDownloadsStr);
+            } else {
+                lastUserCountCheck = 0;
+            }
+        })
+        .catch(/* istanbul ignore next */ () => {
+            Logger.debug(`Failing to connect to ${chromeExtensionUrl}`);
+            return 0;
+        });
+}

src/routes/postSkipSegments.ts

@@ -120,7 +120,7 @@ async function sendWebhooks(apiVideoDetails: videoDetails, userID: string, video
 //   false for a pass - it was confusing and lead to this bug - any use of this function in
 //   the future could have the same problem.
 async function autoModerateSubmission(apiVideoDetails: videoDetails,
-    submission: { videoID: VideoID; userID: UserID; segments: IncomingSegment[], service: Service, videoDuration: number }) {
+    submission: { videoID: VideoID; userID: HashedUserID; segments: IncomingSegment[], service: Service, videoDuration: number }) {
     // get duration from API
     const apiDuration = apiVideoDetails.duration;
     // if API fail or returns 0, get duration from client
@@ -156,7 +156,7 @@ async function autoModerateSubmission(apiVideoDetails: videoDetails,
     return false;
 }
 
-async function checkUserActiveWarning(userID: string): Promise<CheckResult> {
+async function checkUserActiveWarning(userID: HashedUserID): Promise<CheckResult> {
     const MILLISECONDS_IN_HOUR = 3600000;
     const now = Date.now();
     const warnings = (await db.prepare("all",
@@ -337,10 +337,10 @@ async function checkEachSegmentValid(rawIP: IPAddress, paramUserID: UserID, user
     return CHECK_PASS;
 }
 
-async function checkByAutoModerator(videoID: any, userID: any, segments: Array<any>, service:string, apiVideoDetails: videoDetails, videoDuration: number): Promise<CheckResult> {
+async function checkByAutoModerator(videoID: VideoID, userID: HashedUserID, segments: IncomingSegment[], service: Service, apiVideoDetails: videoDetails, videoDuration: number): Promise<CheckResult> {
     // Auto moderator check
     if (service == Service.YouTube) {
-        const autoModerateResult = await autoModerateSubmission(apiVideoDetails, { userID, videoID, segments, service, videoDuration });
+        const autoModerateResult = await autoModerateSubmission(apiVideoDetails, { videoID, userID, segments, service, videoDuration });
         if (autoModerateResult) {
             return {
                 pass: false,
@@ -492,7 +492,10 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
     let { videoID, userID: paramUserID, service, videoDuration, videoDurationParam, segments, userAgent } = preprocessInput(req);
 
     //hash the userID
-    const userID = await getHashCache(paramUserID || "");
+    if (!paramUserID) {
+        return res.status(400).send("No userID provided");
+    }
+    const userID: HashedUserID = await getHashCache(paramUserID);
 
     const invalidCheckResult = await checkInvalidFields(videoID, paramUserID, userID, segments, videoDurationParam, userAgent, service);
     if (!invalidCheckResult.pass) {

src/utils/createMemoryCache.ts

@@ -1,4 +1,5 @@
 export function createMemoryCache(memoryFn: (...args: any[]) => void, cacheTimeMs: number): any {
+    /* istanbul ignore if */
     if (isNaN(cacheTimeMs)) cacheTimeMs = 0;
 
     // holds the promise results

src/utils/getCWSUsers.ts

@@ -0,0 +1,13 @@
+import axios from "axios";
+import { Logger } from "../utils/logger";
+
+export const getCWSUsers = (extID: string): Promise<number | undefined> =>
+    axios.post(`https://chrome.google.com/webstore/ajax/detail?pv=20210820&id=${extID}`)
+        .then(res => res.data.split("\n")[2])
+        .then(data => JSON.parse(data))
+        .then(data => (data[1][1][0][23]).replaceAll(/,|\+/g,""))
+        .then(data => parseInt(data))
+        .catch((err) => {
+            Logger.error(`Error getting chrome users - ${err}`);
+            return 0;
+        });

src/utils/getHashCache.ts

@@ -28,7 +28,7 @@ async function getFromRedis<T extends string>(key: HashedValue): Promise<T & Has
                 Logger.debug(`Got data from redis: ${reply}`);
                 return reply as T & HashedValue;
             }
-        } catch (err) {
+        } catch (err) /* istanbul ignore next */ {
             Logger.error(err as string);
         }
     }
@@ -37,7 +37,7 @@ async function getFromRedis<T extends string>(key: HashedValue): Promise<T & Has
     const data = getHash(key, cachedHashTimes);
 
     if (!config.redis?.disableHashCache) {
-        redis.set(redisKey, data).catch((err) => Logger.error(err));
+        redis.set(redisKey, data).catch(/* istanbul ignore next */ (err) => Logger.error(err));
     }
 
     return data as T & HashedValue;

src/utils/innerTubeAPI.ts

@@ -70,15 +70,16 @@ export async function getPlayerData (videoID: string, ignoreCache = false): Prom
     }
     try {
         const data = await getFromITube(videoID)
-            .catch(err => {
+            .catch(/* istanbul ignore next */ err => {
                 Logger.warn(`InnerTube API Error for ${videoID}: ${err}`);
                 return Promise.reject(err);
             });
         DiskCache.set(cacheKey, data)
             .then(() => Logger.debug(`InnerTube API: video information cache set for: ${videoID}`))
-            .catch((err: any) => Logger.warn(err));
+            .catch(/* istanbul ignore next */ (err: any) => Logger.warn(err));
         return data;
     } catch (err) {
+        /* istanbul ignore next */
         return Promise.reject(err);
     }
 }

src/utils/isUserTempVIP.ts

@@ -11,7 +11,7 @@ export const isUserTempVIP = async (hashedUserID: HashedUserID, videoID: VideoID
     try {
         const reply = await redis.get(tempVIPKey(hashedUserID));
         return reply && reply == channelID;
-    } catch (e) {
+    } catch (e) /* istanbul ignore next */ {
         Logger.error(e as string);
         return false;
     }

src/utils/logger.ts

@@ -45,6 +45,7 @@ class Logger {
     };
 
     constructor() {
+        /* istanbul ignore if */
         if (config.mode === "development") {
             this._settings.INFO = true;
             this._settings.DEBUG = true;
@@ -73,9 +74,11 @@ class Logger {
 
         let color = colors.Bright;
         if (level === LogLevel.ERROR) color = colors.FgRed;
+        /* istanbul ignore if */
         if (level === LogLevel.WARN) color = colors.FgYellow;
 
         let levelStr = level.toString();
+        /* istanbul ignore if */
         if (levelStr.length === 4) {
             levelStr += " ";  // ensure logs are aligned
         }

src/utils/parseParams.ts

@@ -73,3 +73,6 @@ export const parseActionTypes = (req: Request, fallback: ActionType[]): ActionTy
 
 export const parseRequiredSegments = (req: Request): SegmentUUID[] | undefined =>
     syntaxErrorWrapper(getRequiredSegments, req, []); // never fall back
+
+export const validateCategories = (categories: string[]): boolean =>
+    categories.every((category: string) => config.categoryList.includes(category));

src/utils/redis.ts

@@ -135,17 +135,21 @@ if (config.redis?.enabled) {
             .then((reply) => resolve(reply))
             .catch((err) => reject(err))
     );
+    /* istanbul ignore next */
     client.on("error", function(error) {
         lastClientFail = Date.now();
         Logger.error(`Redis Error: ${error}`);
     });
+    /* istanbul ignore next */
     client.on("reconnect", () => {
         Logger.info("Redis: trying to reconnect");
     });
+    /* istanbul ignore next */
     readClient?.on("error", function(error) {
         lastReadFail = Date.now();
         Logger.error(`Redis Read-Only Error: ${error}`);
     });
+    /* istanbul ignore next */
     readClient?.on("reconnect", () => {
         Logger.info("Redis Read-Only: trying to reconnect");
     });

test.json

@@ -50,7 +50,6 @@
             ]
         }
     ],
-    "maxNumberOfActiveWarnings": 3,
     "hoursAfterWarningExpires": 24,
     "rateLimit": {
         "vote": {

test/cases/eTag.ts

@@ -0,0 +1,65 @@
+import assert from "assert";
+import { client } from "../utils/httpClient";
+import redis from "../../src/utils/redis";
+import { config } from "../../src/config";
+import { genRandom } from "../utils/getRandom";
+
+const validateEtag = (expected: string, actual: string): boolean => {
+    const [actualHashType, actualHashKey, actualService] = actual.split(";");
+    const [expectedHashType, expectedHashKey, expectedService] = expected.split(";");
+    return (actualHashType === expectedHashType) && (actualHashKey === expectedHashKey) && (actualService === expectedService);
+};
+
+describe("eTag", () => {
+    before(function() {
+        if (!config.redis?.enabled) this.skip();
+    });
+
+    const endpoint = "/etag";
+    it("Should reject weak etag", (done) => {
+        const etagKey = `W/test-etag-${genRandom()}`;
+        client.get(endpoint, { headers: { "If-None-Match": etagKey } })
+            .then(res => {
+                assert.strictEqual(res.status, 404);
+                done();
+            })
+            .catch(err => done(err));
+    });
+});
+
+describe("304 etag validation", () => {
+    before(function() {
+        if (!config.redis?.enabled) this.skip();
+    });
+
+    const endpoint = "/etag";
+    for (const hashType of ["skipSegments", "skipSegmentsHash", "videoLabel", "videoLabelHash"]) {
+        it(`${hashType} etag should return 304`, (done) => {
+            const etagKey = `${hashType};${genRandom};YouTube;${Date.now()}`;
+            redis.setEx(etagKey, 8400, "test").then(() =>
+                client.get(endpoint, { headers: { "If-None-Match": etagKey } }).then(res => {
+                    assert.strictEqual(res.status, 304);
+                    const etag = res.headers?.etag ?? "";
+                    assert.ok(validateEtag(etagKey, etag));
+                    done();
+                }).catch(err => done(err))
+            );
+        });
+    }
+
+    it(`other etag type should not return 304`, (done) => {
+        const etagKey = `invalidHashType;${genRandom};YouTube;${Date.now()}`;
+        client.get(endpoint, { headers: { "If-None-Match": etagKey } }).then(res => {
+            assert.strictEqual(res.status, 404);
+            done();
+        }).catch(err => done(err));
+    });
+
+    it(`outdated etag type should not return 304`, (done) => {
+        const etagKey = `skipSegments;${genRandom};YouTube;5000`;
+        client.get(endpoint, { headers: { "If-None-Match": etagKey } }).then(res => {
+            assert.strictEqual(res.status, 404);
+            done();
+        }).catch(err => done(err));
+    });
+});

test/cases/getHashCache.ts

@@ -3,11 +3,9 @@ import { getHashCache } from "../../src/utils/getHashCache";
 import { shaHashKey } from "../../src/utils/redisKeys";
 import { getHash } from "../../src/utils/getHash";
 import redis from "../../src/utils/redis";
-import crypto from "crypto";
 import assert from "assert";
 import { setTimeout } from "timers/promises";
-
+import { genRandom } from "../utils/getRandom";
-const genRandom = (bytes=8) => crypto.pseudoRandomBytes(bytes).toString("hex");
 
 const rand1Hash = genRandom(24);
 const rand1Hash_Key = getHash(rand1Hash, 1);

test/cases/getSegmentInfo.ts

@@ -338,4 +338,13 @@ describe("getSegmentInfo", () => {
             })
             .catch(err => done(err));
     });
+
+    it("Should return 400 if no UUIDs not sent", (done) => {
+        client.get(endpoint)
+            .then(res => {
+                if (res.status !== 400) done(`non 400 response code: ${res.status}`);
+                else done(); // pass
+            })
+            .catch(err => done(err));
+    });
 });

test/cases/getSkipSegments.ts

@@ -486,4 +486,13 @@ describe("getSkipSegments", () => {
             })
             .catch(err => done(err));
     });
+
+    it("Should get 400 for invalid category type", (done) => {
+        client.get(endpoint, { params: { videoID: "getSkipSegmentID0", category: 1 } })
+            .then(res => {
+                assert.strictEqual(res.status, 400);
+                done();
+            })
+            .catch(err => done(err));
+    });
 });

test/cases/getTopCategoryUsers.ts

@@ -29,7 +29,7 @@ describe("getTopCategoryUsers", () => {
             .catch(err => done(err));
     });
 
-    it("Should return 400 if invalid sortType provided", (done) => {
+    it("Should return 400 if invalid type of sortType provided", (done) => {
         client.get(endpoint, { params: { sortType: "a" } })
             .then(res => {
                 assert.strictEqual(res.status, 400);
@@ -38,6 +38,15 @@ describe("getTopCategoryUsers", () => {
             .catch(err => done(err));
     });
 
+    it("Should return 400 if invalid sortType number provided", (done) => {
+        client.get(endpoint, { params: { sortType: 15, category: "sponsor" } })
+            .then(res => {
+                assert.strictEqual(res.status, 400);
+                done();
+            })
+            .catch(err => done(err));
+    });
+
     it("Should return 400 if invalid category provided", (done) => {
         client.get(endpoint, { params: { sortType: 1, category: "never_valid_category" } })
             .then(res => {
@@ -121,4 +130,16 @@ describe("getTopCategoryUsers", () => {
             })
             .catch(err => done(err));
     });
+
+    it("Should return no time saved for chapters", (done) => {
+        client.get(endpoint, { params: { sortType: 2, category: "chapter" } })
+            .then(res => {
+                assert.strictEqual(res.status, 200);
+                for (const timeSaved of res.data.minutesSaved) {
+                    assert.strictEqual(timeSaved, 0, "Time saved should be 0");
+                }
+                done();
+            })
+            .catch(err => done(err));
+    });
 });

test/cases/getTopUsers.ts

@@ -81,4 +81,14 @@ describe("getTopUsers", () => {
             })
             .catch(err => done(err));
     });
+
+    it("Should be able to get cached result", (done) => {
+        client.get(endpoint, { params: { sortType: 0 } })// minutesSaved
+            .then(res => {
+                assert.strictEqual(res.status, 200);
+                assert.ok(res.data.userNames.indexOf(user1) < res.data.userNames.indexOf(user2), `Actual Order: ${res.data.userNames}`);
+                done();
+            })
+            .catch(err => done(err));
+    });
 });

test/cases/getTotalStats.ts

@@ -7,7 +7,29 @@ describe("getTotalStats", () => {
     it("Can get total stats", async () => {
         const result = await client({ url: endpoint });
         const data = result.data;
-        assert.ok(data?.userCount ?? true);
+        assert.strictEqual(data.userCount, 0, "User count should default false");
+        assert.ok(data.activeUsers >= 0);
+        assert.ok(data.apiUsers >= 0);
+        assert.ok(data.viewCount >= 0);
+        assert.ok(data.totalSubmissions >= 0);
+        assert.ok(data.minutesSaved >= 0);
+    });
+
+    it("Can get total stats without contributing users", async () => {
+        const result = await client({ url: `${endpoint}?countContributingUsers=false` });
+        const data = result.data;
+        assert.strictEqual(data.userCount, 0);
+        assert.ok(data.activeUsers >= 0);
+        assert.ok(data.apiUsers >= 0);
+        assert.ok(data.viewCount >= 0);
+        assert.ok(data.totalSubmissions >= 0);
+        assert.ok(data.minutesSaved >= 0);
+    });
+
+    it("Can get total stats with contributing users", async () => {
+        const result = await client({ url: `${endpoint}?countContributingUsers=true` });
+        const data = result.data;
+        assert.ok(data.userCount >= 0);
         assert.ok(data.activeUsers >= 0);
         assert.ok(data.apiUsers >= 0);
         assert.ok(data.viewCount >= 0);

test/cases/highLoad.ts

@@ -0,0 +1,36 @@
+import sinon from "sinon";
+import { db } from "../../src/databases/databases";
+import assert from "assert";
+import { client } from "../utils/httpClient";
+client.defaults.validateStatus = (status) => status < 600;
+
+describe("High load test", () => {
+    before(() => {
+        sinon.stub(db, "highLoad").returns(true);
+    });
+
+    after(() => {
+        sinon.restore();
+    });
+
+    it("Should return 503 on getTopUsers", async () => {
+        await client.get("/api/getTopUsers?sortType=0")
+            .then(res => {
+                assert.strictEqual(res.status, 503);
+            });
+    });
+
+    it("Should return 503 on getTopCategoryUsers", async () => {
+        await client.get("/api/getTopCategoryUsers?sortType=0&category=sponsor")
+            .then(res => {
+                assert.strictEqual(res.status, 503);
+            });
+    });
+
+    it("Should return 0 on getTotalStats", async () => {
+        await client.get("/api/getTotalStats")
+            .then(res => {
+                assert.strictEqual(res.status, 200);
+            });
+    });
+});

test/cases/postSkipSegments400.ts

@@ -0,0 +1,316 @@
+import assert from "assert";
+import { postSkipSegmentJSON, postSkipSegmentParam } from "./postSkipSegments";
+
+const videoID = "postSkipSegments-404-video";
+const userID = "postSkipSegments-404-user";
+
+describe("postSkipSegments 400 - missing params", () => {
+    it("Should return 400 for missing params (JSON method) 1", (done) => {
+        postSkipSegmentJSON({
+            userID,
+            segments: [{
+                segment: [9, 10],
+                category: "sponsor",
+            }, {
+                segment: [31, 60],
+                category: "intro",
+            }],
+        })
+            .then(res => {
+                assert.strictEqual(res.status, 400);
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should return 400 for missing params (JSON method) 2", (done) => {
+        postSkipSegmentJSON({
+            userID,
+            videoID,
+        })
+            .then(res => {
+                assert.strictEqual(res.status, 400);
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should return 400 for missing params (JSON method) 3", (done) => {
+        postSkipSegmentJSON({
+            userID,
+            videoID,
+            segments: [{
+                segment: [0],
+                category: "sponsor",
+            }, {
+                segment: [31, 60],
+                category: "intro",
+            }],
+        })
+            .then(res => {
+                assert.strictEqual(res.status, 400);
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should return 400 for missing params (JSON method) 4", (done) => {
+        postSkipSegmentJSON({
+            userID,
+            videoID,
+            segments: [{
+                segment: [9, 10],
+            }, {
+                segment: [31, 60],
+                category: "intro",
+            }],
+        })
+            .then(res => {
+                assert.strictEqual(res.status, 400);
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should return 400 for missing params (JSON method) 5", (done) => {
+        postSkipSegmentJSON({
+            userID,
+            videoID,
+        })
+            .then(res => {
+                assert.strictEqual(res.status, 400);
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should return 400 for missing multiple params (Params method)", (done) => {
+        postSkipSegmentParam({
+            startTime: 9,
+            endTime: 10,
+            userID
+        })
+            .then(res => {
+                if (res.status === 400) done();
+                else done(true);
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should return 400 if videoID is empty", (done) => {
+        const videoID = null as unknown as string;
+        postSkipSegmentParam({
+            videoID,
+            startTime: 1,
+            endTime: 5,
+            category: "sponsor",
+            userID
+        })
+            .then(res => {
+                assert.strictEqual(res.status, 400);
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should return 400 if no segments provided", (done) => {
+        postSkipSegmentJSON({
+            videoID,
+            segments: [],
+            category: "sponsor",
+            userID
+        })
+            .then(res => {
+                assert.strictEqual(res.status, 400);
+                done();
+            })
+            .catch(err => done(err));
+    });
+});
+
+describe("postSkipSegments 400 - Chapters", () => {
+    const actionType = "chapter";
+    const category = actionType;
+
+    it("Should not be able to submit a chapter name that is too long", (done) => {
+        postSkipSegmentParam({
+            videoID,
+            startTime: 1,
+            endTime: 5,
+            category,
+            actionType,
+            description: "a".repeat(256),
+            userID
+        })
+            .then(res => {
+                assert.strictEqual(res.status, 400);
+                done();
+            })
+            .catch(err => done(err));
+    });
+});
+
+describe("postSkipSegments 400 - POI", () => {
+    const category = "poi_highlight";
+    it("Should be rejected if a POI is at less than 1 second", (done) => {
+        postSkipSegmentParam({
+            videoID,
+            startTime: 0.5,
+            endTime: 0.5,
+            category,
+            userID
+        })
+            .then(res => {
+                assert.strictEqual(res.status, 400);
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should be rejected if highlight segment doesn't start and end at the same time", (done) => {
+        postSkipSegmentParam({
+            videoID,
+            startTime: 30,
+            endTime: 30.5,
+            category,
+            userID
+        })
+            .then(res => {
+                assert.strictEqual(res.status, 400);
+                done();
+            })
+            .catch(err => done(err));
+    });
+});
+
+describe("postSkipSegments 400 - Automod", () => {
+    it("Should be rejected if over 80% of the video", (done) => {
+        postSkipSegmentParam({
+            videoID,
+            startTime: 30,
+            endTime: 1000000,
+            userID,
+            category: "sponsor"
+        })
+            .then(res => {
+                assert.strictEqual(res.status, 403);
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should be rejected if a sponsor is less than 1 second", (done) => {
+        postSkipSegmentParam({
+            videoID,
+            category: "sponsor",
+            startTime: 30,
+            endTime: 30.5,
+            userID
+        })
+            .then(res => {
+                assert.strictEqual(res.status, 400);
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should be rejected if non-POI segment starts and ends at the same time", (done) => {
+        postSkipSegmentParam({
+            videoID,
+            startTime: 90,
+            endTime: 90,
+            userID,
+            category: "intro"
+        })
+            .then(res => {
+                assert.strictEqual(res.status, 400);
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should not allow submitting full video not at zero seconds", (done) => {
+        postSkipSegmentParam({
+            videoID,
+            startTime: 0,
+            endTime: 1,
+            category: "sponsor",
+            actionType: "full",
+            userID
+        })
+            .then(res => {
+                assert.strictEqual(res.status, 400);
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should not be able to submit an music_offtopic with mute action type (JSON method)", (done) => {
+        postSkipSegmentJSON({
+            userID,
+            videoID,
+            segments: [{
+                segment: [0, 10],
+                category: "music_offtopic",
+                actionType: "mute"
+            }],
+        })
+            .then(res => {
+                assert.strictEqual(res.status, 400);
+                done();
+            })
+            .catch(err => done(err));
+    });
+});
+
+describe("postSkipSegments 400 - Mismatched Types", () => {
+    it("Should not be able to submit with a category that does not exist", (done) => {
+        postSkipSegmentParam({
+            videoID,
+            startTime: 1,
+            endTime: 5,
+            category: "this-category-will-never-exist",
+            userID
+        })
+            .then(res => {
+                assert.strictEqual(res.status, 400);
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should not be able to submit a chapter with skip action type (JSON method)", (done) => {
+        postSkipSegmentJSON({
+            userID,
+            videoID,
+            segments: [{
+                segment: [0, 10],
+                category: "chapter",
+                actionType: "skip"
+            }],
+        })
+            .then(res => {
+                assert.strictEqual(res.status, 400);
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should not be able to submit a sponsor with a description (JSON method)", (done) => {
+        const videoID = "postSkipChapter5";
+        postSkipSegmentJSON({
+            userID,
+            videoID,
+            segments: [{
+                segment: [0, 10],
+                category: "sponsor",
+                description: "This is a sponsor"
+            }],
+        })
+            .then(res => {
+                assert.strictEqual(res.status, 400);
+                done();
+            })
+            .catch(err => done(err));
+    });
+});

test/cases/postSkipSegments400Stub.ts

@@ -0,0 +1,32 @@
+import assert from "assert";
+import { postSkipSegmentParam } from "./postSkipSegments";
+import { config } from "../../src/config";
+import sinon from "sinon";
+
+const videoID = "postSkipSegments-404-video";
+
+describe("postSkipSegments 400 - stubbed config", () => {
+    const USERID_LIMIT = 30;
+    before(() => {
+        sinon.stub(config, "minUserIDLength").value(USERID_LIMIT);
+    });
+    after(() => {
+        sinon.restore();
+    });
+
+    it("Should return 400 if userID is too short", (done) => {
+        const userID = "a".repeat(USERID_LIMIT - 10);
+        postSkipSegmentParam({
+            videoID,
+            startTime: 1,
+            endTime: 5,
+            category: "sponsor",
+            userID
+        })
+            .then(res => {
+                assert.strictEqual(res.status, 400);
+                done();
+            })
+            .catch(err => done(err));
+    });
+});

test/cases/postSkipSegmentsAutomod.ts

@@ -0,0 +1,121 @@
+import { getHash } from "../../src/utils/getHash";
+import { db } from "../../src/databases/databases";
+import assert from "assert";
+import { arrayDeepEquals } from "../utils/partialDeepEquals";
+import { postSkipSegmentJSON, convertMultipleToDBFormat } from "./postSkipSegments";
+import { YouTubeApiMock } from "../mocks/youtubeMock";
+import { ImportMock } from "ts-mock-imports";
+import * as YouTubeAPIModule from "../../src/utils/youtubeApi";
+
+const mockManager = ImportMock.mockStaticClass(YouTubeAPIModule, "YouTubeAPI");
+const sinonStub = mockManager.mock("listVideos");
+sinonStub.callsFake(YouTubeApiMock.listVideos);
+
+describe("postSkipSegments - Automod 80%", () => {
+    const userID = "postSkipSegments-automodSubmit";
+    const userIDHash = getHash(userID);
+
+    const over80VideoID = "80percent_video";
+
+    const queryDatabaseCategory = (videoID: string) => db.prepare("all", `SELECT "startTime", "endTime", "category" FROM "sponsorTimes" WHERE "videoID" = ? and "votes" > -1`, [videoID]);
+
+    before(() => {
+        const insertSponsorTimeQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "actionType", "videoDuration", "shadowHidden", "hashedVideoID") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)';
+        db.prepare("run", insertSponsorTimeQuery, [over80VideoID, 0, 1000, 0, "80percent-uuid-0", userIDHash, 0, 0, "interaction", "skip", 0, 0, over80VideoID]);
+        db.prepare("run", insertSponsorTimeQuery, [over80VideoID, 1001, 1005, 0, "80percent-uuid-1", userIDHash, 0, 0, "interaction", "skip", 0, 0, over80VideoID]);
+        db.prepare("run", insertSponsorTimeQuery, [over80VideoID, 0, 5000, -2, "80percent-uuid-2", userIDHash, 0, 0, "interaction", "skip", 0, 0, over80VideoID]);
+    });
+
+    it("Should allow multiple times if total is under 80% of video (JSON method)", (done) => {
+        const videoID = "postSkipSegments_80percent_video_blank1";
+        const segments = [{
+            segment: [3, 3000],
+            category: "sponsor",
+        }, {
+            segment: [3002, 3050],
+            category: "intro",
+        }, {
+            segment: [45, 100],
+            category: "interaction",
+        }, {
+            segment: [99, 170],
+            category: "sponsor",
+        }];
+        postSkipSegmentJSON({
+            userID,
+            videoID,
+            segments
+        })
+            .then(async res => {
+                assert.strictEqual(res.status, 200);
+                const rows = await queryDatabaseCategory(videoID);
+                const expected = convertMultipleToDBFormat(segments);
+                assert.ok(arrayDeepEquals(rows, expected));
+                done();
+            })
+            .catch(err => done(err));
+    }).timeout(5000);
+
+    it("Should reject multiple times if total is over 80% of video (JSON method)", (done) => {
+        const videoID = "postSkipSegments_80percent_video_blank2";
+        const segments = [{
+            segment: [0, 2000],
+            category: "interaction",
+        }, {
+            segment: [3000, 4000],
+            category: "sponsor",
+        }, {
+            segment: [1500, 2750],
+            category: "sponsor",
+        }, {
+            segment: [4050, 4750],
+            category: "intro",
+        }];
+        postSkipSegmentJSON({
+            userID,
+            videoID,
+            segments
+        })
+            .then(async res => {
+                assert.strictEqual(res.status, 403);
+                const rows = await queryDatabaseCategory(videoID);
+                assert.deepStrictEqual(rows, []);
+                done();
+            })
+            .catch(err => done(err));
+    }).timeout(5000);
+
+    it("Should reject multiple times if total is over 80% of video including previosuly submitted times (JSON method)", (done) => {
+        const segments = [{
+            segment: [2000, 4000], // adds 2000
+            category: "sponsor",
+        }, {
+            segment: [1500, 2750], // adds 500
+            category: "sponsor",
+        }, {
+            segment: [4050, 4570], // adds 520
+            category: "sponsor",
+        }];
+        const expected = [{
+            startTime: 0,
+            endTime: 1000,
+            category: "interaction"
+        }, {
+            startTime: 1001,
+            endTime: 1005,
+            category: "interaction"
+        }];
+        postSkipSegmentJSON({
+            userID,
+            videoID: over80VideoID,
+            segments: segments
+        })
+            .then(async res => {
+                assert.strictEqual(res.status, 403);
+                const rows = await queryDatabaseCategory(over80VideoID);
+                assert.ok(arrayDeepEquals(rows, expected, true));
+                done();
+            })
+            .catch(err => done(err));
+    }).timeout(5000);
+});

test/cases/postSkipSegmentsDuration.ts

@@ -0,0 +1,205 @@
+import assert from "assert";
+import { postSkipSegmentJSON, postSkipSegmentParam } from "./postSkipSegments";
+import { getHash } from "../../src/utils/getHash";
+import { partialDeepEquals } from "../utils/partialDeepEquals";
+import { db } from "../../src/databases/databases";
+import { ImportMock } from "ts-mock-imports";
+import * as YouTubeAPIModule from "../../src/utils/youtubeApi";
+import { YouTubeApiMock } from "../mocks/youtubeMock";
+import { convertSingleToDBFormat } from "./postSkipSegments";
+
+const mockManager = ImportMock.mockStaticClass(YouTubeAPIModule, "YouTubeAPI");
+const sinonStub = mockManager.mock("listVideos");
+sinonStub.callsFake(YouTubeApiMock.listVideos);
+
+describe("postSkipSegments - duration", () => {
+    const userIDOne = "postSkip-DurationUserOne";
+    const userIDTwo = "postSkip-DurationUserTwo";
+    const videoID = "postSkip-DurationVideo";
+    const noDurationVideoID = "noDuration";
+    const userID = userIDOne;
+
+    const queryDatabaseDuration = (videoID: string) => db.prepare("get", `SELECT "startTime", "endTime", "locked", "category", "videoDuration" FROM "sponsorTimes" WHERE "videoID" = ?`, [videoID]);
+
+    before(() => {
+        const insertSponsorTimeQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "actionType", "videoDuration", "shadowHidden", "hashedVideoID") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)';
+        db.prepare("run", insertSponsorTimeQuery, ["full_video_duration_segment", 0, 0, 0, "full-video-duration-uuid-0", userIDTwo, 0, 0, "sponsor", "full", 123, 0, "full_video_duration_segment"]);
+        db.prepare("run", insertSponsorTimeQuery, ["full_video_duration_segment", 25, 30, 0, "full-video-duration-uuid-1", userIDTwo, 0, 0, "sponsor", "skip", 123, 0, "full_video_duration_segment"]);
+    });
+
+    it("Should be able to submit a single time with a precise duration close to the one from the YouTube API (JSON method)", (done) => {
+        const segment = {
+            segment: [1, 10],
+            category: "sponsor",
+        };
+        postSkipSegmentJSON({
+            userID,
+            videoID,
+            videoDuration: 4980.20,
+            segments: [segment],
+        })
+            .then(async res => {
+                assert.strictEqual(res.status, 200);
+                const row = await queryDatabaseDuration(videoID);
+                const expected = {
+                    ...convertSingleToDBFormat(segment),
+                    locked: 0,
+                    videoDuration: 4980.20,
+                };
+                assert.ok(partialDeepEquals(row, expected));
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should be able to submit a single time with a duration in the body (JSON method)", (done) => {
+        const videoID = "noDuration";
+        const segment = {
+            segment: [0, 10],
+            category: "sponsor",
+        };
+        postSkipSegmentJSON({
+            userID,
+            videoID,
+            videoDuration: 100,
+            segments: [segment],
+        })
+            .then(async res => {
+                assert.strictEqual(res.status, 200);
+                const row = await queryDatabaseDuration(videoID);
+                const expected = {
+                    ...convertSingleToDBFormat(segment),
+                    locked: 0,
+                    videoDuration: 100,
+                };
+                assert.ok(partialDeepEquals(row, expected));
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should be able to submit with a new duration, and hide old submissions and remove segment locks", async () => {
+        const videoID = "noDuration";
+        const segment = {
+            segment: [1, 10],
+            category: "sponsor",
+        };
+        await db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category")
+            VALUES(?, ?, ?)`, [getHash("generic-VIP"), videoID, "sponsor"]);
+        try {
+            const res = await postSkipSegmentJSON({
+                userID,
+                videoID,
+                videoDuration: 100,
+                segments: [segment],
+            });
+            assert.strictEqual(res.status, 200);
+            const lockCategoriesRow = await db.prepare("get", `SELECT * from "lockCategories" WHERE videoID = ?`, [videoID]);
+            const videoRows = await db.prepare("all", `SELECT "startTime", "endTime", "locked", "category", "videoDuration"
+                FROM "sponsorTimes" WHERE "videoID" = ? AND hidden = 0`, [videoID]);
+            const hiddenVideoRows = await db.prepare("all", `SELECT "startTime", "endTime", "locked", "category", "videoDuration"
+                FROM "sponsorTimes" WHERE "videoID" = ? AND hidden = 1`, [videoID]);
+            assert.ok(!lockCategoriesRow);
+            const expected = {
+                ...convertSingleToDBFormat(segment),
+                locked: 0,
+                videoDuration: 100,
+            };
+            assert.ok(partialDeepEquals(videoRows[0], expected));
+            assert.strictEqual(videoRows.length, 1);
+            assert.strictEqual(hiddenVideoRows.length, 1);
+        } catch (e) {
+            return e;
+        }
+    });
+
+    it("Should still not be allowed if youtube thinks duration is 0", (done) => {
+        postSkipSegmentJSON({
+            userID,
+            videoID: noDurationVideoID,
+            videoDuration: 100,
+            segments: [{
+                segment: [30, 10000],
+                category: "sponsor",
+            }],
+        })
+            .then(res => {
+                assert.strictEqual(res.status, 403);
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should be able to submit with a new duration, and not hide full video segments", async () => {
+        const videoID = "full_video_duration_segment";
+        const segment = {
+            segment: [20, 30],
+            category: "sponsor",
+        };
+        const res = await postSkipSegmentJSON({
+            userID,
+            videoID,
+            videoDuration: 100,
+            segments: [segment],
+        });
+        assert.strictEqual(res.status, 200);
+        const videoRows = await db.prepare("all", `SELECT "startTime", "endTime", "locked", "category", "actionType", "videoDuration"
+            FROM "sponsorTimes" WHERE "videoID" = ? AND hidden = 0`, [videoID]);
+        const hiddenVideoRows = await db.prepare("all", `SELECT "startTime", "endTime", "locked", "category", "videoDuration"
+            FROM "sponsorTimes" WHERE "videoID" = ? AND hidden = 1`, [videoID]);
+        assert.strictEqual(videoRows.length, 2);
+        const expected = {
+            ...convertSingleToDBFormat(segment),
+            locked: 0,
+            videoDuration: 100
+        };
+        const fullExpected = {
+            category: "sponsor",
+            actionType: "full"
+        };
+        assert.ok((partialDeepEquals(videoRows[0], fullExpected) && partialDeepEquals(videoRows[1], expected))
+            || (partialDeepEquals(videoRows[1], fullExpected) && partialDeepEquals(videoRows[0], expected)));
+        assert.strictEqual(hiddenVideoRows.length, 1);
+    });
+
+    it("Should be able to submit a single time with a duration from the YouTube API (JSON method)", (done) => {
+        const segment = {
+            segment: [0, 10],
+            category: "sponsor",
+        };
+        const videoID = "postDuration-ytjson";
+        postSkipSegmentJSON({
+            userID,
+            videoID,
+            videoDuration: 100,
+            segments: [segment],
+        })
+            .then(async res => {
+                assert.strictEqual(res.status, 200);
+                const row = await queryDatabaseDuration(videoID);
+                const expected = {
+                    ...convertSingleToDBFormat(segment),
+                    videoDuration: 4980,
+                };
+                assert.ok(partialDeepEquals(row, expected));
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should successfully submit if video is private", (done) => {
+        const videoID = "private-video";
+        postSkipSegmentParam({
+            videoID,
+            startTime: 1,
+            endTime: 5,
+            category: "sponsor",
+            userID
+        })
+            .then(res => {
+                assert.strictEqual(res.status, 200);
+                done();
+            })
+            .catch(err => done(err));
+    });
+});

test/cases/postSkipSegmentsFeatures.ts

@@ -0,0 +1,84 @@
+import { getHash } from "../../src/utils/getHash";
+import { db } from "../../src/databases/databases";
+import assert from "assert";
+import { partialDeepEquals } from "../utils/partialDeepEquals";
+import { genRandom } from "../utils/getRandom";
+import { Feature } from "../../src/types/user.model";
+import { Segment, postSkipSegmentJSON, convertSingleToDBFormat } from "./postSkipSegments";
+
+describe("postSkipSegments Features - Chapters", () => {
+    const submitUser_noPermissions = "postSkipSegments-chapters-noperm";
+    const submitUser_reputation = "postSkipSegments-chapters-reputation";
+    const submitUser_feature = "postSkipSegments-chapters-feature";
+
+    const queryDatabaseChapter = (videoID: string) => db.prepare("get", `SELECT "startTime", "endTime", "category", "actionType", "description" FROM "sponsorTimes" WHERE "videoID" = ?`, [videoID]);
+    function createSegment(): Segment {
+        return {
+            segment: [0, 10],
+            category: "chapter",
+            actionType: "chapter",
+            description: genRandom()
+        };
+    }
+
+    before(() => {
+        const submitNumberOfTimes = 10;
+        const submitUser_reputationHash = getHash(submitUser_reputation);
+        const insertSponsorTimeQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "actionType", "shadowHidden") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)';
+        for (let i = 0; i < submitNumberOfTimes; i++) {
+            const uuid = `post_reputation_uuid-${i}`;
+            const videoID = `post_reputation_video-${i}`;
+            db.prepare("run", insertSponsorTimeQuery, [videoID, 1, 11, 5, 1, uuid, submitUser_reputationHash, 1597240000000, 50, "sponsor", "skip", 0]);
+        }
+        // user feature
+        db.prepare("run", `INSERT INTO "userFeatures" ("userID", "feature", "issuerUserID", "timeSubmitted") VALUES(?, ?, ?, ?)`, [getHash(submitUser_feature), Feature.ChapterSubmitter, "generic-VIP", 0]);
+    });
+
+    it("Should be able to submit a single chapter due to reputation (JSON method)", (done) => {
+        const segment = createSegment();
+        const videoID = "postSkipSegments-chapter-reputation";
+        postSkipSegmentJSON({
+            userID: submitUser_reputation,
+            videoID,
+            segments: [segment]
+        })
+            .then(async res => {
+                assert.strictEqual(res.status, 200);
+                const row = await queryDatabaseChapter(videoID);
+                assert.ok(partialDeepEquals(row, convertSingleToDBFormat(segment)));
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should be able to submit a single chapter due to user feature (JSON method)", (done) => {
+        const segment = createSegment();
+        const videoID = "postSkipSegments-chapter-feature";
+        postSkipSegmentJSON({
+            userID: submitUser_feature,
+            videoID,
+            segments: [segment]
+        })
+            .then(async res => {
+                assert.strictEqual(res.status, 200);
+                const row = await queryDatabaseChapter(videoID);
+                assert.ok(partialDeepEquals(row, convertSingleToDBFormat(segment)));
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should not be able to submit a chapter without permission (JSON method)", (done) => {
+        const videoID = "postSkipSegments-chapter-submit";
+        postSkipSegmentJSON({
+            userID: submitUser_noPermissions,
+            videoID,
+            segments: [createSegment()]
+        })
+            .then(res => {
+                assert.strictEqual(res.status, 400);
+                done();
+            })
+            .catch(err => done(err));
+    });
+});

test/cases/postSkipSegmentsLocked.ts

@@ -0,0 +1,70 @@
+import assert from "assert";
+import { postSkipSegmentJSON } from "./postSkipSegments";
+import { getHash } from "../../src/utils/getHash";
+import { db } from "../../src/databases/databases";
+
+describe("postSkipSegments - LockedVideos", () => {
+    const userIDOne = "postSkip-DurationUserOne";
+    const VIPLockUser = "VIPUser-lockCategories";
+    const videoID = "lockedVideo";
+    const userID = userIDOne;
+
+    before(() => {
+        const insertLockCategoriesQuery = `INSERT INTO "lockCategories" ("userID", "videoID", "category", "reason") VALUES(?, ?, ?, ?)`;
+        db.prepare("run", insertLockCategoriesQuery, [getHash(VIPLockUser), videoID, "sponsor", "Custom Reason"]);
+        db.prepare("run", insertLockCategoriesQuery, [getHash(VIPLockUser), videoID, "intro", ""]);
+    });
+
+    it("Should return 403 and custom reason for submiting in lockedCategory", (done) => {
+        postSkipSegmentJSON({
+            userID,
+            videoID,
+            segments: [{
+                segment: [1, 10],
+                category: "sponsor",
+            }],
+        })
+            .then(res => {
+                assert.strictEqual(res.status, 403);
+                assert.match(res.data, /Reason: /);
+                assert.match(res.data, /Custom Reason/);
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should return not be 403 when submitting with locked category but unlocked actionType", (done) => {
+        postSkipSegmentJSON({
+            userID,
+            videoID,
+            segments: [{
+                segment: [1, 10],
+                category: "sponsor",
+                actionType: "mute"
+            }],
+        })
+            .then(res => {
+                assert.strictEqual(res.status, 200);
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should return 403 for submiting in lockedCategory", (done) => {
+        postSkipSegmentJSON({
+            userID,
+            videoID,
+            segments: [{
+                segment: [1, 10],
+                category: "intro",
+            }],
+        })
+            .then(res => {
+                assert.strictEqual(res.status, 403);
+                assert.doesNotMatch(res.data, /Lock reason: /);
+                assert.doesNotMatch(res.data, /Custom Reason/);
+                done();
+            })
+            .catch(err => done(err));
+    });
+});

test/cases/postSkipSegmentsShadowban.ts

@@ -0,0 +1,68 @@
+import assert from "assert";
+import { postSkipSegmentParam } from "./postSkipSegments";
+import { getHash } from "../../src/utils/getHash";
+import { db } from "../../src/databases/databases";
+import { ImportMock } from "ts-mock-imports";
+import * as YouTubeAPIModule from "../../src/utils/youtubeApi";
+import { YouTubeApiMock } from "../mocks/youtubeMock";
+
+const mockManager = ImportMock.mockStaticClass(YouTubeAPIModule, "YouTubeAPI");
+const sinonStub = mockManager.mock("listVideos");
+sinonStub.callsFake(YouTubeApiMock.listVideos);
+
+describe("postSkipSegments - shadowban", () => {
+    const banUser01 = "postSkip-banUser01";
+    const banUser01Hash = getHash(banUser01);
+
+    const shadowBanVideoID1 = "postSkipBan1";
+    const shadowBanVideoID2 = "postSkipBan2";
+
+    const queryDatabaseShadowhidden = (videoID: string) => db.prepare("get", `SELECT "startTime", "endTime", "shadowHidden", "userID" FROM "sponsorTimes" WHERE "videoID" = ?`, [videoID]);
+
+    before(() => {
+        db.prepare("run", `INSERT INTO "shadowBannedUsers" ("userID") VALUES(?)`, [banUser01Hash]);
+    });
+
+    it("Should automatically shadowban segments if user is banned", (done) => {
+        const videoID = shadowBanVideoID1;
+        postSkipSegmentParam({
+            videoID,
+            startTime: 0,
+            endTime: 10,
+            category: "sponsor",
+            userID: banUser01
+        })
+            .then(async res => {
+                assert.strictEqual(res.status, 200);
+                const row = await queryDatabaseShadowhidden(videoID);
+                const expected = {
+                    startTime: 0,
+                    endTime: 10,
+                    shadowHidden: 1,
+                    userID: banUser01Hash
+                };
+                assert.deepStrictEqual(row, expected);
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should not add full segments to database if user if shadowbanned", (done) => {
+        const videoID = shadowBanVideoID2;
+        postSkipSegmentParam({
+            videoID,
+            startTime: 0,
+            endTime: 0,
+            category: "sponsor",
+            actionType: "full",
+            userID: banUser01
+        })
+            .then(async res => {
+                assert.strictEqual(res.status, 200);
+                const row = await queryDatabaseShadowhidden(videoID);
+                assert.strictEqual(row, undefined);
+                done();
+            })
+            .catch(err => done(err));
+    });
+});

test/cases/postSkipSegmentsUserAgent.ts

@@ -0,0 +1,104 @@
+import assert from "assert";
+import { convertSingleToDBFormat } from "./postSkipSegments";
+import { getHash } from "../../src/utils/getHash";
+import { db } from "../../src/databases/databases";
+import { partialDeepEquals } from "../utils/partialDeepEquals";
+import { client } from "../utils/httpClient";
+
+const endpoint = "/api/skipSegments";
+
+const queryUseragent = (videoID: string) => db.prepare("get", `SELECT "startTime", "endTime", "locked", "category", "userAgent" FROM "sponsorTimes" WHERE "videoID" = ?`, [videoID]);
+
+describe("postSkipSegments - userAgent", () => {
+    const userIDOne = "postSkip-DurationUserOne";
+    const VIPLockUser = "VIPUser-lockCategories";
+    const videoID = "lockedVideo";
+    const userID = userIDOne;
+
+    const segment = {
+        segment: [0, 10],
+        category: "sponsor",
+    };
+    const dbFormatSegment = convertSingleToDBFormat(segment);
+
+    before(() => {
+        const insertLockCategoriesQuery = `INSERT INTO "lockCategories" ("userID", "videoID", "category", "reason") VALUES(?, ?, ?, ?)`;
+        db.prepare("run", insertLockCategoriesQuery, [getHash(VIPLockUser), videoID, "sponsor", "Custom Reason"]);
+        db.prepare("run", insertLockCategoriesQuery, [getHash(VIPLockUser), videoID, "intro", ""]);
+    });
+
+    it("Should be able to submit with empty user-agent", (done) => {
+        const videoID = "userAgent-3";
+        client(endpoint, {
+            method: "POST",
+            data: {
+                userID,
+                videoID,
+                segments: [segment],
+                userAgent: "",
+            }
+        })
+            .then(async res => {
+                assert.strictEqual(res.status, 200);
+                const row = await queryUseragent(videoID);
+                const expected = {
+                    ...dbFormatSegment,
+                    userAgent: "",
+                };
+                assert.ok(partialDeepEquals(row, expected));
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should be able to submit with custom userAgent in body", (done) => {
+        const videoID = "userAgent-4";
+        client(endpoint, {
+            method: "POST",
+            data: {
+                userID,
+                videoID,
+                segments: [segment],
+                userAgent: "MeaBot/5.0"
+            }
+        })
+            .then(async res => {
+                assert.strictEqual(res.status, 200);
+                const row = await queryUseragent(videoID);
+                const expected = {
+                    ...dbFormatSegment,
+                    userAgent: "MeaBot/5.0",
+                };
+                assert.ok(partialDeepEquals(row, expected));
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should be able to submit with custom user-agent 1", (done) => {
+        const videoID = "userAgent-1";
+        client(endpoint, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                "User-Agent": "com.google.android.youtube/5.0"
+            },
+            data: {
+                userID,
+                videoID,
+                segments: [segment],
+            }
+        })
+            .then(async res => {
+                assert.strictEqual(res.status, 200);
+                const row = await queryUseragent(videoID);
+                const expected = {
+                    ...dbFormatSegment,
+                    userAgent: "Vanced/5.0",
+                };
+                assert.ok(partialDeepEquals(row, expected));
+                done();
+            })
+            .catch(err => done(err));
+    });
+});

test/cases/postSkipSegmentsWarnings.ts

@@ -0,0 +1,127 @@
+import { config } from "../../src/config";
+import { getHash } from "../../src/utils/getHash";
+import { db } from "../../src/databases/databases";
+import assert from "assert";
+import { client } from "../utils/httpClient";
+
+describe("postSkipSegments Warnings", () => {
+    // Constant and helpers
+    const warnUser01 = "warn-user01";
+    const warnUser01Hash = getHash(warnUser01);
+    const warnUser02 = "warn-user02";
+    const warnUser02Hash = getHash(warnUser02);
+    const warnUser03 = "warn-user03";
+    const warnUser03Hash = getHash(warnUser03);
+    const warnUser04 = "warn-user04";
+    const warnUser04Hash = getHash(warnUser04);
+
+    const warnVideoID = "postSkipSegments-warn-video";
+
+    const endpoint = "/api/skipSegments";
+    const postSkipSegmentJSON = (data: Record<string, any>) => client({
+        method: "POST",
+        url: endpoint,
+        data
+    });
+
+    before(() => {
+        const now = Date.now();
+
+        const warnVip01Hash = getHash("postSkipSegmentsWarnVIP");
+
+        const reason01 = "Reason01";
+        const reason02 = "";
+        const reason03 = "Reason03";
+
+        const MILLISECONDS_IN_HOUR = 3600000;
+        const WARNING_EXPIRATION_TIME = config.hoursAfterWarningExpires * MILLISECONDS_IN_HOUR;
+
+        const insertWarningQuery = 'INSERT INTO warnings ("userID", "issuerUserID", "enabled", "reason", "issueTime") VALUES(?, ?, ?, ?, ?)';
+        // User 1 | 1 active | custom reason
+        db.prepare("run", insertWarningQuery, [warnUser01Hash, warnVip01Hash, 1, reason01, now]);
+        // User 2 | 1 inactive | default reason
+        db.prepare("run", insertWarningQuery, [warnUser02Hash, warnVip01Hash, 0, reason02, now]);
+        // User 3 | 1 expired, active | custom reason
+        db.prepare("run", insertWarningQuery, [warnUser03Hash, warnVip01Hash, 1, reason03, (now - WARNING_EXPIRATION_TIME - 1000)]);
+        // User 4 | 1 active | default reason
+        db.prepare("run", insertWarningQuery, [warnUser04Hash, warnVip01Hash, 1, reason02, now]);
+    });
+
+    it("Should be rejected with custom message if user has active warnings", (done) => {
+        postSkipSegmentJSON({
+            userID: warnUser01,
+            videoID: warnVideoID,
+            segments: [{
+                segment: [0, 10],
+                category: "sponsor",
+            }],
+        })
+            .then(res => {
+                assert.strictEqual(res.status, 403);
+                const errorMessage = res.data;
+                const reason = "Reason01";
+                const expected = "Submission rejected due to a warning from a moderator. This means that we noticed you were making some common mistakes"
+                + " that are not malicious, and we just want to clarify the rules. "
+                + "Could you please send a message in discord.gg/SponsorBlock or matrix.to/#/#sponsor:ajay.app so we can further help you? "
+                + `Your userID is ${warnUser01Hash}.\n\nWarning reason: '${reason}'`;
+
+                assert.strictEqual(errorMessage, expected);
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should be accepted if user has inactive warning", (done) => {
+        postSkipSegmentJSON({
+            userID: warnUser02,
+            videoID: warnVideoID,
+            segments: [{
+                segment: [50, 60],
+                category: "sponsor",
+            }],
+        })
+            .then(res => {
+                assert.ok(res.status === 200, `Status code was ${res.status} ${res.data}`);
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should be accepted if user has expired warning", (done) => {
+        postSkipSegmentJSON({
+            userID: warnUser03,
+            videoID: warnVideoID,
+            segments: [{
+                segment: [53, 60],
+                category: "sponsor",
+            }],
+        })
+            .then(res => {
+                assert.ok(res.status === 200, `Status code was ${res.status} ${res.data}`);
+                done();
+            })
+            .catch(err => done(err));
+    });
+
+    it("Should be rejected with default message if user has active warning", (done) => {
+        postSkipSegmentJSON({
+            userID: warnUser04,
+            videoID: warnVideoID,
+            segments: [{
+                segment: [0, 10],
+                category: "sponsor",
+            }],
+        })
+            .then(res => {
+                assert.strictEqual(res.status, 403);
+                const errorMessage = res.data;
+                const expected = "Submission rejected due to a warning from a moderator. This means that we noticed you were making some common mistakes"
+                + " that are not malicious, and we just want to clarify the rules. "
+                + "Could you please send a message in discord.gg/SponsorBlock or matrix.to/#/#sponsor:ajay.app so we can further help you? "
+                + `Your userID is ${warnUser04Hash}.`;
+                assert.strictEqual(errorMessage, expected);
+                done();
+            })
+            .catch(err => done(err));
+    });
+});

test/cases/postWarning.ts

@@ -9,16 +9,21 @@ describe("postWarning", () => {
     const endpoint = "/api/warnUser";
     const getWarning = (userID: string) => db.prepare("get", `SELECT "userID", "issueTime", "issuerUserID", enabled, "reason" FROM warnings WHERE "userID" = ?`, [userID]);
 
-    const warnedUser = getHash("warning-0");
+    const warneduserID = "warning-0";
+    const warnedUserPublicID = getHash(warneduserID);
+    const warningVipOne = "warning-vip-1";
+    const warningVipTwo = "warning-vip-2";
+    const nonVipUser = "warning-non-vip";
 
     before(async () => {
-        await db.prepare("run", `INSERT INTO "vipUsers" ("userID") VALUES (?)`, [getHash("warning-vip")]);
+        await db.prepare("run", `INSERT INTO "vipUsers" ("userID") VALUES (?)`, [getHash(warningVipOne)]);
+        await db.prepare("run", `INSERT INTO "vipUsers" ("userID") VALUES (?)`, [getHash(warningVipTwo)]);
     });
 
     it("Should be able to create warning if vip (exp 200)", (done) => {
         const json = {
-            issuerUserID: "warning-vip",
+            issuerUserID: warningVipOne,
-            userID: warnedUser,
+            userID: warnedUserPublicID,
             reason: "warning-reason-0"
         };
         client.post(endpoint, json)
@@ -38,8 +43,8 @@ describe("postWarning", () => {
 
     it("Should be not be able to create a duplicate warning if vip", (done) => {
         const json = {
-            issuerUserID: "warning-vip",
+            issuerUserID: warningVipOne,
-            userID: warnedUser,
+            userID: warnedUserPublicID,
         };
 
         client.post(endpoint, json)
@@ -58,8 +63,8 @@ describe("postWarning", () => {
 
     it("Should be able to remove warning if vip", (done) => {
         const json = {
-            issuerUserID: "warning-vip",
+            issuerUserID: warningVipOne,
-            userID: warnedUser,
+            userID: warnedUserPublicID,
             enabled: false
         };
 
@@ -78,8 +83,8 @@ describe("postWarning", () => {
 
     it("Should not be able to create warning if not vip (exp 403)", (done) => {
         const json = {
-            issuerUserID: "warning-not-vip",
+            issuerUserID: nonVipUser,
-            userID: "warning-1",
+            userID: warnedUserPublicID,
         };
 
         client.post(endpoint, json)
@@ -101,8 +106,8 @@ describe("postWarning", () => {
 
     it("Should re-enable disabled warning", (done) => {
         const json = {
-            issuerUserID: "warning-vip",
+            issuerUserID: warningVipOne,
-            userID: warnedUser,
+            userID: warnedUserPublicID,
             enabled: true
         };
 
@@ -121,14 +126,14 @@ describe("postWarning", () => {
 
     it("Should be able to remove your own warning", (done) => {
         const json = {
-            userID: "warning-0",
+            userID: warneduserID,
             enabled: false
         };
 
         client.post(endpoint, json)
             .then(async res => {
                 assert.strictEqual(res.status, 200);
-                const data = await getWarning(warnedUser);
+                const data = await getWarning(warnedUserPublicID);
                 const expected = {
                     enabled: 0
                 };
@@ -138,15 +143,16 @@ describe("postWarning", () => {
             .catch(err => done(err));
     });
 
-    it("Should be able to add your own warning", (done) => {
+    it("Should not be able to add your own warning", (done) => {
         const json = {
-            userID: "warning-0"
+            userID: warneduserID,
+            enabled: true
         };
 
         client.post(endpoint, json)
             .then(async res => {
                 assert.strictEqual(res.status, 403);
-                const data = await getWarning(warnedUser);
+                const data = await getWarning(warnedUserPublicID);
                 const expected = {
                     enabled: 0
                 };

test/cases/redisTest.ts

@@ -1,9 +1,7 @@
 import { config } from "../../src/config";
 import redis from "../../src/utils/redis";
-import crypto from "crypto";
 import assert from "assert";
-
+import { genRandom } from "../utils/getRandom";
-const genRandom = (bytes=8) => crypto.pseudoRandomBytes(bytes).toString("hex");
 
 const randKey1 = genRandom();
 const randValue1 = genRandom();

test/cases/shadowBanUser.ts

@@ -410,6 +410,27 @@ describe("shadowBanUser", () => {
             .catch(err => done(err));
     });
 
+    it("Should be possible to ban self", (done) => {
+        const userID = VIPuserID;
+        const hashUserID = getHash(userID);
+        client({
+            method: "POST",
+            url: endpoint,
+            params: {
+                enabled: true,
+                userID: hashUserID,
+                categories: `["sponsor"]`,
+                unHideOldSubmissions: true,
+                adminUserID: userID,
+            }
+        })
+            .then(res => {
+                assert.strictEqual(res.status, 200);
+                done();
+            })
+            .catch(err => done(err));
+    });
+
     it("Should be able to ban user by userID and other users who used that IP and hide specific category", (done) => {
         const hashedIP = "shadowBannedIP8";
         const userID = "shadowBanned8";

test/cases/userCounter.ts

@@ -2,6 +2,7 @@ import axios from "axios";
 import assert from "assert";
 import { config } from "../../src/config";
 import { getHash } from "../../src/utils/getHash";
+import { client } from "../utils/httpClient";
 
 describe("userCounter", () => {
     it("Should return 200", function (done) {
@@ -20,4 +21,13 @@ describe("userCounter", () => {
             })
             .catch(err => done(err));
     });
+    it("Should not incremeent counter on OPTIONS", function (done) {
+        /* cannot spy test */
+        if (!config.userCounterURL) this.skip(); // skip if no userCounterURL is set
+        //const spy = sinon.spy(UserCounter);
+        client({ method: "OPTIONS", url: "/api/status" })
+            .then(() => client({ method: "GET", url: "/api/status" }));
+        //assert.strictEqual(spy.callCount, 1);
+        done();
+    });
 });

test/test.ts

@@ -10,6 +10,7 @@ import { ImportMock } from "ts-mock-imports";
 import * as rateLimitMiddlewareModule from "../src/middleware/requestRateLimit";
 import rateLimit from "express-rate-limit";
 import redis from "../src/utils/redis";
+import { resetRedis, resetPostgres } from "./utils/reset";
 
 async function init() {
     ImportMock.mockFunction(rateLimitMiddlewareModule, "rateLimitMiddleware", rateLimit({
@@ -19,6 +20,8 @@ async function init() {
     // delete old test database
     if (fs.existsSync(config.db)) fs.unlinkSync(config.db);
     if (fs.existsSync(config.privateDB)) fs.unlinkSync(config.privateDB);
+    if (config?.redis?.enabled) await resetRedis();
+    if (config?.postgres) await resetPostgres();
 
     await initDb();
 
@@ -59,6 +62,7 @@ async function init() {
                 server.close();
                 redis.quit();
                 process.exitCode = failures ? 1 : 0; // exit with non-zero status if there were failures
+                process.exit();
             });
         });
     });

test/utils/getRandom.ts

@@ -0,0 +1,3 @@
+import crypto from "crypto";
+
+export const genRandom = (bytes=8) => crypto.pseudoRandomBytes(bytes).toString("hex");

test/utils/reset.ts

@@ -0,0 +1,22 @@
+// drop postgres tables
+// reset redis cache
+import { config } from "../../src/config";
+import { createClient } from "redis";
+import { Pool } from "pg";
+import { Logger } from "../../src/utils/logger";
+
+export async function resetRedis() {
+    if (config?.redis?.enabled && config.mode === "test") {
+        const client = createClient(config.redis);
+        await client.connect();
+        await client.flushAll();
+    }
+}
+export async function resetPostgres() {
+    if (process.env.TEST_POSTGRES && config.mode == "test" && config.postgres) {
+        const pool = new Pool({ ...config.postgres });
+        await pool.query(`DROP DATABASE IF EXISTS "sponsorTimes"`);
+        await pool.query(`DROP DATABASE IF EXISTS "privateDB"`);
+        await pool.end().catch(err => Logger.error(`closing db (postgres): ${err}`));
+    }
+}