optimize skipSegments, add eTag

- moved skipSegments parameter parsing to new file
- added oldGetVideoSponsorTimes to getSkipSegments.ts

src/app.ts

@@ -1,7 +1,6 @@
 import express, { Request, RequestHandler, Response, Router } from "express";
 import { config } from "./config";
 import { oldSubmitSponsorTimes } from "./routes/oldSubmitSponsorTimes";
-import { oldGetVideoSponsorTimes } from "./routes/oldGetVideoSponsorTimes";
 import { postSegmentShift } from "./routes/postSegmentShift";
 import { postWarning } from "./routes/postWarning";
 import { getIsUserVIP } from "./routes/getIsUserVIP";
@@ -21,13 +20,13 @@ import { viewedVideoSponsorTime } from "./routes/viewedVideoSponsorTime";
 import { voteOnSponsorTime, getUserID as voteGetUserID } from "./routes/voteOnSponsorTime";
 import { getSkipSegmentsByHash } from "./routes/getSkipSegmentsByHash";
 import { postSkipSegments } from "./routes/postSkipSegments";
-import { endpoint as getSkipSegments } from "./routes/getSkipSegments";
+import { getSkipSegments, oldGetVideoSponsorTimes } from "./routes/getSkipSegments";
 import { userCounter } from "./middleware/userCounter";
 import { loggerMiddleware } from "./middleware/logger";
 import { corsMiddleware } from "./middleware/cors";
 import { apiCspMiddleware } from "./middleware/apiCsp";
 import { rateLimitMiddleware } from "./middleware/requestRateLimit";
-import dumpDatabase, { appExportPath, downloadFile } from "./routes/dumpDatabase";
+import dumpDatabase from "./routes/dumpDatabase";
 import { endpoint as getSegmentInfo } from "./routes/getSegmentInfo";
 import { postClearCache } from "./routes/postClearCache";
 import { addUnlistedVideo } from "./routes/addUnlistedVideo";

src/middleware/etag.ts

@@ -5,6 +5,7 @@ import { skipSegmentsHashKey, skipSegmentsKey, videoLabelsHashKey, videoLabelsKe
 
 type hashType = "skipSegments" | "skipSegmentsHash" | "videoLabel" | "videoLabelHash";
 type ETag = `${hashType};${VideoIDHash};${Service};${number}`;
+type hashKey = string | VideoID | VideoIDHash;
 
 export function cacheMiddlware(req: Request, res: Response, next: NextFunction): void {
     const reqEtag = req.get("If-None-Match") as string;
@@ -25,7 +26,7 @@ export function cacheMiddlware(req: Request, res: Response, next: NextFunction):
         .catch(next);
 }
 
-function getLastModified(hashType: hashType, hashKey: (VideoID | VideoIDHash), service: Service): Promise<Date | null> {
+function getLastModified(hashType: hashType, hashKey: hashKey, service: Service): Promise<Date | null> {
     let redisKey: string | null;
     if (hashType === "skipSegments") redisKey = skipSegmentsKey(hashKey as VideoID, service);
     else if (hashType === "skipSegmentsHash") redisKey = skipSegmentsHashKey(hashKey as VideoIDHash, service);
@@ -35,7 +36,7 @@ function getLastModified(hashType: hashType, hashKey: (VideoID | VideoIDHash), s
     return QueryCacher.getKeyLastModified(redisKey);
 }
 
-export async function getEtag(hashType: hashType, hashKey: VideoIDHash, service: Service): Promise<ETag> {
+export async function getEtag(hashType: hashType, hashKey: hashKey, service: Service): Promise<ETag> {
     const lastModified = await getLastModified(hashType, hashKey, service);
     return `${hashType};${hashKey};${service};${lastModified.getTime()}` as ETag;
 }

src/routes/getSkipSegments.ts

@@ -12,7 +12,8 @@ import { QueryCacher } from "../utils/queryCacher";
 import { getReputation } from "../utils/reputation";
 import { getService } from "../utils/getService";
 import { promiseOrTimeout } from "../utils/promise";
-
+import { parseSkipSegments } from "../utils/parseSkipSegments";
+import { getEtag } from "../middleware/etag";
 
 async function prepareCategorySegments(req: Request, videoID: VideoID, service: Service, segments: DBSegment[], cache: SegmentCache = { shadowHiddenSegmentIPs: {} }, useCache: boolean): Promise<Segment[]> {
     const shouldFilter: boolean[] = await Promise.all(segments.map(async (segment) => {
@@ -86,9 +87,6 @@ async function getSegmentsByVideoID(req: Request, videoID: VideoID, categories:
     }
 
     try {
-        categories = categories.filter((category) => !/[^a-z|_|-]/.test(category));
-        if (categories.length === 0) return null;
-
         const segments: DBSegment[] = (await getSegmentsFromDBByVideoID(videoID, service))
             .map((segment: DBSegment) => {
                 if (filterRequiredSegments(segment.UUID, requiredSegments)) segment.required = true;
@@ -139,9 +137,6 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash,
     try {
         type SegmentPerVideoID = SBRecord<VideoID, { segments: DBSegment[] }>;
 
-        categories = categories.filter((category) => !(/[^a-z|_|-]/.test(category)));
-        if (categories.length === 0) return null;
-
         const segmentPerVideoID: SegmentPerVideoID = (await getSegmentsFromDBByHash(hashedVideoIDPrefix, service))
             .reduce((acc: SegmentPerVideoID, segment: DBSegment) => {
                 acc[segment.videoID] = acc[segment.videoID] || {
@@ -396,75 +391,59 @@ function splitPercentOverlap(groups: OverlappingSegmentGroup[]): OverlappingSegm
     });
 }
 
-/**
- *
- * Returns what would be sent to the client.
- * Will respond with errors if required. Returns false if it errors.
- *
- * @param req
- * @param res
- *
- * @returns
- */
-async function handleGetSegments(req: Request, res: Response): Promise<Segment[] | false> {
+async function getSkipSegments(req: Request, res: Response): Promise<Response> {
     const videoID = req.query.videoID as VideoID;
     if (!videoID) {
-        res.status(400).send("videoID not specified");
-        return false;
-    }
-    // Default to sponsor
-    // If using params instead of JSON, only one category can be pulled
-    const categories: Category[] = req.query.categories
-        ? JSON.parse(req.query.categories as string)
-        : req.query.category
-            ? Array.isArray(req.query.category)
-                ? req.query.category
-                : [req.query.category]
-            : ["sponsor"];
-    if (!Array.isArray(categories)) {
-        res.status(400).send("Categories parameter does not match format requirements.");
-        return false;
+        return res.status(400).send("videoID not specified");
     }
 
-    const actionTypes: ActionType[] = req.query.actionTypes
-        ? JSON.parse(req.query.actionTypes as string)
-        : req.query.actionType
-            ? Array.isArray(req.query.actionType)
-                ? req.query.actionType
-                : [req.query.actionType]
-            : [ActionType.Skip];
-    if (!Array.isArray(actionTypes)) {
-        res.status(400).send("actionTypes parameter does not match format requirements.");
-        return false;
+    const parseResult = parseSkipSegments(req);
+    if (parseResult.errors.length > 0) {
+        return res.status(400).send(parseResult.errors);
     }
 
-    const requiredSegments: SegmentUUID[] = req.query.requiredSegments
-        ? JSON.parse(req.query.requiredSegments as string)
-        : req.query.requiredSegment
-            ? Array.isArray(req.query.requiredSegment)
-                ? req.query.requiredSegment
-                : [req.query.requiredSegment]
-            : [];
-    if (!Array.isArray(requiredSegments)) {
-        res.status(400).send("requiredSegments parameter does not match format requirements.");
-        return false;
-    }
-
-    const service = getService(req.query.service, req.body.service);
-
+    const { categories, actionTypes, requiredSegments, service } = parseResult;
     const segments = await getSegmentsByVideoID(req, videoID, categories, actionTypes, requiredSegments, service);
 
     if (segments === null || segments === undefined) {
-        res.sendStatus(500);
-        return false;
+        return res.sendStatus(500);
+    } else if (segments.length === 0) {
+        return res.sendStatus(404);
     }
 
-    if (segments.length === 0) {
-        res.sendStatus(404);
-        return false;
+    await getEtag("skipSegments", (videoID as string), service)
+        .then(etag => res.set("ETag", etag))
+        .catch(() => null);
+    return res.send(segments);
+}
+
+async function oldGetVideoSponsorTimes(req: Request, res: Response): Promise<Response> {
+    const videoID = req.query.videoID as VideoID;
+    if (!videoID) {
+        return res.status(400).send("videoID not specified");
     }
 
-    return segments;
+    const segments = await getSegmentsByVideoID(req, videoID, ["sponsor"] as Category[], [ActionType.Skip], [], Service.YouTube);
+
+    if (segments === null || segments === undefined) {
+        return res.sendStatus(500);
+    } else if (segments.length === 0) {
+        return res.sendStatus(404);
+    }
+
+    // Convert to old outputs
+    const sponsorTimes = [];
+    const UUIDs = [];
+
+    for (const segment of segments) {
+        sponsorTimes.push(segment.segment);
+        UUIDs.push(segment.UUID);
+    }
+
+    return res.send({
+        sponsorTimes,
+        UUIDs,
+    });
 }
 
 const filterRequiredSegments = (UUID: SegmentUUID, requiredSegments: SegmentUUID[]): boolean => {
@@ -474,25 +453,9 @@ const filterRequiredSegments = (UUID: SegmentUUID, requiredSegments: SegmentUUID
     return false;
 };
 
-async function endpoint(req: Request, res: Response): Promise<Response> {
-    try {
-        const segments = await handleGetSegments(req, res);
-
-        // If false, res.send has already been called
-        if (segments) {
-            //send result
-            return res.send(segments);
-        }
-    } catch (err) /* istanbul ignore next */ {
-        if (err instanceof SyntaxError) {
-            return res.status(400).send("Categories parameter does not match format requirements.");
-        } else return res.sendStatus(500);
-    }
-}
-
 export {
     getSegmentsByVideoID,
     getSegmentsByHash,
-    endpoint,
-    handleGetSegments
+    getSkipSegments,
+    oldGetVideoSponsorTimes
 };

src/routes/getSkipSegmentsByHash.ts

@@ -1,9 +1,9 @@
 import { hashPrefixTester } from "../utils/hashPrefixTester";
 import { getSegmentsByHash } from "./getSkipSegments";
 import { Request, Response } from "express";
-import { ActionType, Category, SegmentUUID, VideoIDHash, Service } from "../types/segments.model";
-import { getService } from "../utils/getService";
+import { VideoIDHash } from "../types/segments.model";
 import { Logger } from "../utils/logger";
+import { parseSkipSegments } from "../utils/parseSkipSegments";
 import { getEtag } from "../middleware/etag";
 
 export async function getSkipSegmentsByHash(req: Request, res: Response): Promise<Response> {
@@ -13,58 +13,11 @@ export async function getSkipSegmentsByHash(req: Request, res: Response): Promis
     }
     hashPrefix = hashPrefix.toLowerCase() as VideoIDHash;
 
-    let categories: Category[] = [];
-    try {
-        categories = req.query.categories
-            ? JSON.parse(req.query.categories as string)
-            : req.query.category
-                ? Array.isArray(req.query.category)
-                    ? req.query.category
-                    : [req.query.category]
-                : ["sponsor"];
-        if (!Array.isArray(categories)) {
-            return res.status(400).send("Categories parameter does not match format requirements.");
-        }
-    } catch(error) {
-        return res.status(400).send("Bad parameter: categories (invalid JSON)");
+    const parseResult = parseSkipSegments(req);
+    if (parseResult.errors.length > 0) {
+        return res.status(400).send(parseResult.errors);
     }
-
-    let actionTypes: ActionType[] = [];
-    try {
-        actionTypes = req.query.actionTypes
-            ? JSON.parse(req.query.actionTypes as string)
-            : req.query.actionType
-                ? Array.isArray(req.query.actionType)
-                    ? req.query.actionType
-                    : [req.query.actionType]
-                : [ActionType.Skip];
-        if (!Array.isArray(actionTypes)) {
-            return res.status(400).send("actionTypes parameter does not match format requirements.");
-        }
-    } catch(error) {
-        return res.status(400).send("Bad parameter: actionTypes (invalid JSON)");
-    }
-
-    let requiredSegments: SegmentUUID[] = [];
-    try {
-        requiredSegments = req.query.requiredSegments
-            ? JSON.parse(req.query.requiredSegments as string)
-            : req.query.requiredSegment
-                ? Array.isArray(req.query.requiredSegment)
-                    ? req.query.requiredSegment
-                    : [req.query.requiredSegment]
-                : [];
-        if (!Array.isArray(requiredSegments)) {
-            return res.status(400).send("requiredSegments parameter does not match format requirements.");
-        }
-    } catch(error) {
-        return res.status(400).send("Bad parameter: requiredSegments (invalid JSON)");
-    }
-
-    const service: Service = getService(req.query.service, req.body.service);
-
-    // filter out none string elements, only flat array with strings is valid
-    categories = categories.filter((item: any) => typeof item === "string");
+    const { categories, actionTypes, requiredSegments, service } = parseResult;
 
     // Get all video id's that match hash prefix
     const segments = await getSegmentsByHash(req, hashPrefix, categories, actionTypes, requiredSegments, service);

src/routes/oldGetVideoSponsorTimes.ts

@@ -1,24 +0,0 @@
-import { handleGetSegments } from "./getSkipSegments";
-import { Request, Response } from "express";
-
-export async function oldGetVideoSponsorTimes(req: Request, res: Response): Promise<Response> {
-    const segments = await handleGetSegments(req, res);
-
-    if (segments) {
-        // Convert to old outputs
-        const sponsorTimes = [];
-        const UUIDs = [];
-
-        for (const segment of segments) {
-            sponsorTimes.push(segment.segment);
-            UUIDs.push(segment.UUID);
-        }
-
-        return res.send({
-            sponsorTimes,
-            UUIDs,
-        });
-    }
-
-    // Error has already been handled in the other method
-}

src/routes/viewedVideoSponsorTime.ts

@@ -2,9 +2,9 @@ import { db } from "../databases/databases";
 import { Request, Response } from "express";
 
 export async function viewedVideoSponsorTime(req: Request, res: Response): Promise<Response> {
-    const UUID = req.query.UUID;
+    const UUID = req.query?.UUID;
 
-    if (UUID == undefined) {
+    if (!UUID) {
         //invalid request
         return res.sendStatus(400);
     }

src/utils/parseSkipSegments.ts

@@ -0,0 +1,75 @@
+import { Request } from "express";
+import { ActionType, SegmentUUID, Category, Service } from "../types/segments.model";
+import { getService } from "./getService";
+
+type fn = (req: Request) => any[];
+
+const syntaxErrorWrapper = (fn: fn, req: Request) => {
+    try { return fn(req); }
+    catch (e) { return undefined; }
+};
+
+// Default to sponsor
+const getCategories = (req: Request): Category[] =>
+    req.query.categories
+        ? JSON.parse(req.query.categories as string)
+        : req.query.category
+            ? Array.isArray(req.query.category)
+                ? req.query.category
+                : [req.query.category]
+            : ["sponsor"];
+
+// Default to skip
+const getActionTypes = (req: Request): ActionType[] =>
+    req.query.actionTypes
+        ? JSON.parse(req.query.actionTypes as string)
+        : req.query.actionType
+            ? Array.isArray(req.query.actionType)
+                ? req.query.actionType
+                : [req.query.actionType]
+            : [ActionType.Skip];
+
+// Default to empty array
+const getRequiredSegments = (req: Request): SegmentUUID[] =>
+    req.query.requiredSegments
+        ? JSON.parse(req.query.requiredSegments as string)
+        : req.query.requiredSegment
+            ? Array.isArray(req.query.requiredSegment)
+                ? req.query.requiredSegment
+                : [req.query.requiredSegment]
+            : [];
+
+const errorMessage = (parameter: string) => `${parameter} parameter does not match format requirements.`;
+
+export function parseSkipSegments(req: Request): {
+    categories: Category[];
+    actionTypes: ActionType[];
+    requiredSegments: SegmentUUID[];
+    service: Service;
+    errors: string[];
+} {
+    let categories: Category[] = syntaxErrorWrapper(getCategories, req);
+    const actionTypes: ActionType[] = syntaxErrorWrapper(getActionTypes, req);
+    const requiredSegments: SegmentUUID[] = syntaxErrorWrapper(getRequiredSegments, req);
+    const service: Service = getService(req.query.service, req.body.services);
+    const errors: string[] = [];
+    if (!Array.isArray(categories)) errors.push(errorMessage("categories"));
+    else {
+        // check category names for invalid characters
+        // and none string elements
+        categories = categories
+            .filter((item: any) => typeof item === "string")
+            .filter((category) => !(/[^a-z|_|-]/.test(category)));
+        if (categories.length === 0) errors.push("No valid categories provided.");
+    }
+    if (!Array.isArray(actionTypes)) errors.push(errorMessage("actionTypes"));
+    if (!Array.isArray(requiredSegments)) errors.push(errorMessage("requiredSegments"));
+    // finished parsing
+    return {
+        categories,
+        actionTypes,
+        requiredSegments,
+        service,
+        errors
+    };
+}