Merge pull request #533 from mchangrh/cache-control

add ETag to skipSegments byHash

package.json

@@ -9,8 +9,8 @@
     "cover:report": "nyc report",
     "dev": "nodemon",
     "dev:bash": "nodemon -x 'npm test ; npm start'",
-    "postgres:docker": "docker run --rm -p 5432:5432 -e POSTGRES_USER=ci_db_user -e POSTGRES_PASSWORD=ci_db_pass postgres:alpine",
-    "redis:docker": "docker run --rm -p 6379:6379 redis:alpine",
+    "postgres:docker": "docker run --rm -p 5432:5432 -e POSTGRES_USER=ci_db_user -e POSTGRES_PASSWORD=ci_db_pass postgres:14-alpine",
+    "redis:docker": "docker run --rm -p 6379:6379 redis:7-alpine --save '' --appendonly no",
     "start": "ts-node src/index.ts",
     "tsc": "tsc -p tsconfig.json",
     "lint": "eslint src test",

src/app.ts

@@ -1,7 +1,6 @@
 import express, { Request, RequestHandler, Response, Router } from "express";
 import { config } from "./config";
 import { oldSubmitSponsorTimes } from "./routes/oldSubmitSponsorTimes";
-import { oldGetVideoSponsorTimes } from "./routes/oldGetVideoSponsorTimes";
 import { postSegmentShift } from "./routes/postSegmentShift";
 import { postWarning } from "./routes/postWarning";
 import { getIsUserVIP } from "./routes/getIsUserVIP";
@@ -21,13 +20,13 @@ import { viewedVideoSponsorTime } from "./routes/viewedVideoSponsorTime";
 import { voteOnSponsorTime, getUserID as voteGetUserID } from "./routes/voteOnSponsorTime";
 import { getSkipSegmentsByHash } from "./routes/getSkipSegmentsByHash";
 import { postSkipSegments } from "./routes/postSkipSegments";
-import { endpoint as getSkipSegments } from "./routes/getSkipSegments";
+import { getSkipSegments, oldGetVideoSponsorTimes } from "./routes/getSkipSegments";
 import { userCounter } from "./middleware/userCounter";
 import { loggerMiddleware } from "./middleware/logger";
 import { corsMiddleware } from "./middleware/cors";
 import { apiCspMiddleware } from "./middleware/apiCsp";
 import { rateLimitMiddleware } from "./middleware/requestRateLimit";
-import dumpDatabase, { appExportPath, downloadFile } from "./routes/dumpDatabase";
+import dumpDatabase from "./routes/dumpDatabase";
 import { endpoint as getSegmentInfo } from "./routes/getSegmentInfo";
 import { postClearCache } from "./routes/postClearCache";
 import { addUnlistedVideo } from "./routes/addUnlistedVideo";
@@ -50,6 +49,7 @@ import { getVideoLabelsByHash } from "./routes/getVideoLabelByHash";
 import { addFeature } from "./routes/addFeature";
 import { generateTokenRequest } from "./routes/generateToken";
 import { verifyTokenRequest } from "./routes/verifyToken";
+import { cacheMiddlware } from "./middleware/etag";
 
 export function createServer(callback: () => void): Server {
     // Create a service (the app object is just a callback).
@@ -57,11 +57,13 @@ export function createServer(callback: () => void): Server {
 
     const router = ExpressPromiseRouter();
     app.use(router);
+    app.set("etag", false); // disable built in etag
 
     //setup CORS correctly
     router.use(corsMiddleware);
     router.use(loggerMiddleware);
     router.use("/api/", apiCspMiddleware);
+    router.use(cacheMiddlware);
     router.use(express.json());
 
     if (config.userCounterURL) router.use(userCounter);

src/middleware/cors.ts

@@ -3,6 +3,6 @@ import { NextFunction, Request, Response } from "express";
 export function corsMiddleware(req: Request, res: Response, next: NextFunction): void {
     res.header("Access-Control-Allow-Origin", "*");
     res.header("Access-Control-Allow-Methods", "GET, POST, OPTIONS, DELETE");
-    res.header("Access-Control-Allow-Headers", "Content-Type");
+    res.header("Access-Control-Allow-Headers", "Content-Type, If-None-Match");
     next();
 }

src/middleware/etag.ts

@@ -0,0 +1,49 @@
+import { NextFunction, Request, Response } from "express";
+import { VideoID, VideoIDHash, Service } from "../types/segments.model";
+import { QueryCacher } from "../utils/queryCacher";
+import { skipSegmentsHashKey, skipSegmentsKey, videoLabelsHashKey, videoLabelsKey } from "../utils/redisKeys";
+
+type hashType = "skipSegments" | "skipSegmentsHash" | "videoLabel" | "videoLabelHash";
+type ETag = `${hashType};${VideoIDHash};${Service};${number}`;
+type hashKey = string | VideoID | VideoIDHash;
+
+export function cacheMiddlware(req: Request, res: Response, next: NextFunction): void {
+    const reqEtag = req.get("If-None-Match") as string;
+    // if weak etag, do not handle
+    if (!reqEtag || reqEtag.startsWith("W/")) return next();
+    // split into components
+    const [hashType, hashKey, service, lastModified] = reqEtag.split(";");
+    // fetch last-modified
+    getLastModified(hashType as hashType, hashKey as VideoIDHash, service as Service)
+        .then(redisLastModified => {
+            if (redisLastModified <= new Date(Number(lastModified) + 1000)) {
+                // match cache, generate etag
+                const etag = `${hashType};${hashKey};${service};${redisLastModified.getTime()}` as ETag;
+                res.status(304).set("etag", etag).send();
+            }
+            else next();
+        })
+        .catch(next);
+}
+
+function getLastModified(hashType: hashType, hashKey: hashKey, service: Service): Promise<Date | null> {
+    let redisKey: string | null;
+    if (hashType === "skipSegments") redisKey = skipSegmentsKey(hashKey as VideoID, service);
+    else if (hashType === "skipSegmentsHash") redisKey = skipSegmentsHashKey(hashKey as VideoIDHash, service);
+    else if (hashType === "videoLabel") redisKey = videoLabelsKey(hashKey as VideoID, service);
+    else if (hashType === "videoLabelHash") redisKey = videoLabelsHashKey(hashKey as VideoIDHash, service);
+    else return Promise.reject();
+    return QueryCacher.getKeyLastModified(redisKey);
+}
+
+export async function getEtag(hashType: hashType, hashKey: hashKey, service: Service): Promise<ETag> {
+    const lastModified = await getLastModified(hashType, hashKey, service);
+    return `${hashType};${hashKey};${service};${lastModified.getTime()}` as ETag;
+}
+
+/* example usage
+import { getEtag } from "../middleware/etag";
+await getEtag(hashType, hashPrefix, service)
+    .then(etag => res.set("ETag", etag))
+    .catch(() => null);
+*/

src/routes/getSkipSegments.ts

@@ -12,7 +12,8 @@ import { QueryCacher } from "../utils/queryCacher";
 import { getReputation } from "../utils/reputation";
 import { getService } from "../utils/getService";
 import { promiseOrTimeout } from "../utils/promise";
-
+import { parseSkipSegments } from "../utils/parseSkipSegments";
+import { getEtag } from "../middleware/etag";
 
 async function prepareCategorySegments(req: Request, videoID: VideoID, service: Service, segments: DBSegment[], cache: SegmentCache = { shadowHiddenSegmentIPs: {} }, useCache: boolean): Promise<Segment[]> {
     const shouldFilter: boolean[] = await Promise.all(segments.map(async (segment) => {
@@ -86,9 +87,6 @@ async function getSegmentsByVideoID(req: Request, videoID: VideoID, categories:
     }
 
     try {
-        categories = categories.filter((category) => !/[^a-z|_|-]/.test(category));
-        if (categories.length === 0) return null;
-
         const segments: DBSegment[] = (await getSegmentsFromDBByVideoID(videoID, service))
             .map((segment: DBSegment) => {
                 if (filterRequiredSegments(segment.UUID, requiredSegments)) segment.required = true;
@@ -139,9 +137,6 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash,
     try {
         type SegmentPerVideoID = SBRecord<VideoID, { segments: DBSegment[] }>;
 
-        categories = categories.filter((category) => !(/[^a-z|_|-]/.test(category)));
-        if (categories.length === 0) return null;
-
         const segmentPerVideoID: SegmentPerVideoID = (await getSegmentsFromDBByHash(hashedVideoIDPrefix, service))
             .reduce((acc: SegmentPerVideoID, segment: DBSegment) => {
                 acc[segment.videoID] = acc[segment.videoID] || {
@@ -396,75 +391,59 @@ function splitPercentOverlap(groups: OverlappingSegmentGroup[]): OverlappingSegm
     });
 }
 
-/**
- *
- * Returns what would be sent to the client.
- * Will respond with errors if required. Returns false if it errors.
- *
- * @param req
- * @param res
- *
- * @returns
- */
-async function handleGetSegments(req: Request, res: Response): Promise<Segment[] | false> {
+async function getSkipSegments(req: Request, res: Response): Promise<Response> {
     const videoID = req.query.videoID as VideoID;
     if (!videoID) {
-        res.status(400).send("videoID not specified");
-        return false;
-    }
-    // Default to sponsor
-    // If using params instead of JSON, only one category can be pulled
-    const categories: Category[] = req.query.categories
-        ? JSON.parse(req.query.categories as string)
-        : req.query.category
-            ? Array.isArray(req.query.category)
-                ? req.query.category
-                : [req.query.category]
-            : ["sponsor"];
-    if (!Array.isArray(categories)) {
-        res.status(400).send("Categories parameter does not match format requirements.");
-        return false;
+        return res.status(400).send("videoID not specified");
     }
 
-    const actionTypes: ActionType[] = req.query.actionTypes
-        ? JSON.parse(req.query.actionTypes as string)
-        : req.query.actionType
-            ? Array.isArray(req.query.actionType)
-                ? req.query.actionType
-                : [req.query.actionType]
-            : [ActionType.Skip];
-    if (!Array.isArray(actionTypes)) {
-        res.status(400).send("actionTypes parameter does not match format requirements.");
-        return false;
+    const parseResult = parseSkipSegments(req);
+    if (parseResult.errors.length > 0) {
+        return res.status(400).send(parseResult.errors);
     }
 
-    const requiredSegments: SegmentUUID[] = req.query.requiredSegments
-        ? JSON.parse(req.query.requiredSegments as string)
-        : req.query.requiredSegment
-            ? Array.isArray(req.query.requiredSegment)
-                ? req.query.requiredSegment
-                : [req.query.requiredSegment]
-            : [];
-    if (!Array.isArray(requiredSegments)) {
-        res.status(400).send("requiredSegments parameter does not match format requirements.");
-        return false;
-    }
-
-    const service = getService(req.query.service, req.body.service);
-
+    const { categories, actionTypes, requiredSegments, service } = parseResult;
     const segments = await getSegmentsByVideoID(req, videoID, categories, actionTypes, requiredSegments, service);
 
     if (segments === null || segments === undefined) {
-        res.sendStatus(500);
-        return false;
+        return res.sendStatus(500);
+    } else if (segments.length === 0) {
+        return res.sendStatus(404);
     }
 
-    if (segments.length === 0) {
-        res.sendStatus(404);
-        return false;
+    await getEtag("skipSegments", (videoID as string), service)
+        .then(etag => res.set("ETag", etag))
+        .catch(() => null);
+    return res.send(segments);
+}
+
+async function oldGetVideoSponsorTimes(req: Request, res: Response): Promise<Response> {
+    const videoID = req.query.videoID as VideoID;
+    if (!videoID) {
+        return res.status(400).send("videoID not specified");
     }
 
-    return segments;
+    const segments = await getSegmentsByVideoID(req, videoID, ["sponsor"] as Category[], [ActionType.Skip], [], Service.YouTube);
+
+    if (segments === null || segments === undefined) {
+        return res.sendStatus(500);
+    } else if (segments.length === 0) {
+        return res.sendStatus(404);
+    }
+
+    // Convert to old outputs
+    const sponsorTimes = [];
+    const UUIDs = [];
+
+    for (const segment of segments) {
+        sponsorTimes.push(segment.segment);
+        UUIDs.push(segment.UUID);
+    }
+
+    return res.send({
+        sponsorTimes,
+        UUIDs,
+    });
 }
 
 const filterRequiredSegments = (UUID: SegmentUUID, requiredSegments: SegmentUUID[]): boolean => {
@@ -474,25 +453,9 @@ const filterRequiredSegments = (UUID: SegmentUUID, requiredSegments: SegmentUUID
     return false;
 };
 
-async function endpoint(req: Request, res: Response): Promise<Response> {
-    try {
-        const segments = await handleGetSegments(req, res);
-
-        // If false, res.send has already been called
-        if (segments) {
-            //send result
-            return res.send(segments);
-        }
-    } catch (err) /* istanbul ignore next */ {
-        if (err instanceof SyntaxError) {
-            return res.status(400).send("Categories parameter does not match format requirements.");
-        } else return res.sendStatus(500);
-    }
-}
-
 export {
     getSegmentsByVideoID,
     getSegmentsByHash,
-    endpoint,
-    handleGetSegments
+    getSkipSegments,
+    oldGetVideoSponsorTimes
 };

src/routes/getSkipSegmentsByHash.ts

@@ -1,9 +1,10 @@
 import { hashPrefixTester } from "../utils/hashPrefixTester";
 import { getSegmentsByHash } from "./getSkipSegments";
 import { Request, Response } from "express";
-import { ActionType, Category, SegmentUUID, VideoIDHash, Service } from "../types/segments.model";
-import { getService } from "../utils/getService";
+import { VideoIDHash } from "../types/segments.model";
 import { Logger } from "../utils/logger";
+import { parseSkipSegments } from "../utils/parseSkipSegments";
+import { getEtag } from "../middleware/etag";
 
 export async function getSkipSegmentsByHash(req: Request, res: Response): Promise<Response> {
     let hashPrefix = req.params.prefix as VideoIDHash;
@@ -12,63 +13,19 @@ export async function getSkipSegmentsByHash(req: Request, res: Response): Promis
     }
     hashPrefix = hashPrefix.toLowerCase() as VideoIDHash;
 
-    let categories: Category[] = [];
-    try {
-        categories = req.query.categories
-            ? JSON.parse(req.query.categories as string)
-            : req.query.category
-                ? Array.isArray(req.query.category)
-                    ? req.query.category
-                    : [req.query.category]
-                : ["sponsor"];
-        if (!Array.isArray(categories)) {
-            return res.status(400).send("Categories parameter does not match format requirements.");
-        }
-    } catch(error) {
-        return res.status(400).send("Bad parameter: categories (invalid JSON)");
+    const parseResult = parseSkipSegments(req);
+    if (parseResult.errors.length > 0) {
+        return res.status(400).send(parseResult.errors);
     }
-
-    let actionTypes: ActionType[] = [];
-    try {
-        actionTypes = req.query.actionTypes
-            ? JSON.parse(req.query.actionTypes as string)
-            : req.query.actionType
-                ? Array.isArray(req.query.actionType)
-                    ? req.query.actionType
-                    : [req.query.actionType]
-                : [ActionType.Skip];
-        if (!Array.isArray(actionTypes)) {
-            return res.status(400).send("actionTypes parameter does not match format requirements.");
-        }
-    } catch(error) {
-        return res.status(400).send("Bad parameter: actionTypes (invalid JSON)");
-    }
-
-    let requiredSegments: SegmentUUID[] = [];
-    try {
-        requiredSegments = req.query.requiredSegments
-            ? JSON.parse(req.query.requiredSegments as string)
-            : req.query.requiredSegment
-                ? Array.isArray(req.query.requiredSegment)
-                    ? req.query.requiredSegment
-                    : [req.query.requiredSegment]
-                : [];
-        if (!Array.isArray(requiredSegments)) {
-            return res.status(400).send("requiredSegments parameter does not match format requirements.");
-        }
-    } catch(error) {
-        return res.status(400).send("Bad parameter: requiredSegments (invalid JSON)");
-    }
-
-    const service: Service = getService(req.query.service, req.body.service);
-
-    // filter out none string elements, only flat array with strings is valid
-    categories = categories.filter((item: any) => typeof item === "string");
+    const { categories, actionTypes, requiredSegments, service } = parseResult;
 
     // Get all video id's that match hash prefix
     const segments = await getSegmentsByHash(req, hashPrefix, categories, actionTypes, requiredSegments, service);
 
     try {
+        await getEtag("skipSegmentsHash", hashPrefix, service)
+            .then(etag => res.set("ETag", etag))
+            .catch(() => null);
         const output = Object.entries(segments).map(([videoID, data]) => ({
             videoID,
             segments: data.segments,

src/routes/oldGetVideoSponsorTimes.ts

@@ -1,24 +0,0 @@
-import { handleGetSegments } from "./getSkipSegments";
-import { Request, Response } from "express";
-
-export async function oldGetVideoSponsorTimes(req: Request, res: Response): Promise<Response> {
-    const segments = await handleGetSegments(req, res);
-
-    if (segments) {
-        // Convert to old outputs
-        const sponsorTimes = [];
-        const UUIDs = [];
-
-        for (const segment of segments) {
-            sponsorTimes.push(segment.segment);
-            UUIDs.push(segment.UUID);
-        }
-
-        return res.send({
-            sponsorTimes,
-            UUIDs,
-        });
-    }
-
-    // Error has already been handled in the other method
-}

src/routes/viewedVideoSponsorTime.ts

@@ -2,9 +2,9 @@ import { db } from "../databases/databases";
 import { Request, Response } from "express";
 
 export async function viewedVideoSponsorTime(req: Request, res: Response): Promise<Response> {
-    const UUID = req.query.UUID;
+    const UUID = req.query?.UUID;
 
-    if (UUID == undefined) {
+    if (!UUID) {
         //invalid request
         return res.sendStatus(400);
     }

src/utils/parseSkipSegments.ts

@@ -0,0 +1,75 @@
+import { Request } from "express";
+import { ActionType, SegmentUUID, Category, Service } from "../types/segments.model";
+import { getService } from "./getService";
+
+type fn = (req: Request) => any[];
+
+const syntaxErrorWrapper = (fn: fn, req: Request) => {
+    try { return fn(req); }
+    catch (e) { return undefined; }
+};
+
+// Default to sponsor
+const getCategories = (req: Request): Category[] =>
+    req.query.categories
+        ? JSON.parse(req.query.categories as string)
+        : req.query.category
+            ? Array.isArray(req.query.category)
+                ? req.query.category
+                : [req.query.category]
+            : ["sponsor"];
+
+// Default to skip
+const getActionTypes = (req: Request): ActionType[] =>
+    req.query.actionTypes
+        ? JSON.parse(req.query.actionTypes as string)
+        : req.query.actionType
+            ? Array.isArray(req.query.actionType)
+                ? req.query.actionType
+                : [req.query.actionType]
+            : [ActionType.Skip];
+
+// Default to empty array
+const getRequiredSegments = (req: Request): SegmentUUID[] =>
+    req.query.requiredSegments
+        ? JSON.parse(req.query.requiredSegments as string)
+        : req.query.requiredSegment
+            ? Array.isArray(req.query.requiredSegment)
+                ? req.query.requiredSegment
+                : [req.query.requiredSegment]
+            : [];
+
+const errorMessage = (parameter: string) => `${parameter} parameter does not match format requirements.`;
+
+export function parseSkipSegments(req: Request): {
+    categories: Category[];
+    actionTypes: ActionType[];
+    requiredSegments: SegmentUUID[];
+    service: Service;
+    errors: string[];
+} {
+    let categories: Category[] = syntaxErrorWrapper(getCategories, req);
+    const actionTypes: ActionType[] = syntaxErrorWrapper(getActionTypes, req);
+    const requiredSegments: SegmentUUID[] = syntaxErrorWrapper(getRequiredSegments, req);
+    const service: Service = getService(req.query.service, req.body.services);
+    const errors: string[] = [];
+    if (!Array.isArray(categories)) errors.push(errorMessage("categories"));
+    else {
+        // check category names for invalid characters
+        // and none string elements
+        categories = categories
+            .filter((item: any) => typeof item === "string")
+            .filter((category) => !(/[^a-z|_|-]/.test(category)));
+        if (categories.length === 0) errors.push("No valid categories provided.");
+    }
+    if (!Array.isArray(actionTypes)) errors.push(errorMessage("actionTypes"));
+    if (!Array.isArray(requiredSegments)) errors.push(errorMessage("requiredSegments"));
+    // finished parsing
+    return {
+        categories,
+        actionTypes,
+        requiredSegments,
+        service,
+        errors
+    };
+}

src/utils/queryCacher.ts

@@ -87,6 +87,17 @@ function clearSegmentCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoID
     }
 }
 
+async function getKeyLastModified(key: string): Promise<Date> {
+    if (!config.redis?.enabled) return Promise.reject("ETag - Redis not enabled");
+    return await redis.ttl(key)
+        .then(ttl => {
+            const sinceLive = config.redis?.expiryTime - ttl;
+            const now = Math.floor(Date.now() / 1000);
+            return new Date((now-sinceLive) * 1000);
+        })
+        .catch(() => Promise.reject("ETag - Redis error"));
+}
+
 function clearRatingCache(videoInfo: { hashedVideoID: VideoIDHash; service: Service;}): void {
     if (videoInfo) {
         redis.del(ratingHashKey(videoInfo.hashedVideoID, videoInfo.service)).catch((err) => Logger.error(err));
@@ -101,6 +112,7 @@ export const QueryCacher = {
     get,
     getAndSplit,
     clearSegmentCache,
+    getKeyLastModified,
     clearRatingCache,
-    clearFeatureCache
+    clearFeatureCache,
 };

src/utils/redis.ts

@@ -19,6 +19,7 @@ interface RedisSB {
     del(...keys: [RedisCommandArgument]): Promise<number>;
     increment?(key: RedisCommandArgument): Promise<RedisCommandRawReply[]>;
     sendCommand(args: RedisCommandArguments, options?: RedisClientOptions): Promise<RedisReply>;
+    ttl(key: RedisCommandArgument): Promise<number>;
     quit(): Promise<void>;
 }
 
@@ -30,6 +31,7 @@ let exportClient: RedisSB = {
     increment: () => new Promise((resolve) => resolve(null)),
     sendCommand: () => new Promise((resolve) => resolve(null)),
     quit: () => new Promise((resolve) => resolve(null)),
+    ttl: () => new Promise((resolve) => resolve(null)),
 };
 
 let lastClientFail = 0;