Improve performance of segment querying endpoints

src/routes/getSkipSegments.js

@@ -8,51 +8,102 @@ var logger = require('../utils/logger.js');
 var getHash = require('../utils/getHash.js');
 var getIP = require('../utils/getIP.js');
 
-function cleanGetSegments(req, videoID, categories) {
-    let userHashedIP, shadowHiddenSegments;
+function prepareCategorySegments(req, videoID, category, segments, cache = {shadowHiddenSegments: {}}) {
+    const filteredSegments = segments.filter((segment) => {
+        if (segment.votes < -1) {
+            return false; //too untrustworthy, just ignore it
+        }
 
-    let segments = [];
+        //check if shadowHidden
+        //this means it is hidden to everyone but the original ip that submitted it
+        if (segment.shadowHidden != 1) {
+            return true;
+        }
+
+        if (cache.shadowHiddenSegments[videoID] === undefined) {
+            cache.shadowHiddenSegments[videoID] = privateDB.prepare('all', 'SELECT hashedIP FROM sponsorTimes WHERE videoID = ?', [videoID]);
+        }
+
+        //if this isn't their ip, don't send it to them
+        return cache.shadowHiddenSegments[videoID].some((shadowHiddenSegment) => {
+            if (cache.userHashedIP === undefined) {
+                //hash the IP only if it's strictly necessary
+                cache.userHashedIP = getHash(getIP(req) + config.globalSalt);
+            }
+
+            return shadowHiddenSegment.hashedIP === cache.userHashedIP;
+        });
+    });
+
+    return chooseSegments(filteredSegments).map((chosenSegment) => ({
+        category,
+        segment: [chosenSegment.startTime, chosenSegment.endTime],
+        UUID: chosenSegment.UUID,
+    }));
+}
+
+function getSegmentsByVideoID(req, videoID, categories) {
+    const cache = {};
+    const segments = [];
 
     try {
-        for (const category of categories) {
-            const categorySegments = db
-                .prepare(
-                    'all',
-                    'SELECT startTime, endTime, votes, UUID, shadowHidden FROM sponsorTimes WHERE videoID = ? and category = ? ORDER BY startTime',
-                    [videoID, category]
-                )
-                .filter(segment => {
-                    if (segment.votes < -1) {
-                        return false; //too untrustworthy, just ignore it
-                    }
+        const segmentsByCategory = db
+            .prepare(
+                'all',
+                `SELECT startTime, endTime, votes, UUID, category, shadowHidden FROM sponsorTimes WHERE videoID = ? AND category IN (${Array(categories.length).fill('?').join()}) ORDER BY startTime`,
+                [videoID, categories]
+            ).reduce((acc, segment) => {
+                acc[segment.category] = acc[segment.category] || [];
+                acc[segment.category].push(segment);
 
-                    //check if shadowHidden
-                    //this means it is hidden to everyone but the original ip that submitted it
-                    if (segment.shadowHidden != 1) {
-                        return true;
-                    }
+                return acc;
+            }, {});
 
-                    if (shadowHiddenSegments === undefined) {
-                        shadowHiddenSegments = privateDB.prepare('all', 'SELECT hashedIP FROM sponsorTimes WHERE videoID = ?', [videoID]);
-                    }
+        for (const [category, categorySegments] of Object.entries(segmentsByCategory)) {
+            segments.push(...prepareCategorySegments(req, videoID, category, categorySegments, cache));
+        }
 
-                    //if this isn't their ip, don't send it to them
-                    return shadowHiddenSegments.some(shadowHiddenSegment => {
-                        if (userHashedIP === undefined) {
-                            //hash the IP only if it's strictly necessary
-                            userHashedIP = getHash(getIP(req) + config.globalSalt);
-                        }
-                        return shadowHiddenSegment.hashedIP === userHashedIP;
-                    });
-                });
+        return segments;
+    } catch (err) {
+        if (err) {
+            logger.error(err);
+            return null;
+        }
+    }
+}
 
-            chooseSegments(categorySegments).forEach(chosenSegment => {
-                segments.push({
-                    category,
-                    segment: [chosenSegment.startTime, chosenSegment.endTime],
-                    UUID: chosenSegment.UUID,
-                });
-            });
+function getSegmentsByHash(req, hashedVideoIDPrefix, categories) {
+    const cache = {};
+    const segments = {};
+
+    try {
+        const allSegments = db
+            .prepare(
+                'all',
+                `SELECT videoID, startTime, endTime, votes, UUID, category, shadowHidden, hashedVideoID FROM sponsorTimes WHERE hashedVideoID LIKE ? AND category IN (${Array(categories.length).fill('?').join()}) ORDER BY startTime`,
+                [hashedVideoIDPrefix + '%', categories]
+            ).reduce((acc, segment) => {
+                acc[segment.videoID] = acc[segment.videoID] || {
+                    hash: segment.hashedVideoID,
+                    categories: {},
+                };
+                const videoCategories = acc[segment.videoID].categories;
+
+                videoCategories[segment.category] = videoCategories[segment.category] || [];
+                videoCategories[segment.category].push(segment);
+
+                return acc;
+            }, {});
+
+        for (const [videoID, videoData] of Object.entries(allSegments)) {
+            segments[videoID] = {
+                hash: videoData.hash,
+                segments: [],
+            };
+
+            for (const [category, categorySegments] of Object.entries(videoData.categories)) {
+                segments[videoID].segments.push(...prepareCategorySegments(req, videoID, category, categorySegments, cache));
+            }
         }
 
         return segments;
@@ -160,14 +211,14 @@ function handleGetSegments(req, res) {
         ? [req.query.category]
         : ['sponsor'];
 
-    let segments = cleanGetSegments(req, videoID, categories);
+    const segments = getSegmentsByVideoID(req, videoID, categories);
 
     if (segments === null || segments === undefined) {
         res.sendStatus(500);
         return false;
     }
 
-    if (segments.length == 0) {
+    if (segments.length === 0) {
         res.sendStatus(404);
         return false;
     }
@@ -177,7 +228,8 @@ function handleGetSegments(req, res) {
 
 module.exports = {
     handleGetSegments,
-    cleanGetSegments,
+    getSegmentsByVideoID,
+    getSegmentsByHash,
     endpoint: function (req, res) {
         let segments = handleGetSegments(req, res);
 

src/routes/getSkipSegmentsByHash.js

@@ -1,5 +1,5 @@
 const hashPrefixTester = require('../utils/hashPrefixTester.js');
-const getSegments = require('./getSkipSegments.js').cleanGetSegments;
+const getSegments = require('./getSkipSegments.js').getSegmentsByHash;
 
 const databases = require('../databases/databases.js');
 const logger = require('../utils/logger.js');
@@ -19,15 +19,15 @@ module.exports = async function (req, res) {
     : ['sponsor'];
 
     // Get all video id's that match hash prefix
-    const videoIds = db.prepare('all', 'SELECT DISTINCT videoId, hashedVideoID from sponsorTimes WHERE hashedVideoID LIKE ?', [hashPrefix+'%']);
+    const segments = getSegments(req, hashPrefix, categories);
 
-    let segments = videoIds.map((video) => {
-        return {
-            videoID: video.videoID,
-            hash: video.hashedVideoID,
-            segments: getSegments(req, video.videoID, categories)
-        };
-    });
+    if (!segments) return res.status(404).json([]);
 
-    res.status((segments.length === 0) ? 404 : 200).json(segments);
-}
+    const output = Object.entries(segments).map(([videoID, data]) => ({
+        videoID,
+        hash: data.hash,
+        segments: data.segments,
+    }));
+
+    res.status(output.length === 0 ? 404 : 200).json(output);
+}

test/cases/getSegmentsByHash.js

@@ -24,17 +24,17 @@ describe('getSegmentsByHash', () => {
       });
   });
 
-  it('Should be able to get a 200 with empty segments for video but no matching categories', (done) => {
+  it('Should return 404 if no segments are found even if a video for the given hash is known', (done) => {
     request.get(utils.getbaseURL() 
      + '/api/skipSegments/3272f?categories=["shilling"]', null, 
       (err, res, body) => {
         if (err) done("Couldn't call endpoint");
-        else if (res.statusCode !== 200) done("non 200 status code, was " + res.statusCode);
+        else if (res.statusCode !== 404) done("non 404 status code, was " + res.statusCode);
         else {
-          if (JSON.parse(body) && JSON.parse(body).length > 0 && JSON.parse(body)[0].segments.length === 0) {
+          if (body === '[]') {
             done(); // pass
           } else {
-            done("response had segments");
+            done("response had videos");
           }
         }
       });