Merge branch 'master' of https://github.com/ajayyy/SponsorBlockServer into eslint-gh-actions

databases/_upgrade_sponsorTimes_18.sql

@@ -0,0 +1,9 @@
+BEGIN TRANSACTION;
+
+/* Add hash field */
+ALTER TABLE "lockCategories" ADD "hashedVideoID" TEXT NOT NULL default '';
+UPDATE "lockCategories" SET "hashedVideoID" = sha256("videoID");
+
+UPDATE "config" SET value = 18 WHERE key = 'version';
+
+COMMIT;

package.json

@@ -9,8 +9,8 @@
     "dev:bash": "nodemon -x 'npm test ; npm start'",
     "start": "ts-node src/index.ts",
     "tsc": "tsc -p tsconfig.json",
-    "lint": "eslint src",
-    "lint:fix": "eslint src --fix"
+    "lint": "eslint src test",
+    "lint:fix": "eslint src test --fix"
   },
   "author": "Ajay Ramachandran",
   "license": "MIT",

src/app.ts

@@ -33,6 +33,8 @@ import {postClearCache} from './routes/postClearCache';
 import { addUnlistedVideo } from './routes/addUnlistedVideo';
 import {postPurgeAllSegments} from './routes/postPurgeAllSegments';
 import {getUserID} from './routes/getUserID';
+import {getLockCategories} from './routes/getLockCategories';
+import {getLockCategoriesByHash} from './routes/getLockCategoriesByHash';
 import ExpressPromiseRouter from 'express-promise-router';
 import { Server } from 'http';
 
@@ -156,6 +158,12 @@ function setupRoutes(router: Router) {
     // get userID from username
     router.get('/api/userID', getUserID);
 
+    // get lock categores from userID
+    router.get('/api/lockCategories', getLockCategories);
+
+    // get privacy protecting lock categories functions
+    router.get('/api/lockCategories/:prefix', getLockCategoriesByHash);
+
     if (config.postgres) {
         router.get('/database', (req, res) => dumpDatabase(req, res, true));
         router.get('/database.json', (req, res) => dumpDatabase(req, res, false));

src/config.ts

@@ -84,7 +84,6 @@ addDefaults(config, {
 function addDefaults(config: SBSConfig, defaults: SBSConfig) {
     for (const key in defaults) {
         if (!Object.prototype.hasOwnProperty.call(config, key)) {
-            // @ts-ignore
             config[key] = defaults[key];
         }
     }

src/routes/getLockCategories.ts

@@ -0,0 +1,27 @@
+import {db} from '../databases/databases';
+import {Logger} from '../utils/logger';
+import {Request, Response} from 'express';
+import { Category, VideoID } from "../types/segments.model";
+
+export async function getLockCategories(req: Request, res: Response): Promise<Response> {
+    const videoID = req.query.videoID as VideoID;
+
+    if (videoID == undefined) {
+        //invalid request
+        return res.sendStatus(400);
+    }
+
+    try {
+        // Get existing lock categories markers
+        const lockedCategories = await db.prepare('all', 'SELECT "category" from "lockCategories" where "videoID" = ?', [videoID]) as {category: Category}[];
+        if (lockedCategories.length === 0 || !lockedCategories[0]) return res.sendStatus(404);
+        // map to array in JS becaues of SQL incompatibilities
+        const categories = Object.values(lockedCategories).map((entry) => entry.category);
+        return res.send({
+            categories
+        });
+    } catch (err) {
+        Logger.error(err);
+        return res.sendStatus(500);
+    }
+}

src/routes/getLockCategoriesByHash.ts

@@ -0,0 +1,56 @@
+import {db} from '../databases/databases';
+import {Logger} from '../utils/logger';
+import {Request, Response} from 'express';
+import {hashPrefixTester} from '../utils/hashPrefixTester';
+import { Category, VideoID, VideoIDHash } from "../types/segments.model";
+
+interface LockResultByHash {
+    videoID: VideoID,
+    hash: VideoIDHash,
+    categories: Category[]
+}
+
+interface DBLock {
+    videoID: VideoID,
+    hash: VideoIDHash,
+    category: Category
+}
+
+const mergeLocks = (source: DBLock[]) => {
+    const dest: LockResultByHash[] = [];
+    for (const obj of source) {
+        // videoID already exists
+        const destMatch = dest.find(s => s.videoID === obj.videoID);
+        if (destMatch) {
+            // push to categories
+            destMatch.categories.push(obj.category);
+        } else {
+            dest.push({
+                videoID: obj.videoID,
+                hash: obj.hash,
+                categories: [obj.category]
+            });
+        }
+    }
+    return dest;
+};
+
+
+export async function getLockCategoriesByHash(req: Request, res: Response): Promise<Response> {
+    let hashPrefix = req.params.prefix as VideoIDHash;
+    if (!hashPrefixTester(req.params.prefix)) {
+        return res.status(400).send("Hash prefix does not match format requirements."); // Exit early on faulty prefix
+    }
+    hashPrefix = hashPrefix.toLowerCase() as VideoIDHash;
+
+    try {
+        // Get existing lock categories markers
+        const lockedRows = await db.prepare('all', 'SELECT "videoID", "hashedVideoID" as "hash", "category" from "lockCategories" where "hashedVideoID" LIKE ?', [hashPrefix + '%']) as DBLock[];
+        if (lockedRows.length === 0 || !lockedRows[0]) return res.sendStatus(404);
+        // merge all locks
+        return res.send(mergeLocks(lockedRows));
+    } catch (err) {
+        Logger.error(err);
+        return res.sendStatus(500);
+    }
+}

src/types/config.model.ts

@@ -3,6 +3,7 @@ import * as redis from 'redis';
 import { CacheOptions } from "@ajayyy/lru-diskcache";
 
 export interface SBSConfig {
+    [index: string]: any
     port: number;
     mockPort?: number;
     globalSalt: string;

src/utils/createMemoryCache.ts

@@ -1,4 +1,4 @@
-export function createMemoryCache(memoryFn: (...args: any[]) => void, cacheTimeMs: number) {
+export function createMemoryCache(memoryFn: (...args: any[]) => void, cacheTimeMs: number): any {
     if (isNaN(cacheTimeMs)) cacheTimeMs = 0;
 
     // holds the promise results
@@ -22,8 +22,8 @@ export function createMemoryCache(memoryFn: (...args: any[]) => void, cacheTimeM
                 }
             }
             // create new promise
-            const promise = new Promise(async (resolve) => {
-                resolve((await memoryFn(...args)));
+            const promise = new Promise((resolve) => {
+                resolve(memoryFn(...args));
             });
             // store promise reference until fulfilled
             promiseMemory.set(cacheKey, promise);

src/utils/diskCache.ts

@@ -8,6 +8,7 @@ if (config.diskCache) {
     DiskCache.init();
 } else {
     DiskCache = {
+        /* eslint-disable @typescript-eslint/no-unused-vars */
         // constructor(rootPath, options): {};
 
         init(): void { return; },
@@ -16,16 +17,17 @@ if (config.diskCache) {
 
         has(key: string): boolean { return false; },
 
-        get(key: string, opts): string { return null; },
+        get(key: string, opts?: {encoding?: string}): string { return null; },
 
         // Returns size
-        set(key: string, dataOrSteam): Promise<number> { return new Promise(() => 0); },
+        set(key: string, dataOrSteam: string): Promise<number> { return new Promise(() => 0); },
 
         del(key: string): void { return; },
 
         size(): number { return 0; },
 
         prune(): void {return; },
+        /* eslint-enable @typescript-eslint/no-unused-vars */
     };
 }
 

test/cases/getLockCategories.ts

@@ -0,0 +1,84 @@
+import fetch from 'node-fetch';
+import {Done, getbaseURL} from '../utils';
+import {getHash} from '../../src/utils/getHash';
+import {db} from '../../src/databases/databases';
+
+
+describe('getLockCategories', () => {
+    before(async () => {
+        const insertVipUserQuery = 'INSERT INTO "vipUsers" ("userID") VALUES (?)';
+        await db.prepare("run", insertVipUserQuery, [getHash("VIPUser-getLockCategories")]);
+ 
+        const insertLockCategoryQuery = 'INSERT INTO "lockCategories" ("userID", "videoID", "category") VALUES (?, ?, ?)';
+        await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-getLockCategories"), 'getLock-1', 'sponsor']);
+        await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-getLockCategories"), 'getLock-1', 'interaction']);
+ 
+        await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-getLockCategories"), 'getLock-2', 'preview']);
+ 
+        await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-getLockCategories"), 'getLock-3', 'nonmusic']);
+    });
+
+    it('Should update the database version when starting the application', async () => {
+        const version = (await db.prepare('get', 'SELECT key, value FROM config where key = ?', ['version'])).value;
+        if (version > 1) return;
+        else return 'Version isn\'t greater than 1. Version is ' + version;
+    });
+
+    it('Should be able to get multiple locks', (done: Done) => {
+        fetch(getbaseURL() + '/api/lockCategories?videoID=getLock-1')
+        .then(async res => {
+            if (res.status !== 200) {
+                done("non 200");
+            } else {
+                const data = await res.json();
+                if (data.categories.length !== 2) {
+                    done(`Returned incorrect number of locks "${data.categories.length}"`);
+                } else if (data.categories[0] !== "sponsor") {
+                    done(`Returned incorrect category "${data.categories[0]}"`);
+                } else if (data.categories[1] !== "interaction") {
+                    done(`Returned incorrect category "${data.categories[1]}"`);
+                } else {
+                    done(); // pass
+                }
+            }
+        })
+        .catch(() => ("couldn't call endpoint"));
+    });
+
+    it('Should be able to get single locks', (done: Done) => {
+        fetch(getbaseURL() + '/api/lockCategories?videoID=getLock-2')
+        .then(async res => {
+            if (res.status !== 200) {
+                done("non 200");
+            } else {
+                const data = await res.json();
+                if (data.categories.length !== 1) {
+                    done('Returned incorrect number of locks "' + data.categories.length + '"');
+                } else if (data.categories[0] !== "preview") {
+                    done(`Returned incorrect category "${data.categories[0].category}"`);
+                } else {
+                    done(); // pass
+                }
+            }
+        })
+        .catch(() => ("couldn't call endpoint"));
+    });
+
+    it('should return 404 if no lock exists', (done: Done) => {
+        fetch(getbaseURL() + '/api/lockCategories?videoID=getLock-0')
+        .then(res => {
+            if (res.status !== 404) done('non 404 (' + res.status + ')');
+            else done(); // pass
+        })
+        .catch(() => ("couldn't call endpoint"));
+    });
+
+    it('should return 400 if no videoID specified', (done: Done) => {
+        fetch(getbaseURL() + '/api/lockCategories')
+        .then(res => {
+            if (res.status !== 400) done('non 400 (' + res.status + ')');
+            else done(); // pass
+        })
+        .catch(() => ("couldn't call endpoint"));
+    });
+});

test/cases/getLockCategoriesByHash.ts

@@ -0,0 +1,176 @@
+import fetch from 'node-fetch';
+import {Done, getbaseURL} from '../utils';
+import {getHash} from '../../src/utils/getHash';
+import {db} from '../../src/databases/databases';
+
+
+describe('getLockCategoriesByHash', () => {
+    before(async () => {
+        const insertVipUserQuery = 'INSERT INTO "vipUsers" ("userID") VALUES (?)';
+        await db.prepare("run", insertVipUserQuery, [getHash("VIPUser-getLockCategories")]);
+ 
+        const insertLockCategoryQuery = 'INSERT INTO "lockCategories" ("userID", "videoID", "category", "hashedVideoID") VALUES (?, ?, ?, ?)';
+        await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-getLockCategories"), 'getLockHash-1', 'sponsor', '67a654898fda3a5541774aea345796c7709982bb6018cb08d22a18eeddccc1d0']);
+        await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-getLockCategories"), 'getLockHash-1', 'interaction', '67a654898fda3a5541774aea345796c7709982bb6018cb08d22a18eeddccc1d0']);
+ 
+        await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-getLockCategories"), 'getLockHash-2', 'preview', 'dff09120437b4bd594dffae5f3cde3cfc5f6099fb01d0ef4051919b2908d9a50']);
+ 
+        await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-getLockCategories"), 'getLockHash-3', 'nonmusic', 'bf1b122fd5630e0df8626d00c4a95c58954ad715e5595b0f75a19ac131e28928']);
+
+        await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-getLockCategories"), 'fakehash-1', 'outro', 'b05a20424f24a53dac1b059fb78d861ba9723645026be2174c93a94f9106bb35']);
+        await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-getLockCategories"), 'fakehash-2', 'intro', 'b05acd1cd6ec7dffe5ffea64ada91ae7469d6db2ce21c7e30ad7fa62075d450']);
+        await db.prepare("run", insertLockCategoryQuery, [getHash("VIPUser-getLockCategories"), 'fakehash-2', 'preview', 'b05acd1cd6ec7dffe5ffea64ada91ae7469d6db2ce21c7e30ad7fa62075d450']);
+    });
+
+    it('Database should be greater or equal to version 18', async () => {
+        const version = (await db.prepare('get', 'SELECT key, value FROM config where key = ?', ['version'])).value;
+        if (version >= 18) return;
+        else return 'Version isn\'t greater than 18. Version is ' + version;
+    });
+
+    it('Should be able to get multiple locks in one object', (done: Done) => {
+        fetch(getbaseURL() + '/api/lockCategories/67a65')
+        .then(async res => {
+            if (res.status !== 200) {
+                done("non 200");
+            } else {
+                const data = await res.json();
+                if (data.length !== 1) {
+                    done(`Returned incorrect number of videos "${data.length}"`);
+                } else if (data[0].videoID !== "getLockHash-1") {
+                    done(`Returned incorrect videoID "${data[0].videoID}"`);
+                } else if (data[0].hash !== getHash("getLockHash-1", 1)) {
+                    done(`Returned incorrect hash "${data[0].hash}"`);
+                } else if (data[0].categories[0] !== "sponsor") {
+                    done(`Returned incorrect category "${data[0].categories[0]}"`);
+                } else if (data[0].categories[1] !== "interaction") {
+                    done(`Returned incorrect category "${data[0].categories[1]}"`);
+                } else {
+                    done(); // pass
+                }
+            }
+        })
+        .catch(() => ("couldn't call endpoint"));
+    });
+
+    it('Should be able to get single lock', (done: Done) => {
+        fetch(getbaseURL() + '/api/lockCategories/dff09')
+        .then(async res => {
+            if (res.status !== 200) {
+                done("non 200");
+            } else {
+                const data = await res.json();
+                if (data.length !== 1) {
+                    done('Returned incorrect number of videos "' + data.length + '"');
+                } else if (data[0].videoID !== "getLockHash-2") {
+                    done(`Returned incorrect videoID "${data[0].videoID}"`);
+                } else if (data[0].hash !== getHash("getLockHash-2", 1)) {
+                    done(`Returned incorrect hashedVideoID hash "${data[0].hash}"`);
+                } else if (data[0].categories.length !== 1) {
+                    done(`Returned incorrect number of categories "${data[0].categories.length}"`);
+                } else if (data[0].categories[0] !== "preview") {
+                    done(`Returned incorrect category "${data[0].categories[0]}"`);
+                } else {
+                    done(); // pass
+                }
+            }
+        })
+        .catch(() => ("couldn't call endpoint"));
+    });
+    
+    it('Should be able to get by half full hash', (done: Done) => {
+        fetch(getbaseURL() + '/api/lockCategories/bf1b122fd5630e0df8626d00c4a95c58')
+        .then(async res => {
+            if (res.status !== 200) {
+                done("non 200");
+            } else {
+                const data = await res.json();
+                if (data.length !== 1) {
+                    done('Returned incorrect number of videos "' + data.length + '"');
+                } else if (data[0].videoID !== "getLockHash-3") {
+                    done(`Returned incorrect videoID "${data[0].videoID}"`);
+                } else if (data[0].hash !== getHash("getLockHash-3", 1)) {
+                    done(`Returned incorrect hashedVideoID hash "${data[0].hash}"`);
+                } else if (data[0].categories.length !== 1) {
+                    done(`Returned incorrect number of categories "${data[0].categories.length}"`);
+                } else if (data[0].categories[0] !== "nonmusic") {
+                    done(`Returned incorrect category "${data[0].categories[0]}"`);
+                } else {
+                    done(); // pass
+                }
+            }
+        })
+        .catch(() => ("couldn't call endpoint"));
+    });
+
+    it('Should be able to get multiple by similar hash with multiple categories', (done: Done) => {
+        fetch(getbaseURL() + '/api/lockCategories/b05a')
+        .then(async res => {
+            if (res.status !== 200) {
+                done("non 200");
+            } else {
+                const data = await res.json();
+                if (data.length !== 2) {
+                    done(`Returned incorrect number of locks "${data.length}"`);
+                } else if (data[0].videoID !== "fakehash-1") {
+                    done(`Returned incorrect videoID "${data[0].videoID}"`);
+                } else if (data[1].videoID !== "fakehash-2") {
+                    done(`Returned incorrect videoID "${data[1].videoID}"`);
+                } else if (data[0].hash !== "b05a20424f24a53dac1b059fb78d861ba9723645026be2174c93a94f9106bb35") {
+                    done(`Returned incorrect hashedVideoID hash "${data[0].hash}"`);
+                } else if (data[1].hash !== "b05acd1cd6ec7dffe5ffea64ada91ae7469d6db2ce21c7e30ad7fa62075d450") {
+                    done(`Returned incorrect hashedVideoID hash "${data[1].hash}"`);
+                } else if (data[0].categories.length !== 1) {
+                    done(`Returned incorrect number of categories "${data[0].categories.length}"`);
+                } else if (data[1].categories.length !== 2) {
+                    done(`Returned incorrect number of categories "${data[1].categories.length}"`);
+                } else if (data[0].categories[0] !== "outro") {
+                    done(`Returned incorrect category "${data[0].category}"`);
+                } else if (data[1].categories[0] !== "intro") {
+                    done(`Returned incorrect category "${data[1].category}"`);
+                } else if (data[1].categories[1] !== "preview") {
+                    done(`Returned incorrect category "${data[1].category}"`);
+                } else {
+                    done(); // pass
+                }
+            }
+        })
+        .catch(() => ("couldn't call endpoint"));
+    });
+
+    it('should return 404 once hash prefix varies', (done: Done) => {
+        fetch(getbaseURL() + '/api/lockCategories/aaaaaa')
+        .then(res => {
+            if (res.status !== 404) done('non 404 (' + res.status + ')');
+            else done(); // pass
+        })
+        .catch(() => ("couldn't call endpoint"));
+    });
+
+    it('should return 404 if no lock exists', (done: Done) => {
+        fetch(getbaseURL() + '/api/lockCategories/aaaaaa')
+        .then(res => {
+            if (res.status !== 404) done('non 404 (' + res.status + ')');
+            else done(); // pass
+        })
+        .catch(() => ("couldn't call endpoint"));
+    });
+
+    it('should return 400 if no videoID specified', (done: Done) => {
+        fetch(getbaseURL() + '/api/lockCategories/')
+        .then(res => {
+            if (res.status !== 400) done('non 400 (' + res.status + ')');
+            else done(); // pass
+        })
+        .catch(() => ("couldn't call endpoint"));
+    });
+
+    it('should return 400 if full hash sent', (done: Done) => {
+        fetch(getbaseURL() + '/api/lockCategories/b05a20424f24a53dac1b059fb78d861ba9723645026be2174c93a94f9106bb35')
+        .then(res => {
+            if (res.status !== 400) done('non 400 (' + res.status + ')');
+            else done(); // pass
+        })
+        .catch(() => ("couldn't call endpoint"));
+    });
+});

test/cases/getSkipSegments.ts

@@ -326,7 +326,7 @@ describe('getSkipSegments', () => {
                 else done();
             }
         })
-        .catch(err => done("Couldn't call endpoint"));
+        .catch(() => done("Couldn't call endpoint"));
     });
 
     it('Should be able to get specific segments with repeating requiredSegment', (done: Done) => {
@@ -341,6 +341,6 @@ describe('getSkipSegments', () => {
                 else done();
             }
         })
-        .catch(err => done("Couldn't call endpoint"));
+        .catch(() => done("Couldn't call endpoint"));
     });
 });

test/cases/getSkipSegmentsByHash.ts

@@ -252,7 +252,7 @@ describe('getSegmentsByHash', () => {
                 }
             }
         })
-        .catch(err => ("Couldn't call endpoint"));
+        .catch(() => ("Couldn't call endpoint"));
     });
 
     it('Should be able to get specific segments with requiredSegments', (done: Done) => {
@@ -268,7 +268,7 @@ describe('getSegmentsByHash', () => {
                 else done();
             }
         })
-        .catch(err => done("Couldn't call endpoint"));
+        .catch(() => done("Couldn't call endpoint"));
     });
 
     it('Should be able to get specific segments with repeating requiredSegment', (done: Done) => {
@@ -284,6 +284,6 @@ describe('getSegmentsByHash', () => {
                 else done();
             }
         })
-        .catch(err => done("Couldn't call endpoint"));
+        .catch(() => done("Couldn't call endpoint"));
     });
 });

test/cases/getUserInfo.ts

@@ -165,7 +165,7 @@ describe('getUserInfo', () => {
             const data = await res.json();
             for (const value in data) {
                 if (data[value] === null && value !== "lastSegmentID")  {
-                    done(`returned null for ${value}`)
+                    done(`returned null for ${value}`);
                 }
             }
             done(); // pass

test/cases/setUsername.ts

@@ -46,6 +46,7 @@ async function getLastLogUserNameChange(userID: string) {
 }
 
 function wellFormatUserName(userName: string) {
+    // eslint-disable-next-line no-control-regex
     return userName.replace(/[\u0000-\u001F\u007F-\u009F]/g, '');
 }
 

test/mocks.ts

@@ -1,5 +1,6 @@
 import express from 'express';
 import {config} from '../src/config';
+import { Server } from 'http';
 
 const app = express();
 
@@ -46,6 +47,6 @@ app.post('/CustomWebhook', (req, res) => {
     res.sendStatus(200);
 });
 
-export function createMockServer(callback: () => void) {
+export function createMockServer(callback: () => void): Server {
     return app.listen(config.mockPort, callback);
 }

test/test.ts

@@ -18,15 +18,15 @@ async function init() {
     }));
 
     // delete old test database
-    if (fs.existsSync(config.db)) fs.unlinkSync(config.db)
+    if (fs.existsSync(config.db)) fs.unlinkSync(config.db);
     if (fs.existsSync(config.privateDB)) fs.unlinkSync(config.privateDB);
 
     await initDb();
 
     const dbMode = config.mysql ? 'mysql'
         : config.postgres ? 'postgres'
-        : 'sqlite'
-    Logger.info('Database Mode: ' + dbMode)
+        : 'sqlite';
+    Logger.info('Database Mode: ' + dbMode);
 
     // Instantiate a Mocha instance.
     const mocha = new Mocha();