From ca46f4c9ac8517c55e1b4456f3666fdf7f3b06ec Mon Sep 17 00:00:00 2001 From: Martijn van der Ven Date: Sat, 23 May 2020 16:53:45 +0200 Subject: [PATCH 01/40] Enable database upgrades to run --- src/databases/databases.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/databases/databases.js b/src/databases/databases.js index 7c3991f..d7008a0 100644 --- a/src/databases/databases.js +++ b/src/databases/databases.js @@ -49,11 +49,11 @@ function ugradeDB(db, prefix) { let versionCodeInfo = db.prepare("SELECT value FROM config WHERE key = ?").get("version"); let versionCode = versionCodeInfo ? versionCodeInfo.value : 0; - let path = config.schemaFolder + "/_upgrade_" + prefix + "_" + (versionCode + 1) + ".sql"; + let path = config.schemaFolder + "/_upgrade_" + prefix + "_" + (parseInt(versionCode) + 1) + ".sql"; while (fs.existsSync(path)) { db.exec(fs.readFileSync(path).toString()); versionCode = db.prepare("SELECT value FROM config WHERE key = ?").get("version").value; - path = config.schemaFolder + "/_upgrade_" + prefix + "_" + (versionCode + 1) + ".sql"; + path = config.schemaFolder + "/_upgrade_" + prefix + "_" + (parseInt(versionCode) + 1) + ".sql"; } } \ No newline at end of file From 803b1fa5050233c43b062c934cebd0a1365957ca Mon Sep 17 00:00:00 2001 From: Martijn van der Ven Date: Sat, 23 May 2020 16:54:21 +0200 Subject: [PATCH 02/40] Add initial version of querying by hash prefix --- databases/_upgrade_sponsorTimes_2.sql | 26 ++++++++++ src/app.js | 4 ++ src/databases/databases.js | 4 ++ src/routes/getSkipSegmentsByHash.js | 69 +++++++++++++++++++++++++++ 4 files changed, 103 insertions(+) create mode 100644 databases/_upgrade_sponsorTimes_2.sql create mode 100644 src/routes/getSkipSegmentsByHash.js diff --git a/databases/_upgrade_sponsorTimes_2.sql b/databases/_upgrade_sponsorTimes_2.sql new file mode 100644 index 0000000..5472499 --- /dev/null +++ b/databases/_upgrade_sponsorTimes_2.sql @@ -0,0 +1,26 @@ +BEGIN TRANSACTION; + +/* Add hash field */ +CREATE TABLE "sqlb_temp_table_1" ( + "videoID" TEXT NOT NULL, + "startTime" REAL NOT NULL, + "endTime" REAL NOT NULL, + "votes" INTEGER NOT NULL, + "incorrectVotes" INTEGER NOT NULL default '1', + "UUID" TEXT NOT NULL UNIQUE, + "userID" TEXT NOT NULL, + "timeSubmitted" INTEGER NOT NULL, + "views" INTEGER NOT NULL, + "category" TEXT NOT NULL DEFAULT "sponsor", + "shadowHidden" INTEGER NOT NULL, + "hashedVideoID" TEXT NOT NULL +); +INSERT INTO sqlb_temp_table_1 SELECT *, sha256(videoID) FROM sponsorTimes; + +DROP TABLE sponsorTimes; +ALTER TABLE sqlb_temp_table_1 RENAME TO "sponsorTimes"; + +/* Bump version in config */ +UPDATE config SET value = 2 WHERE key = "version"; + +COMMIT; \ No newline at end of file diff --git a/src/app.js b/src/app.js index 9561dc1..5595625 100644 --- a/src/app.js +++ b/src/app.js @@ -10,6 +10,7 @@ var loggerMiddleware = require('./middleware/logger.js'); // Routes var getSkipSegments = require('./routes/getSkipSegments.js').endpoint; var postSkipSegments = require('./routes/postSkipSegments.js'); +var getSkipSegmentsByHash = require('./routes/getSkipSegmentsByHash.js'); var voteOnSponsorTime = require('./routes/voteOnSponsorTime.js'); var viewedVideoSponsorTime = require('./routes/viewedVideoSponsorTime.js'); var setUsername = require('./routes/setUsername.js'); @@ -48,6 +49,9 @@ app.post('/api/postVideoSponsorTimes', oldSubmitSponsorTimes); app.get('/api/skipSegments', getSkipSegments); app.post('/api/skipSegments', postSkipSegments); +// add the privacy protecting skip segments functions +app.get('/api/skipSegments/:prefix', getSkipSegmentsByHash); + //voting endpoint app.get('/api/voteOnSponsorTime', voteOnSponsorTime); app.post('/api/voteOnSponsorTime', voteOnSponsorTime); diff --git a/src/databases/databases.js b/src/databases/databases.js index d7008a0..621ad8b 100644 --- a/src/databases/databases.js +++ b/src/databases/databases.js @@ -26,6 +26,10 @@ if (config.createDatabaseIfNotExist && !config.readOnly) { // Upgrade database if required if (!config.readOnly) { + // Register hashing function needed for running database upgrade + db.function("sha256", function (string) { + return require('crypto').createHash("sha256").update(string).digest("hex"); + }); ugradeDB(db, "sponsorTimes"); ugradeDB(privateDB, "private") } diff --git a/src/routes/getSkipSegmentsByHash.js b/src/routes/getSkipSegmentsByHash.js new file mode 100644 index 0000000..3672eed --- /dev/null +++ b/src/routes/getSkipSegmentsByHash.js @@ -0,0 +1,69 @@ +const config = require('../config.js'); +const { db, privateDB } = require('../databases/databases.js'); + +const getHash = require('../utils/getHash.js'); +const getIP = require('../utils/getIP.js'); + +/** + * @typedef {Object} Segment + * @property {string} videoID YouTube video ID the segment is meant for + * @property {number[]} segment Tuple of start and end times in seconds + * @property {string} category Category of content to skip + * @property {string} UUID Unique identifier for the specific segment + */ + +/** + * @param {string} prefix Lowercased hexadecimal hash prefix + * @param {string} hashedIP Custom hash of the visitor’s IP address + * @returns {Segment[]} + */ +function getSkipSegmentsByHash(prefix, hashedIP) { + /** + * @constant + * @type {Segment[]} + * @default + */ + const segments = []; + + const rows = db.prepare('SELECT videoID, startTime, endTime, UUID, category, shadowHidden FROM sponsorTimes WHERE votes >= -1 AND hashedVideoID LIKE ? ORDER BY startTime') + .all(prefix + '%'); + + const onlyForCurrentUser = privateDB.prepare('SELECT videoID FROM sponsorTimes WHERE hashedIP = ?').all(hashedIP).map(row => row.videoID); + + for (const row of rows) { + /** @TODO check if this logic does what is expected. */ + if (row.shadowHidden === 1 && onlyForCurrentUser.indexOf(row.videoID) === -1) { + // The current visitor’s IP did not submit for the current video. + // Do not send shadowHidden segments to them. + continue; + } + + segments.push({ + videoID: row.videoID, + segment: [row.startTime, row.endTime], + category: row.category, + UUID: row.UUID + }); + } + + return segments; +} + +const minimumPrefix = config.minimumPrefix || '3'; +const maximumPrefix = config.maximumPrefix || '32'; // Half the hash. +const prefixChecker = new RegExp('^[\\dA-F]{' + minimumPrefix + ',' + maximumPrefix + '}$', 'i'); + +module.exports = async function (req, res) { + if (!prefixChecker.test(req.params.prefix)) { + res.sendStatus(400).end(); // Exit early on faulty prefix + } + + const segments = getSkipSegmentsByHash( + req.params.prefix.toLowerCase(), + getHash(getIP(req) + config.globalSalt) + ); + + if (segments) { + res.send(segments) + } +} \ No newline at end of file From d8203dce77a5962076b93ab08fcc9213dc9d7d54 Mon Sep 17 00:00:00 2001 From: Martijn van der Ven Date: Sat, 23 May 2020 17:57:33 +0200 Subject: [PATCH 03/40] Make sure to always insert hashed video ID --- src/routes/postSkipSegments.js | 6 +++--- test/cases/getSavedTimeForUser.js | 4 ++-- test/cases/getSkipSegments.js | 17 +++++++++-------- test/cases/oldGetSponsorTime.js | 7 ++++--- test/cases/voteOnSponsorTime.js | 16 ++++++++-------- 5 files changed, 26 insertions(+), 24 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index e178ee6..e13280f 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -232,9 +232,9 @@ module.exports = async function postSkipSegments(req, res) { try { db.prepare("INSERT INTO sponsorTimes " + - "(videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden)" + - "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)").run(videoID, segmentInfo.segment[0], - segmentInfo.segment[1], startingVotes, UUID, userID, timeSubmitted, 0, segmentInfo.category, shadowBanned); + "(videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID)" + + "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)").run(videoID, segmentInfo.segment[0], + segmentInfo.segment[1], startingVotes, UUID, userID, timeSubmitted, 0, segmentInfo.category, shadowBanned, getHash(videoID, 0)); //add to private db as well privateDB.prepare("INSERT INTO sponsorTimes VALUES(?, ?, ?)").run(videoID, hashedIP, timeSubmitted); diff --git a/test/cases/getSavedTimeForUser.js b/test/cases/getSavedTimeForUser.js index 5df8a48..242204d 100644 --- a/test/cases/getSavedTimeForUser.js +++ b/test/cases/getSavedTimeForUser.js @@ -5,8 +5,8 @@ var getHash = require('../../src/utils/getHash.js'); describe('getSavedTimeForUser', () => { before(() => { - let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden) VALUES"; - db.exec(startOfQuery + "('getSavedTimeForUser', 1, 11, 2, 'abc1239999', '" + getHash("testman") + "', 0, 50, 'sponsor', 0)"); + let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES"; + db.exec(startOfQuery + "('getSavedTimeForUser', 1, 11, 2, 'abc1239999', '" + getHash("testman") + "', 0, 50, 'sponsor', 0, '" + getHash('getSavedTimeForUser', 0) + "')"); }); it('Should be able to get a 200', (done) => { diff --git a/test/cases/getSkipSegments.js b/test/cases/getSkipSegments.js index 5c15c63..db4698e 100644 --- a/test/cases/getSkipSegments.js +++ b/test/cases/getSkipSegments.js @@ -1,6 +1,7 @@ var request = require('request'); var db = require('../../src/databases/databases.js').db; var utils = require('../utils.js'); +var getHash = require('../../src/utils/getHash.js'); /* *CREATE TABLE IF NOT EXISTS "sponsorTimes" ( @@ -18,14 +19,14 @@ var utils = require('../utils.js'); describe('getSkipSegments', () => { before(() => { - let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden) VALUES"; - db.exec(startOfQuery + "('testtesttest', 1, 11, 2, '1-uuid-0', 'testman', 0, 50, 'sponsor', 0)"); - db.exec(startOfQuery + "('testtesttest', 20, 33, 2, '1-uuid-2', 'testman', 0, 50, 'intro', 0)"); - db.exec(startOfQuery + "('testtesttest,test', 1, 11, 2, '1-uuid-1', 'testman', 0, 50, 'sponsor', 0)"); - db.exec(startOfQuery + "('test3', 1, 11, 2, '1-uuid-4', 'testman', 0, 50, 'sponsor', 0)"); - db.exec(startOfQuery + "('test3', 7, 22, -3, '1-uuid-5', 'testman', 0, 50, 'sponsor', 0)"); - db.exec(startOfQuery + "('multiple', 1, 11, 2, '1-uuid-6', 'testman', 0, 50, 'intro', 0)"); - db.exec(startOfQuery + "('multiple', 20, 33, 2, '1-uuid-7', 'testman', 0, 50, 'intro', 0)"); + let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES"; + db.exec(startOfQuery + "('testtesttest', 1, 11, 2, '1-uuid-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('testtesttest', 0) + "')"); + db.exec(startOfQuery + "('testtesttest', 20, 33, 2, '1-uuid-2', 'testman', 0, 50, 'intro', 0, '" + getHash('testtesttest', 0) + "')"); + db.exec(startOfQuery + "('testtesttest,test', 1, 11, 2, '1-uuid-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('testtesttest,test', 0) + "')"); + db.exec(startOfQuery + "('test3', 1, 11, 2, '1-uuid-4', 'testman', 0, 50, 'sponsor', 0, '" + getHash('test3', 0) + "')"); + db.exec(startOfQuery + "('test3', 7, 22, -3, '1-uuid-5', 'testman', 0, 50, 'sponsor', 0, '" + getHash('test3', 0) + "')"); + db.exec(startOfQuery + "('multiple', 1, 11, 2, '1-uuid-6', 'testman', 0, 50, 'intro', 0, '" + getHash('multiple', 0) + "')"); + db.exec(startOfQuery + "('multiple', 20, 33, 2, '1-uuid-7', 'testman', 0, 50, 'intro', 0, '" + getHash('multiple', 0) + "')"); }); diff --git a/test/cases/oldGetSponsorTime.js b/test/cases/oldGetSponsorTime.js index 569b109..8fe19d2 100644 --- a/test/cases/oldGetSponsorTime.js +++ b/test/cases/oldGetSponsorTime.js @@ -1,6 +1,7 @@ var request = require('request'); var db = require('../../src/databases/databases.js').db; var utils = require('../utils.js'); +var getHash = require('../../src/utils/getHash.js'); /* @@ -19,9 +20,9 @@ var utils = require('../utils.js'); describe('getVideoSponsorTime (Old get method)', () => { before(() => { - let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden) VALUES"; - db.exec(startOfQuery + "('old-testtesttest', 1, 11, 2, 'uuid-0', 'testman', 0, 50, 'sponsor', 0)"); - db.exec(startOfQuery + "('old-testtesttest,test', 1, 11, 2, 'uuid-1', 'testman', 0, 50, 'sponsor', 0)"); + let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES"; + db.exec(startOfQuery + "('old-testtesttest', 1, 11, 2, 'uuid-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('old-testtesttest', 0) + "')"); + db.exec(startOfQuery + "('old-testtesttest,test', 1, 11, 2, 'uuid-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('old-testtesttest,test', 0) + "')"); }); it('Should be able to get a time', (done) => { diff --git a/test/cases/voteOnSponsorTime.js b/test/cases/voteOnSponsorTime.js index db618d2..437bbea 100644 --- a/test/cases/voteOnSponsorTime.js +++ b/test/cases/voteOnSponsorTime.js @@ -5,14 +5,14 @@ var getHash = require('../../src/utils/getHash.js') describe('voteOnSponsorTime', () => { before(() => { - let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden) VALUES"; - db.exec(startOfQuery + "('vote-testtesttest', 1, 11, 2, 'vote-uuid-0', 'testman', 0, 50, 'sponsor', 0)"); - db.exec(startOfQuery + "('vote-testtesttest', 20, 33, 10, 'vote-uuid-2', 'testman', 0, 50, 'intro', 0)"); - db.exec(startOfQuery + "('vote-testtesttest,test', 1, 11, 100, 'vote-uuid-3', 'testman', 0, 50, 'sponsor', 0)"); - db.exec(startOfQuery + "('vote-test3', 1, 11, 2, 'vote-uuid-4', 'testman', 0, 50, 'sponsor', 0)"); - db.exec(startOfQuery + "('vote-test3', 7, 22, -3, 'vote-uuid-5', 'testman', 0, 50, 'intro', 0)"); - db.exec(startOfQuery + "('vote-multiple', 1, 11, 2, 'vote-uuid-6', 'testman', 0, 50, 'intro', 0)"); - db.exec(startOfQuery + "('vote-multiple', 20, 33, 2, 'vote-uuid-7', 'testman', 0, 50, 'intro', 0)"); + let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES"; + db.exec(startOfQuery + "('vote-testtesttest', 1, 11, 2, 'vote-uuid-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('vote-testtesttest', 0) + "')"); + db.exec(startOfQuery + "('vote-testtesttest', 20, 33, 10, 'vote-uuid-2', 'testman', 0, 50, 'intro', 0, '" + getHash('vote-testtesttest', 0) + "')"); + db.exec(startOfQuery + "('vote-testtesttest,test', 1, 11, 100, 'vote-uuid-3', 'testman', 0, 50, 'sponsor', 0, '" + getHash('vote-testtesttest,test', 0) + "')"); + db.exec(startOfQuery + "('vote-test3', 1, 11, 2, 'vote-uuid-4', 'testman', 0, 50, 'sponsor', 0, '" + getHash('vote-test3', 0) + "')"); + db.exec(startOfQuery + "('vote-test3', 7, 22, -3, 'vote-uuid-5', 'testman', 0, 50, 'intro', 0, '" + getHash('vote-test3', 0) + "')"); + db.exec(startOfQuery + "('vote-multiple', 1, 11, 2, 'vote-uuid-6', 'testman', 0, 50, 'intro', 0, '" + getHash('vote-multiple', 0) + "')"); + db.exec(startOfQuery + "('vote-multiple', 20, 33, 2, 'vote-uuid-7', 'testman', 0, 50, 'intro', 0, '" + getHash('vote-multiple', 0) + "')"); db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("VIPUser") + "')"); }); From c2510d302a3934a8c895c03b943403721b6df763 Mon Sep 17 00:00:00 2001 From: Martijn van der Ven Date: Sat, 23 May 2020 21:17:47 +0200 Subject: [PATCH 04/40] Limit overlapping segments to just one through weighted randomness --- src/routes/getSkipSegmentsByHash.js | 90 +++++++++++++++++++++++------ 1 file changed, 72 insertions(+), 18 deletions(-) diff --git a/src/routes/getSkipSegmentsByHash.js b/src/routes/getSkipSegmentsByHash.js index 3672eed..8bb5fcc 100644 --- a/src/routes/getSkipSegmentsByHash.js +++ b/src/routes/getSkipSegmentsByHash.js @@ -12,24 +12,64 @@ const getIP = require('../utils/getIP.js'); * @property {string} UUID Unique identifier for the specific segment */ +/** + * @typedef {Object} Row + * @property {string} videoID + * @property {number} startTime + * @property {number} endTime + * @property {number} votes + * @property {string} UUID + * @property {string} category + * @property {number} shadowHidden + */ + +/** + * Input an array of database records and get only one back, weighed on votes. + * The logic is taken from getWeightedRandomChoice, just simplified input and output to not work on indices only. + * + * @param {Row[]} rows + * @returns {?Row} + */ +function pickWeightedRandomRow(rows) { + if (rows.length === 0) { + return null; + } else if (rows.length === 1) { + return rows[0]; + } + + const sqrtWeightsList = []; + let totalSqrtWeights = 0; + for (const row of rows) { + let sqrtVote = Math.sqrt((row.votes + 3) * 10); + sqrtWeightsList.push(sqrtVote); + totalSqrtWeights += sqrtVote; + } + + const randomNumber = Math.random(); + let currentVoteNumber = 0; + for (let i = 0; i < sqrtWeightsList.length; i++) { + if (randomNumber > currentVoteNumber / totalSqrtWeights && randomNumber < (currentVoteNumber + sqrtWeightsList[i]) / totalSqrtWeights) { + return rows[i]; + } + currentVoteNumber += sqrtWeightsList[i]; + } +} /** * @param {string} prefix Lowercased hexadecimal hash prefix * @param {string} hashedIP Custom hash of the visitor’s IP address - * @returns {Segment[]} + * @returns {Object.} */ function getSkipSegmentsByHash(prefix, hashedIP) { - /** - * @constant - * @type {Segment[]} - * @default - */ - const segments = []; - - const rows = db.prepare('SELECT videoID, startTime, endTime, UUID, category, shadowHidden FROM sponsorTimes WHERE votes >= -1 AND hashedVideoID LIKE ? ORDER BY startTime') + /** @type Row[] */ + const rows = db.prepare('SELECT videoID, startTime, endTime, votes, UUID, category, shadowHidden FROM sponsorTimes WHERE votes >= -1 AND hashedVideoID LIKE ? ORDER BY videoID, startTime') .all(prefix + '%'); - + /** @type {string[]} */ const onlyForCurrentUser = privateDB.prepare('SELECT videoID FROM sponsorTimes WHERE hashedIP = ?').all(hashedIP).map(row => row.videoID); + /** @type {Object.} */ + const rowGroupsPerVideo = {}; + let previousVideoID = null; + let previousEndTime = null; for (const row of rows) { /** @TODO check if this logic does what is expected. */ if (row.shadowHidden === 1 && onlyForCurrentUser.indexOf(row.videoID) === -1) { @@ -37,16 +77,30 @@ function getSkipSegmentsByHash(prefix, hashedIP) { // Do not send shadowHidden segments to them. continue; } - - segments.push({ - videoID: row.videoID, - segment: [row.startTime, row.endTime], - category: row.category, - UUID: row.UUID - }); + // Split up the rows per video and group overlapping segments together. + if (!(row.videoID in rowGroupsPerVideo)) { + rowGroupsPerVideo[row.videoID] = []; + } + if (previousVideoID === row.videoID && row.startTime <= previousEndTime) { + rowGroupsPerVideo[row.videoID][rowGroupsPerVideo[row.videoID].length - 1].push(row); + previousEndTime = Math.max(previousEndTime, row.endTime); + } else { + rowGroupsPerVideo[row.videoID].push([row]); + previousVideoID = row.videoID; + previousEndTime = row.endTime; + } } - return segments; + /** @type {Object.} */ + const output = {}; + for (const videoID in rowGroupsPerVideo) { + const pickedVideosForVideoID = []; + for (const group of rowGroupsPerVideo[videoID]) { + pickedVideosForVideoID.push(pickWeightedRandomRow(group)); + } + output[videoID] = pickedVideosForVideoID.map(row => ({ videoID: row.videoID, segment: [row.startTime, row.endTime], category: row.category, UUID: row.UUID })); + } + return output; } const minimumPrefix = config.minimumPrefix || '3'; From c7fd603933e7d33661c7d3ba7fb0486411b619b4 Mon Sep 17 00:00:00 2001 From: Martijn van der Ven Date: Sat, 23 May 2020 21:25:38 +0200 Subject: [PATCH 05/40] Respond Not Found when a prefix is empty --- src/routes/getSkipSegmentsByHash.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/routes/getSkipSegmentsByHash.js b/src/routes/getSkipSegmentsByHash.js index 8bb5fcc..1762ef3 100644 --- a/src/routes/getSkipSegmentsByHash.js +++ b/src/routes/getSkipSegmentsByHash.js @@ -117,7 +117,9 @@ module.exports = async function (req, res) { getHash(getIP(req) + config.globalSalt) ); - if (segments) { - res.send(segments) + if (Object.keys(segments).length > 0) { + res.send(segments); + } else { + res.sendStatus(404); // No skipable segments within this prefix } } \ No newline at end of file From 3167c24f75bcd9c390baff39c72a47130e2d16f8 Mon Sep 17 00:00:00 2001 From: Martijn van der Ven Date: Sat, 23 May 2020 21:27:21 +0200 Subject: [PATCH 06/40] Put hashedVideoID column before shadowHidden column --- databases/_upgrade_sponsorTimes_2.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/databases/_upgrade_sponsorTimes_2.sql b/databases/_upgrade_sponsorTimes_2.sql index 5472499..1c7270b 100644 --- a/databases/_upgrade_sponsorTimes_2.sql +++ b/databases/_upgrade_sponsorTimes_2.sql @@ -12,8 +12,8 @@ CREATE TABLE "sqlb_temp_table_1" ( "timeSubmitted" INTEGER NOT NULL, "views" INTEGER NOT NULL, "category" TEXT NOT NULL DEFAULT "sponsor", - "shadowHidden" INTEGER NOT NULL, - "hashedVideoID" TEXT NOT NULL + "hashedVideoID" TEXT NOT NULL, + "shadowHidden" INTEGER NOT NULL ); INSERT INTO sqlb_temp_table_1 SELECT *, sha256(videoID) FROM sponsorTimes; From 44ea0c418a6d34490fc84c0145534b048401c12f Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Sat, 22 Aug 2020 02:14:19 +0100 Subject: [PATCH 07/40] fixed db update, started no segments --- databases/_upgrade_sponsorTimes_2.sql | 13 ++ index.js | 3 +- src/app.js | 4 + src/databases/databases.js | 11 +- src/routes/postNoSegments.js | 31 +++++ src/routes/voteOnSponsorTime.js | 2 +- src/utils/isUserVIP.js | 8 ++ src/utils/logger.js | 2 + test.js | 5 +- test/cases/postNoSegments.js | 175 ++++++++++++++++++++++++++ 10 files changed, 247 insertions(+), 7 deletions(-) create mode 100644 databases/_upgrade_sponsorTimes_2.sql create mode 100644 src/routes/postNoSegments.js create mode 100644 src/utils/isUserVIP.js create mode 100644 test/cases/postNoSegments.js diff --git a/databases/_upgrade_sponsorTimes_2.sql b/databases/_upgrade_sponsorTimes_2.sql new file mode 100644 index 0000000..46c0fad --- /dev/null +++ b/databases/_upgrade_sponsorTimes_2.sql @@ -0,0 +1,13 @@ +BEGIN TRANSACTION; + +/* Add incorrectVotes field */ +CREATE TABLE "noSegments" ( + "videoID" TEXT NOT NULL, + "userID" TEXT NOT NULL, + "category" TEXT NOT NULL +); + +/* Add version to config */ +UPDATE config SET value = 2 WHERE key = 'version'; + +COMMIT; \ No newline at end of file diff --git a/index.js b/index.js index 508c089..a877bab 100644 --- a/index.js +++ b/index.js @@ -1,5 +1,6 @@ var config = require('./src/config.js'); var createServer = require('./src/app.js'); +const logger = require('./src/utils/logger.js'); var server = createServer(() => { - console.log("Server started on port " + config.port + "."); + logger.info("Server started on port " + config.port + "."); }); diff --git a/src/app.js b/src/app.js index 1ce1674..ef6c554 100644 --- a/src/app.js +++ b/src/app.js @@ -21,6 +21,7 @@ var getViewsForUser = require('./routes/getViewsForUser.js'); var getTopUsers = require('./routes/getTopUsers.js'); var getTotalStats = require('./routes/getTotalStats.js'); var getDaysSavedFormatted = require('./routes/getDaysSavedFormatted.js'); +var postNoSegments = require('./routes/postNoSegments.js'); // Old Routes var oldGetVideoSponsorTimes = require('./routes/oldGetVideoSponsorTimes.js'); @@ -86,6 +87,9 @@ app.get('/api/getTotalStats', getTotalStats); //send out a formatted time saved total app.get('/api/getDaysSavedFormatted', getDaysSavedFormatted); +//submit video containing no segments +app.post('/api/postNoSegments', postNoSegments); + app.get('/database.db', function (req, res) { res.sendFile("./databases/sponsorTimes.db", { root: "./" }); }); diff --git a/src/databases/databases.js b/src/databases/databases.js index 1a1b448..a3aeb59 100644 --- a/src/databases/databases.js +++ b/src/databases/databases.js @@ -3,7 +3,8 @@ var Sqlite3 = require('better-sqlite3'); var fs = require('fs'); var path = require('path'); var Sqlite = require('./Sqlite.js') -var Mysql = require('./Mysql.js') +var Mysql = require('./Mysql.js'); +const logger = require('../utils/logger.js'); let options = { readonly: config.readOnly, @@ -60,12 +61,16 @@ if (config.mysql) { let versionCodeInfo = db.prepare("SELECT value FROM config WHERE key = ?").get("version"); let versionCode = versionCodeInfo ? versionCodeInfo.value : 0; - let path = config.schemaFolder + "/_upgrade_" + prefix + "_" + (versionCode + 1) + ".sql"; + let path = config.schemaFolder + "/_upgrade_" + prefix + "_" + (parseInt(versionCode) + 1) + ".sql"; + logger.debug('db update: trying ' + path); while (fs.existsSync(path)) { + logger.debug('db update: updating ' + path); db.exec(fs.readFileSync(path).toString()); versionCode = db.prepare("SELECT value FROM config WHERE key = ?").get("version").value; - path = config.schemaFolder + "/_upgrade_" + prefix + "_" + (versionCode + 1) + ".sql"; + path = config.schemaFolder + "/_upgrade_" + prefix + "_" + (parseInt(versionCode) + 1) + ".sql"; + logger.debug('db update: trying ' + path); } + logger.debug('db update: no file ' + path); } } \ No newline at end of file diff --git a/src/routes/postNoSegments.js b/src/routes/postNoSegments.js new file mode 100644 index 0000000..58074b8 --- /dev/null +++ b/src/routes/postNoSegments.js @@ -0,0 +1,31 @@ +const getHash = require('../utils/getHash.js'); +const isUserVIP = require('../utils/isUserVIP.js'); + +module.exports = (req, res) => { + // Collect user input data + let videoID = req.body.videoID; + let userID = req.body.userID; + let categorys = req.body.categorys; + + // Check input data is valid + if (!videoID + || !userID + || !categorys + || !Array.isArray(categorys) + || categorys.length === 0 + ) { + res.status(400).json({}); + return; + } + + // Check if user is VIP + userID = getHash(userID); + let userIsVIP = isUserVIP(userID); + + if (!userIsVIP) { + res.status(403).json({}); + return; + } + + res.status(200).json({status: 200}); +}; \ No newline at end of file diff --git a/src/routes/voteOnSponsorTime.js b/src/routes/voteOnSponsorTime.js index 0e9fdf8..9ce6afa 100644 --- a/src/routes/voteOnSponsorTime.js +++ b/src/routes/voteOnSponsorTime.js @@ -4,7 +4,7 @@ var config = require('../config.js'); var getHash = require('../utils/getHash.js'); var getIP = require('../utils/getIP.js'); var getFormattedTime = require('../utils/getFormattedTime.js'); -var isUserTrustworthy = require('../utils/isUserTrustworthy.js') +var isUserTrustworthy = require('../utils/isUserTrustworthy.js'); var databases = require('../databases/databases.js'); var db = databases.db; diff --git a/src/utils/isUserVIP.js b/src/utils/isUserVIP.js new file mode 100644 index 0000000..eeb25dc --- /dev/null +++ b/src/utils/isUserVIP.js @@ -0,0 +1,8 @@ +const databases = require('../databases/databases.js'); +const db = databases.db; + +module.exports = (userID) => { + return db.prepare('get', "SELECT count(*) as userCount FROM vipUsers WHERE userID = ?", [userID]).userCount > 0; +} + + diff --git a/src/utils/logger.js b/src/utils/logger.js index 70fe13e..351e45c 100644 --- a/src/utils/logger.js +++ b/src/utils/logger.js @@ -17,6 +17,8 @@ const settings = { if (config.mode === 'development') { settings.INFO = true; settings.DEBUG = true; +} else if (config.mode === 'test') { + settings.DEBUG = true; } function log(level, string) { diff --git a/test.js b/test.js index 45ace90..9810786 100644 --- a/test.js +++ b/test.js @@ -9,6 +9,7 @@ if (fs.existsSync(config.privateDB)) fs.unlinkSync(config.privateDB); var createServer = require('./src/app.js'); var createMockServer = require('./test/mocks.js'); +const logger = require('./src/utils/logger.js'); // Instantiate a Mocha instance. var mocha = new Mocha(); @@ -27,9 +28,9 @@ fs.readdirSync(testDir).filter(function(file) { }); var mockServer = createMockServer(() => { - console.log("Started mock HTTP Server"); + logger.info("Started mock HTTP Server"); var server = createServer(() => { - console.log("Started main HTTP server"); + logger.info("Started main HTTP server"); // Run the tests. mocha.run(function(failures) { mockServer.close(); diff --git a/test/cases/postNoSegments.js b/test/cases/postNoSegments.js new file mode 100644 index 0000000..5ab2b3d --- /dev/null +++ b/test/cases/postNoSegments.js @@ -0,0 +1,175 @@ +var request = require('request'); + +var utils = require('../utils.js'); +const getHash = require('../../src/utils/getHash.js'); + +var databases = require('../../src/databases/databases.js'); +var db = databases.db; + +describe('postNoSegments', () => { + before(() => { + db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("VIPUser-noSegments") + "')"); + }); + + it('should update the database version when starting the application', (done) => { + let version = db.prepare('get', 'SELECT key, value FROM config where key = ?', ['version']).value; + if (version > 1) done(); + else done('Version isn\'t greater that 1. Version is ' + version); + }); + + it('Should be able to submit no segments', (done) => { + let json = { + videoID: 'noSegmentsTestVideoID', + userID: 'VIPUser-noSegments', + categorys: [ + 'sponsor' + ] + }; + + request.post(utils.getbaseURL() + + "/api/postNoSegments", {json}, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + //let row = db.prepare('get', "SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?", ["noSegmentsTestVideoID"]); + //if (row.startTime === 2 && row.endTime === 10 && row.category === "sponsor") { + done() + //} else { + // done("Submitted times were not saved. Actual submission: " + JSON.stringify(row)); + //} + } else { + console.log(body); + done("Status code was " + res.statusCode); + } + }); + }); + + it('Should return 400 for missing params', (done) => { + request.post(utils.getbaseURL() + + "/api/postNoSegments", {json: {}}, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 400) { + done() + } else { + done("Status code was " + res.statusCode); + } + }); + }); + + it('Should return 400 for no categorys', (done) => { + let json = { + videoID: 'test', + userID: 'test', + categorys: [] + }; + + request.post(utils.getbaseURL() + + "/api/postNoSegments", {json}, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 400) { + done() + } else { + done("Status code was " + res.statusCode); + } + }); + }); + + it('Should return 400 for no userID', (done) => { + let json = { + videoID: 'test', + userID: null, + categorys: ['sponsor'] + }; + + request.post(utils.getbaseURL() + + "/api/postNoSegments", {json}, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 400) { + done() + } else { + done("Status code was " + res.statusCode); + } + }); + }); + + it('Should return 400 for no videoID', (done) => { + let json = { + videoID: null, + userID: 'test', + categorys: ['sponsor'] + }; + + request.post(utils.getbaseURL() + + "/api/postNoSegments", {json}, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 400) { + done() + } else { + done("Status code was " + res.statusCode); + } + }); + }); + + it('Should return 400 object categorys)', (done) => { + let json = { + videoID: 'test', + userID: 'test', + categorys: {} + }; + + request.post(utils.getbaseURL() + + "/api/postNoSegments", {json}, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 400) { + done() + } else { + done("Status code was " + res.statusCode); + } + }); + }); + + it('Should return 400 bad format categorys', (done) => { + let json = { + videoID: 'test', + userID: 'test', + categorys: 'sponsor' + }; + + request.post(utils.getbaseURL() + + "/api/postNoSegments", {json}, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 400) { + done() + } else { + done("Status code was " + res.statusCode); + } + }); + }); + + it('Should return 403 if user is not VIP', (done) => { + let json = { + videoID: 'test', + userID: 'test', + categorys: [ + 'sponsor' + ] + }; + + request.post(utils.getbaseURL() + + "/api/postNoSegments", {json}, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 403) { + done() + } else { + done("Status code was " + res.statusCode); + } + }); + }); +}); \ No newline at end of file From f53c541538c613254ba620fc7dc46ee4d713bbf8 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Sat, 22 Aug 2020 03:05:51 +0100 Subject: [PATCH 08/40] added ability for vips to submit segments not in a video --- src/routes/getSkipSegments.js | 1 - src/routes/postNoSegments.js | 33 ++++++++++++++++++++++++++++++++- test/cases/postNoSegments.js | 33 +++++++++++++++++++++++---------- 3 files changed, 55 insertions(+), 12 deletions(-) diff --git a/src/routes/getSkipSegments.js b/src/routes/getSkipSegments.js index ba23262..7b68d3f 100644 --- a/src/routes/getSkipSegments.js +++ b/src/routes/getSkipSegments.js @@ -1,4 +1,3 @@ -var fs = require('fs'); var config = require('../config.js'); var databases = require('../databases/databases.js'); diff --git a/src/routes/postNoSegments.js b/src/routes/postNoSegments.js index 58074b8..f2e681b 100644 --- a/src/routes/postNoSegments.js +++ b/src/routes/postNoSegments.js @@ -1,5 +1,7 @@ +const db = require('../databases/databases.js').db; const getHash = require('../utils/getHash.js'); const isUserVIP = require('../utils/isUserVIP.js'); +const logger = require('../utils/logger.js'); module.exports = (req, res) => { // Collect user input data @@ -27,5 +29,34 @@ module.exports = (req, res) => { return; } - res.status(200).json({status: 200}); + // Get existing no segment markers + let noSegmentList = db.prepare('all', 'SELECT category from noSegments where videoID = ?', [videoID]); + if (!noSegmentList || noSegmentList.length === 0) { + noSegmentList = []; + } else { + noSegmentList = noSegmentList.map((obj) => { + return obj.category; + }); + } + + // get user categorys not already submitted + let categorysToMark = categorys.filter((category) => { + return noSegmentList.indexOf(category) === -1; + }); + + // remove any duplicates + categorysToMark = categorysToMark.filter((category, index) => { + return categorysToMark.indexOf(category) === index; + }); + + // create database entry + categorysToMark.forEach((category) => { + db.prepare('run', "INSERT INTO noSegments (videoID, userID, category) VALUES(?, ?, ?)", [videoID, userID, category]); + //ogger.debug('submitting ' + category); + }); + + res.status(200).json({ + status: 200, + submitted: categorysToMark + }); }; \ No newline at end of file diff --git a/test/cases/postNoSegments.js b/test/cases/postNoSegments.js index 5ab2b3d..7d88e7d 100644 --- a/test/cases/postNoSegments.js +++ b/test/cases/postNoSegments.js @@ -9,6 +9,9 @@ var db = databases.db; describe('postNoSegments', () => { before(() => { db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("VIPUser-noSegments") + "')"); + + db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'no-segments-video-id', 'sponsor')"); + db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'no-segments-video-id', 'intro')"); }); it('should update the database version when starting the application', (done) => { @@ -17,12 +20,23 @@ describe('postNoSegments', () => { else done('Version isn\'t greater that 1. Version is ' + version); }); - it('Should be able to submit no segments', (done) => { + it('Should be able to submit categorys not in video', (done) => { let json = { - videoID: 'noSegmentsTestVideoID', + videoID: 'no-segments-video-id', userID: 'VIPUser-noSegments', categorys: [ - 'sponsor' + 'outro', + 'shilling', + 'shilling', + 'intro' + ] + }; + + let expected = { + status: 200, + submitted: [ + 'outro', + 'shilling' ] }; @@ -31,12 +45,11 @@ describe('postNoSegments', () => { (err, res, body) => { if (err) done(err); else if (res.statusCode === 200) { - //let row = db.prepare('get', "SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?", ["noSegmentsTestVideoID"]); - //if (row.startTime === 2 && row.endTime === 10 && row.category === "sponsor") { - done() - //} else { - // done("Submitted times were not saved. Actual submission: " + JSON.stringify(row)); - //} + if (JSON.stringify(body) === JSON.stringify(expected)) { + done(); + } else { + done("Incorrect response: expected " + JSON.stringify(expected) + " got " + JSON.stringify(body)); + } } else { console.log(body); done("Status code was " + res.statusCode); @@ -166,7 +179,7 @@ describe('postNoSegments', () => { (err, res, body) => { if (err) done(err); else if (res.statusCode === 403) { - done() + done(); } else { done("Status code was " + res.statusCode); } From db3de8ce6f140cbbd26ac97eb87d9e5b3ea602bd Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Sat, 22 Aug 2020 03:21:10 +0100 Subject: [PATCH 09/40] changed copy/pasted comment --- databases/_upgrade_sponsorTimes_2.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/databases/_upgrade_sponsorTimes_2.sql b/databases/_upgrade_sponsorTimes_2.sql index 46c0fad..f6816bc 100644 --- a/databases/_upgrade_sponsorTimes_2.sql +++ b/databases/_upgrade_sponsorTimes_2.sql @@ -1,6 +1,6 @@ BEGIN TRANSACTION; -/* Add incorrectVotes field */ +/* Add new table: noSegments */ CREATE TABLE "noSegments" ( "videoID" TEXT NOT NULL, "userID" TEXT NOT NULL, From 6bde59c14ad0e6ede742c5928aefea5e0f6a0464 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Sat, 22 Aug 2020 03:29:16 +0100 Subject: [PATCH 10/40] added extra sql test --- test/cases/postNoSegments.js | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/test/cases/postNoSegments.js b/test/cases/postNoSegments.js index 7d88e7d..7f89b77 100644 --- a/test/cases/postNoSegments.js +++ b/test/cases/postNoSegments.js @@ -12,6 +12,9 @@ describe('postNoSegments', () => { db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'no-segments-video-id', 'sponsor')"); db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'no-segments-video-id', 'intro')"); + + db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'no-segments-video-id-1', 'sponsor')"); + db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'no-segments-video-id-1', 'intro')"); }); it('should update the database version when starting the application', (done) => { @@ -20,7 +23,7 @@ describe('postNoSegments', () => { else done('Version isn\'t greater that 1. Version is ' + version); }); - it('Should be able to submit categorys not in video', (done) => { + it('Should be able to submit categorys not in video (http response)', (done) => { let json = { videoID: 'no-segments-video-id', userID: 'VIPUser-noSegments', @@ -57,6 +60,37 @@ describe('postNoSegments', () => { }); }); + it('Should be able to submit categorys not in video (sql check)', (done) => { + let json = { + videoID: 'no-segments-video-id-1', + userID: 'VIPUser-noSegments', + categorys: [ + 'outro', + 'shilling', + 'shilling', + 'intro' + ] + }; + + request.post(utils.getbaseURL() + + "/api/postNoSegments", {json}, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + let result = db.prepare('all', 'SELECT * FROM noSegments WHERE videoID = ?', ['no-segments-video-id-1']); + if (result.length !== 4) { + console.log(result); + done("Expected 4 entrys in db, got " + result.length); + } else { + done(); + } + } else { + console.log(body); + done("Status code was " + res.statusCode); + } + }); + }); + it('Should return 400 for missing params', (done) => { request.post(utils.getbaseURL() + "/api/postNoSegments", {json: {}}, From c946d2309ed8e687fc977286a91a2cbdcc117cbe Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Sat, 22 Aug 2020 03:58:27 +0100 Subject: [PATCH 11/40] add is VIP endpoint --- src/app.js | 5 ++++ src/routes/getIsUserVIP.js | 31 +++++++++++++++++++++ test/cases/getIsUserVIP.js | 57 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+) create mode 100644 src/routes/getIsUserVIP.js create mode 100644 test/cases/getIsUserVIP.js diff --git a/src/app.js b/src/app.js index ef6c554..8617566 100644 --- a/src/app.js +++ b/src/app.js @@ -22,6 +22,7 @@ var getTopUsers = require('./routes/getTopUsers.js'); var getTotalStats = require('./routes/getTotalStats.js'); var getDaysSavedFormatted = require('./routes/getDaysSavedFormatted.js'); var postNoSegments = require('./routes/postNoSegments.js'); +var getIsUserVIP = require('./routes/getIsUserVIP.js'); // Old Routes var oldGetVideoSponsorTimes = require('./routes/oldGetVideoSponsorTimes.js'); @@ -90,6 +91,10 @@ app.get('/api/getDaysSavedFormatted', getDaysSavedFormatted); //submit video containing no segments app.post('/api/postNoSegments', postNoSegments); +//get if user is a vip +app.get('/api/getIsUserVIP', getIsUserVIP); + + app.get('/database.db', function (req, res) { res.sendFile("./databases/sponsorTimes.db", { root: "./" }); }); diff --git a/src/routes/getIsUserVIP.js b/src/routes/getIsUserVIP.js new file mode 100644 index 0000000..d38ef9d --- /dev/null +++ b/src/routes/getIsUserVIP.js @@ -0,0 +1,31 @@ +var db = require('../databases/databases.js').db; + +var getHash = require('../utils/getHash.js'); +const logger = require('../utils/logger.js'); +const isUserVIP = require('../utils/isUserVIP.js'); + +module.exports = function getUsername (req, res) { + let userID = req.query.userID; + + if (userID == undefined) { + //invalid request + res.sendStatus(400); + return; + } + + //hash the userID + userID = getHash(userID); + + try { + let vipState = isUserVIP(userID); + res.status(200).json({ + hashedUserID: userID, + vip: vipState + }); + } catch (err) { + logger.error(err); + res.sendStatus(500); + + return; + } +} \ No newline at end of file diff --git a/test/cases/getIsUserVIP.js b/test/cases/getIsUserVIP.js new file mode 100644 index 0000000..0b25e7d --- /dev/null +++ b/test/cases/getIsUserVIP.js @@ -0,0 +1,57 @@ +var request = require('request'); +var utils = require('../utils.js'); +var db = require('../../src/databases/databases.js').db; +var getHash = require('../../src/utils/getHash.js'); + +describe('getSavedTimeForUser', () => { + before(() => { + db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("supertestman") + "')"); + }); + + it('Should be able to get a 200', (done) => { + request.get(utils.getbaseURL() + + "/api/getIsUserVIP?userID=supertestman", null, + (err, res, body) => { + if (err) done("couldn't call endpoint"); + else if (res.statusCode !== 200) done("non 200: " + res.statusCode); + else done(); // pass + }); + }); + + + it('Should get a 400 if no userID', (done) => { + request.get(utils.getbaseURL() + + "/api/getIsUserVIP", null, + (err, res, body) => { + if (err) done("couldn't call endpoint"); + else if (res.statusCode !== 400) done("non 400: " + res.statusCode); + else done(); // pass + }); + }); + + it('Should say a VIP is a VIP', (done) => { + request.get(utils.getbaseURL() + + "/api/getIsUserVIP?userID=supertestman", null, + (err, res, body) => { + if (err) done("couldn't call endpoint"); + else if (res.statusCode !== 200) done("non 200: " + res.statusCode); + else { + if (JSON.parse(body).vip === true) done(); // pass + else done("Result was non-vip when should have been vip"); + } + }); + }); + + it('Should say a normal user is not a VIP', (done) => { + request.get(utils.getbaseURL() + + "/api/getIsUserVIP?userID=regulartestman", null, + (err, res, body) => { + if (err) done("couldn't call endpoint"); + else if (res.statusCode !== 200) done("non 200: " + res.statusCode); + else { + if (JSON.parse(body).vip === false) done(); // pass + else done("Result was vip when should have been non-vip"); + } + }); + }); +}); \ No newline at end of file From a971072cb8be995133d4a07c87d31a8391d8e38b Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Sat, 22 Aug 2020 04:23:28 +0100 Subject: [PATCH 12/40] reject submissions that have meen marked as invalid by a vip --- src/routes/postSkipSegments.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 1dbaf21..40a3e25 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -157,6 +157,8 @@ module.exports = async function postSkipSegments(req, res) { //hash the ip 5000 times so no one can get it from the database let hashedIP = getHash(getIP(req) + config.globalSalt); + let noSegmentList = db.prepare('all', 'SELECT category from noSegments where videoID = ?', [videoID]); + // Check if all submissions are correct for (let i = 0; i < segments.length; i++) { if (segments[i] === undefined || segments[i].segment === undefined || segments[i].category === undefined) { @@ -165,6 +167,14 @@ module.exports = async function postSkipSegments(req, res) { return; } + // Reject segemnt if it's in the no segments list + if (noSegmentList.indexOf(segments[i].category) !== -1) { + // TODO: Do something about the fradulent submission + res.sendStatus(403); + return; + } + + let startTime = parseFloat(segments[i].segment[0]); let endTime = parseFloat(segments[i].segment[1]); From a5cf59d85467715bb61b108b63ed8f2ac7440dd0 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Sat, 22 Aug 2020 20:39:13 +0100 Subject: [PATCH 13/40] improved error handling --- src/routes/postNoSegments.js | 26 +++++++++++++++++++++----- test/cases/postNoSegments.js | 4 ++++ 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/routes/postNoSegments.js b/src/routes/postNoSegments.js index f2e681b..6494095 100644 --- a/src/routes/postNoSegments.js +++ b/src/routes/postNoSegments.js @@ -16,7 +16,10 @@ module.exports = (req, res) => { || !Array.isArray(categorys) || categorys.length === 0 ) { - res.status(400).json({}); + res.status(400).json({ + status: 400, + message: 'Bad Format' + }); return; } @@ -25,7 +28,10 @@ module.exports = (req, res) => { let userIsVIP = isUserVIP(userID); if (!userIsVIP) { - res.status(403).json({}); + res.status(403).json({ + status: 403, + message: 'Must be a VIP to mark videos.' + }); return; } @@ -39,8 +45,10 @@ module.exports = (req, res) => { }); } - // get user categorys not already submitted + // get user categorys not already submitted that match accepted format let categorysToMark = categorys.filter((category) => { + return !!category.match(/^[a-zA-Z]+$/); + }).filter((category) => { return noSegmentList.indexOf(category) === -1; }); @@ -51,8 +59,16 @@ module.exports = (req, res) => { // create database entry categorysToMark.forEach((category) => { - db.prepare('run', "INSERT INTO noSegments (videoID, userID, category) VALUES(?, ?, ?)", [videoID, userID, category]); - //ogger.debug('submitting ' + category); + try { + db.prepare('run', "INSERT INTO noSegments (videoID, userID, category) VALUES(?, ?, ?)", [videoID, userID, category]); + } catch (err) { + logger.error("Error submitting 'noSegment' marker for category '" + category + "' for video '" + videoID + "'"); + logger.error(err); + res.status(500).json({ + status: 500, + message: "Internal Server Error: Could not write marker to the database." + }); + } }); res.status(200).json({ diff --git a/test/cases/postNoSegments.js b/test/cases/postNoSegments.js index 7f89b77..8489f61 100644 --- a/test/cases/postNoSegments.js +++ b/test/cases/postNoSegments.js @@ -31,6 +31,8 @@ describe('postNoSegments', () => { 'outro', 'shilling', 'shilling', + 'shil ling', + '', 'intro' ] }; @@ -68,6 +70,8 @@ describe('postNoSegments', () => { 'outro', 'shilling', 'shilling', + 'shil ling', + '', 'intro' ] }; From 5f23fdd590e69685c26943286900fb6d4148478e Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Sun, 30 Aug 2020 21:47:02 +0100 Subject: [PATCH 14/40] Added tests and finished no-segments interface. --- src/routes/postSkipSegments.js | 9 +- ...{postNoSegments.js => noSegmentRecords.js} | 103 +++++++++++++++++- 2 files changed, 107 insertions(+), 5 deletions(-) rename test/cases/{postNoSegments.js => noSegmentRecords.js} (67%) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 15ab309..08380fc 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -187,8 +187,7 @@ module.exports = async function postSkipSegments(req, res) { //hash the ip 5000 times so no one can get it from the database let hashedIP = getHash(getIP(req) + config.globalSalt); - let noSegmentList = db.prepare('all', 'SELECT category from noSegments where videoID = ?', [videoID]); - + let noSegmentList = db.prepare('all', 'SELECT category from noSegments where videoID = ?', [videoID]).map((list) => { return list.category }); // Check if all submissions are correct for (let i = 0; i < segments.length; i++) { if (segments[i] === undefined || segments[i].segment === undefined || segments[i].category === undefined) { @@ -200,7 +199,11 @@ module.exports = async function postSkipSegments(req, res) { // Reject segemnt if it's in the no segments list if (noSegmentList.indexOf(segments[i].category) !== -1) { // TODO: Do something about the fradulent submission - res.sendStatus(403); + logger.warn("Caught a no-segment submission. userID: '" + userID + "', videoID: '" + videoID + "', category: '" + segments[i].category + "'"); + res.status(403).send( + "Request rejected by auto moderator: This video has been reported as not containing any segments with the category '" + + segments[i].category + "'. If you believe this is incorrect, contact someone on Discord." + ); return; } diff --git a/test/cases/postNoSegments.js b/test/cases/noSegmentRecords.js similarity index 67% rename from test/cases/postNoSegments.js rename to test/cases/noSegmentRecords.js index 8489f61..d902478 100644 --- a/test/cases/postNoSegments.js +++ b/test/cases/noSegmentRecords.js @@ -4,9 +4,10 @@ var utils = require('../utils.js'); const getHash = require('../../src/utils/getHash.js'); var databases = require('../../src/databases/databases.js'); +const logger = require('../../src/utils/logger.js'); var db = databases.db; -describe('postNoSegments', () => { +describe('noSegmentRecords', () => { before(() => { db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("VIPUser-noSegments") + "')"); @@ -15,9 +16,10 @@ describe('postNoSegments', () => { db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'no-segments-video-id-1', 'sponsor')"); db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'no-segments-video-id-1', 'intro')"); + db.exec("INSERT INTO noSegments (userID, videoID, category) VALUES ('" + getHash("VIPUser-noSegments") + "', 'noSubmitVideo', 'sponsor')"); }); - it('should update the database version when starting the application', (done) => { + it('Should update the database version when starting the application', (done) => { let version = db.prepare('get', 'SELECT key, value FROM config where key = ?', ['version']).value; if (version > 1) done(); else done('Version isn\'t greater that 1. Version is ' + version); @@ -223,4 +225,101 @@ describe('postNoSegments', () => { } }); }); + + /* + * Submission tests in this file do not check database records, only status codes. + * To test the submission code properly see ./test/cases/postSkipSegments.js + */ + + it('Should not be able to submit a segment to a video with a no-segment record (single submission)', (done) => { + request.post(utils.getbaseURL() + + "/api/postVideoSponsorTimes", { + json: { + userID: "testman42", + videoID: "noSubmitVideo", + segments: [{ + segment: [20, 40], + category: "sponsor" + }] + } + }, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 403) { + done() + } else { + done("Status code was " + res.statusCode); + } + }); + }); + + it('Should not be able to submit segments to a video where any of the submissions with a no-segment record', (done) => { + request.post(utils.getbaseURL() + + "/api/postVideoSponsorTimes", { + json: { + userID: "testman42", + videoID: "noSubmitVideo", + segments: [{ + segment: [20, 40], + category: "sponsor" + },{ + segment: [50, 60], + category: "intro" + }] + } + }, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 403) { + done() + } else { + done("Status code was " + res.statusCode); + } + }); + }); + + + it('Should be able to submit a segment to a video with a different no-segment record', (done) => { + request.post(utils.getbaseURL() + + "/api/postVideoSponsorTimes", { + json: { + userID: "testman42", + videoID: "noSubmitVideo", + segments: [{ + segment: [20, 40], + category: "intro" + }] + } + }, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + done() + } else { + done("Status code was " + res.statusCode); + } + }); + }); + + it('Should be able to submit a segment to a video with no no-segment records', (done) => { + request.post(utils.getbaseURL() + + "/api/postVideoSponsorTimes", { + json: { + userID: "testman42", + videoID: "normalVideo", + segments: [{ + segment: [20, 40], + category: "intro" + }] + } + }, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + done() + } else { + done("Status code was " + res.statusCode); + } + }); + }); }); \ No newline at end of file From 1a06502806fe2e09a5c19abd8b20bd7fd90900fc Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Mon, 31 Aug 2020 00:45:06 +0100 Subject: [PATCH 15/40] added get segments by hash prefix --- src/routes/getSkipSegments.js | 118 +++++++++++----------- src/routes/getSkipSegmentsByHash.js | 145 ++++++---------------------- src/routes/voteOnSponsorTime.js | 9 +- src/utils/hashPrefixTester.js | 11 +++ test/cases/getSegmentsByHash.js | 85 ++++++++++++++++ test/cases/oldGetSponsorTime.js | 4 +- test/cases/voteOnSponsorTime.js | 32 +++--- 7 files changed, 206 insertions(+), 198 deletions(-) create mode 100644 src/utils/hashPrefixTester.js create mode 100644 test/cases/getSegmentsByHash.js diff --git a/src/routes/getSkipSegments.js b/src/routes/getSkipSegments.js index 81aae1c..db6ce76 100644 --- a/src/routes/getSkipSegments.js +++ b/src/routes/getSkipSegments.js @@ -8,6 +8,62 @@ var logger = require('../utils/logger.js'); var getHash = require('../utils/getHash.js'); var getIP = require('../utils/getIP.js'); +function cleanGetSegments(videoID, categories) { + let userHashedIP, shadowHiddenSegments; + + let segments = []; + + try { + for (const category of categories) { + const categorySegments = db + .prepare( + 'all', + 'SELECT startTime, endTime, votes, UUID, shadowHidden FROM sponsorTimes WHERE videoID = ? and category = ? ORDER BY startTime', + [videoID, category] + ) + .filter(segment => { + if (segment.votes < -1) { + return false; //too untrustworthy, just ignore it + } + + //check if shadowHidden + //this means it is hidden to everyone but the original ip that submitted it + if (segment.shadowHidden != 1) { + return true; + } + + if (shadowHiddenSegments === undefined) { + shadowHiddenSegments = privateDB.prepare('all', 'SELECT hashedIP FROM sponsorTimes WHERE videoID = ?', [videoID]); + } + + //if this isn't their ip, don't send it to them + return shadowHiddenSegments.some(shadowHiddenSegment => { + if (userHashedIP === undefined) { + //hash the IP only if it's strictly necessary + userHashedIP = getHash(getIP(req) + config.globalSalt); + } + return shadowHiddenSegment.hashedIP === userHashedIP; + }); + }); + + chooseSegments(categorySegments).forEach(chosenSegment => { + segments.push({ + category, + segment: [chosenSegment.startTime, chosenSegment.endTime], + UUID: chosenSegment.UUID, + }); + }); + } + + return segments; + } catch (err) { + if (err) { + logger.error('j 2 Query failed'); + return undefined; + } + } +} + //gets a weighted random choice from the choices array based on their `votes` property. //amountOfChoices specifies the maximum amount of choices to return, 1 or more. //choices are unique @@ -104,58 +160,11 @@ function handleGetSegments(req, res) { ? [req.query.category] : ['sponsor']; - /** - * @type {Array<{ - * segment: number[], - * category: string, - * UUID: string - * }> - * } - */ - const segments = []; + let segments = cleanGetSegments(videoID, categories); - let userHashedIP, shadowHiddenSegments; - - try { - for (const category of categories) { - const categorySegments = db - .prepare( - 'all', - 'SELECT startTime, endTime, votes, UUID, shadowHidden FROM sponsorTimes WHERE videoID = ? and category = ? ORDER BY startTime', - [videoID, category] - ) - .filter(segment => { - if (segment.votes < -1) { - return false; //too untrustworthy, just ignore it - } - - //check if shadowHidden - //this means it is hidden to everyone but the original ip that submitted it - if (segment.shadowHidden != 1) { - return true; - } - - if (shadowHiddenSegments === undefined) { - shadowHiddenSegments = privateDB.prepare('all', 'SELECT hashedIP FROM sponsorTimes WHERE videoID = ?', [videoID]); - } - - //if this isn't their ip, don't send it to them - return shadowHiddenSegments.some(shadowHiddenSegment => { - if (userHashedIP === undefined) { - //hash the IP only if it's strictly necessary - userHashedIP = getHash(getIP(req) + config.globalSalt); - } - return shadowHiddenSegment.hashedIP === userHashedIP; - }); - }); - - chooseSegments(categorySegments).forEach(chosenSegment => { - segments.push({ - category, - segment: [chosenSegment.startTime, chosenSegment.endTime], - UUID: chosenSegment.UUID, - }); - }); + if (segments === undefined) { + res.sendStatus(500); + return false; } if (segments.length == 0) { @@ -164,16 +173,11 @@ function handleGetSegments(req, res) { } return segments; - } catch (error) { - logger.error(error); - res.sendStatus(500); - - return false; - } } module.exports = { handleGetSegments, + cleanGetSegments, endpoint: function (req, res) { let segments = handleGetSegments(req, res); diff --git a/src/routes/getSkipSegmentsByHash.js b/src/routes/getSkipSegmentsByHash.js index 1762ef3..eec0e0e 100644 --- a/src/routes/getSkipSegmentsByHash.js +++ b/src/routes/getSkipSegmentsByHash.js @@ -1,125 +1,36 @@ -const config = require('../config.js'); -const { db, privateDB } = require('../databases/databases.js'); +const hashPrefixTester = require('../utils/hashPrefixTester.js'); +const getSegments = require('./getSkipSegments.js').cleanGetSegments; -const getHash = require('../utils/getHash.js'); -const getIP = require('../utils/getIP.js'); - -/** - * @typedef {Object} Segment - * @property {string} videoID YouTube video ID the segment is meant for - * @property {number[]} segment Tuple of start and end times in seconds - * @property {string} category Category of content to skip - * @property {string} UUID Unique identifier for the specific segment - */ - -/** - * @typedef {Object} Row - * @property {string} videoID - * @property {number} startTime - * @property {number} endTime - * @property {number} votes - * @property {string} UUID - * @property {string} category - * @property {number} shadowHidden - */ - -/** - * Input an array of database records and get only one back, weighed on votes. - * The logic is taken from getWeightedRandomChoice, just simplified input and output to not work on indices only. - * - * @param {Row[]} rows - * @returns {?Row} - */ -function pickWeightedRandomRow(rows) { - if (rows.length === 0) { - return null; - } else if (rows.length === 1) { - return rows[0]; - } - - const sqrtWeightsList = []; - let totalSqrtWeights = 0; - for (const row of rows) { - let sqrtVote = Math.sqrt((row.votes + 3) * 10); - sqrtWeightsList.push(sqrtVote); - totalSqrtWeights += sqrtVote; - } - - const randomNumber = Math.random(); - let currentVoteNumber = 0; - for (let i = 0; i < sqrtWeightsList.length; i++) { - if (randomNumber > currentVoteNumber / totalSqrtWeights && randomNumber < (currentVoteNumber + sqrtWeightsList[i]) / totalSqrtWeights) { - return rows[i]; - } - currentVoteNumber += sqrtWeightsList[i]; - } -} -/** - * @param {string} prefix Lowercased hexadecimal hash prefix - * @param {string} hashedIP Custom hash of the visitor’s IP address - * @returns {Object.} - */ -function getSkipSegmentsByHash(prefix, hashedIP) { - /** @type Row[] */ - const rows = db.prepare('SELECT videoID, startTime, endTime, votes, UUID, category, shadowHidden FROM sponsorTimes WHERE votes >= -1 AND hashedVideoID LIKE ? ORDER BY videoID, startTime') - .all(prefix + '%'); - /** @type {string[]} */ - const onlyForCurrentUser = privateDB.prepare('SELECT videoID FROM sponsorTimes WHERE hashedIP = ?').all(hashedIP).map(row => row.videoID); - /** @type {Object.} */ - const rowGroupsPerVideo = {}; - - let previousVideoID = null; - let previousEndTime = null; - for (const row of rows) { - /** @TODO check if this logic does what is expected. */ - if (row.shadowHidden === 1 && onlyForCurrentUser.indexOf(row.videoID) === -1) { - // The current visitor’s IP did not submit for the current video. - // Do not send shadowHidden segments to them. - continue; - } - // Split up the rows per video and group overlapping segments together. - if (!(row.videoID in rowGroupsPerVideo)) { - rowGroupsPerVideo[row.videoID] = []; - } - if (previousVideoID === row.videoID && row.startTime <= previousEndTime) { - rowGroupsPerVideo[row.videoID][rowGroupsPerVideo[row.videoID].length - 1].push(row); - previousEndTime = Math.max(previousEndTime, row.endTime); - } else { - rowGroupsPerVideo[row.videoID].push([row]); - previousVideoID = row.videoID; - previousEndTime = row.endTime; - } - } - - /** @type {Object.} */ - const output = {}; - for (const videoID in rowGroupsPerVideo) { - const pickedVideosForVideoID = []; - for (const group of rowGroupsPerVideo[videoID]) { - pickedVideosForVideoID.push(pickWeightedRandomRow(group)); - } - output[videoID] = pickedVideosForVideoID.map(row => ({ videoID: row.videoID, segment: [row.startTime, row.endTime], category: row.category, UUID: row.UUID })); - } - return output; -} - -const minimumPrefix = config.minimumPrefix || '3'; -const maximumPrefix = config.maximumPrefix || '32'; // Half the hash. -const prefixChecker = new RegExp('^[\\dA-F]{' + minimumPrefix + ',' + maximumPrefix + '}$', 'i'); +const databases = require('../databases/databases.js'); +const db = databases.db; module.exports = async function (req, res) { - if (!prefixChecker.test(req.params.prefix)) { - res.sendStatus(400).end(); // Exit early on faulty prefix + let hashPrefix = req.params.prefix; + if (!hashPrefixTester(req.params.prefix)) { + res.status(400).send("Hash prefix does not match format requirements."); // Exit early on faulty prefix + return; } - const segments = getSkipSegmentsByHash( - req.params.prefix.toLowerCase(), - getHash(getIP(req) + config.globalSalt) - ); + const categories = req.query.categories + ? JSON.parse(req.query.categories) + : req.query.category + ? [req.query.category] + : ['sponsor']; - if (Object.keys(segments).length > 0) { - res.send(segments); - } else { - res.sendStatus(404); // No skipable segments within this prefix + // Get all video id's that match hash prefix + const videoIds = db.prepare('all', 'SELECT DISTINCT videoId, hashedVideoID from sponsorTimes WHERE hashedVideoID LIKE ?', [hashPrefix+'%']); + if (videoIds.length === 0) { + res.sendStatus(404); + return; } + + let segments = videoIds.map((video) => { + return { + videoID: video.videoID, + hash: video.hashedVideoID, + segments: getSegments(video.videoID, categories) + }; + }); + + res.status(200).json(segments); } \ No newline at end of file diff --git a/src/routes/voteOnSponsorTime.js b/src/routes/voteOnSponsorTime.js index 8f9b381..5ebdc6d 100644 --- a/src/routes/voteOnSponsorTime.js +++ b/src/routes/voteOnSponsorTime.js @@ -1,4 +1,3 @@ -var fs = require('fs'); var config = require('../config.js'); var getHash = require('../utils/getHash.js'); @@ -370,8 +369,6 @@ async function voteOnSponsorTime(req, res) { } module.exports = { - voteOnSponsorTime, - endpoint: function (req, res) { - voteOnSponsorTime(req, res); - }, - }; + voteOnSponsorTime, + endpoint: voteOnSponsorTime +}; diff --git a/src/utils/hashPrefixTester.js b/src/utils/hashPrefixTester.js new file mode 100644 index 0000000..0a6639e --- /dev/null +++ b/src/utils/hashPrefixTester.js @@ -0,0 +1,11 @@ +const config = require('../config.js'); +const logger = require('./logger.js'); + +const minimumPrefix = config.minimumPrefix || '3'; +const maximumPrefix = config.maximumPrefix || '32'; // Half the hash. + +const prefixChecker = new RegExp('^[\\da-f]{' + minimumPrefix + ',' + maximumPrefix + '}$', 'i'); + +module.exports = (prefix) => { + return prefixChecker.test(prefix); +}; \ No newline at end of file diff --git a/test/cases/getSegmentsByHash.js b/test/cases/getSegmentsByHash.js new file mode 100644 index 0000000..6b6de50 --- /dev/null +++ b/test/cases/getSegmentsByHash.js @@ -0,0 +1,85 @@ +var request = require('request'); +var db = require('../../src/databases/databases.js').db; +var utils = require('../utils.js'); +var getHash = require('../../src/utils/getHash.js'); + + +/* + *CREATE TABLE IF NOT EXISTS "sponsorTimes" ( + "videoID" TEXT NOT NULL, + "startTime" REAL NOT NULL, + "endTime" REAL NOT NULL, + "votes" INTEGER NOT NULL, + "UUID" TEXT NOT NULL UNIQUE, + "userID" TEXT NOT NULL, + "timeSubmitted" INTEGER NOT NULL, + "views" INTEGER NOT NULL, + "shadowHidden" INTEGER NOT NULL +); + */ + +describe('getSegmentsByHash', () => { + before(() => { + let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES"; + db.exec(startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910 + db.exec(startOfQuery + "('getSegmentsByHash-0', 20, 30, 2, 'getSegmentsByHash-0-1', 'testman', 100, 150, 'intro', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910 + db.exec(startOfQuery + "('getSegmentsByHash-noMatchHash', 40, 50, 2, 'getSegmentsByHash-noMatchHash', 'testman', 0, 50, 'sponsor', 0, 'fdaffnoMatchHash')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910 + db.exec(startOfQuery + "('getSegmentsByHash-1', 60, 70, 2, 'getSegmentsByHash-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('getSegmentsByHash-1', 1) + "')"); // hash = 3272fa85ee0927f6073ef6f07ad5f3146047c1abba794cfa364d65ab9921692b + }); + + it('Should be able to get a 200', (done) => { + request.get(utils.getbaseURL() + + '/api/skipSegments/3272f?categories=["sponsor", "intro"]', null, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode !== 200) done("non 200 status code, was " + res.statusCode); + else { + done(); + } // pass + }); + }); + + it('Should be able to get a 200 with empty segments for video but no matching categories', (done) => { + request.get(utils.getbaseURL() + + '/api/skipSegments/3272f?categories=["shilling"]', null, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode !== 200) done("non 200 status code, was " + res.statusCode); + else { + if (JSON.parse(body) && JSON.parse(body).length > 0 && JSON.parse(body)[0].segments.length === 0) { + done(); // pass + } else { + done("response had segments"); + } + } + }); + }); + + it('Should be able to get a 404 if no videos', (done) => { + request.get(utils.getbaseURL() + + '/api/skipSegments/11111?categories=["shilling"]', null, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode !== 404) done("non 404 status code, was " + res.statusCode); + else { + done(); // pass + } + }); + }); + + it('Should be able to get multiple videos', (done) => { + request.get(utils.getbaseURL() + + '/api/skipSegments/fdaf?categories=["sponsor","intro"]', null, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode !== 200) done("non 200 status code, was " + res.statusCode); + else { + body = JSON.parse(body); + if (body.length !== 2) done("expected 2 video, got " + body.length); + else if (body[0].segments.length !== 2) done("expected 2 segments for first video, got " + body[0].segments.length); + else if (body[1].segments.length !== 1) done("expected 1 segment for second video, got " + body[1].segments.length); + else done(); + } + }); + }); +}); \ No newline at end of file diff --git a/test/cases/oldGetSponsorTime.js b/test/cases/oldGetSponsorTime.js index 8fe19d2..d4b9a88 100644 --- a/test/cases/oldGetSponsorTime.js +++ b/test/cases/oldGetSponsorTime.js @@ -21,8 +21,8 @@ var getHash = require('../../src/utils/getHash.js'); describe('getVideoSponsorTime (Old get method)', () => { before(() => { let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES"; - db.exec(startOfQuery + "('old-testtesttest', 1, 11, 2, 'uuid-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('old-testtesttest', 0) + "')"); - db.exec(startOfQuery + "('old-testtesttest,test', 1, 11, 2, 'uuid-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('old-testtesttest,test', 0) + "')"); + db.exec(startOfQuery + "('old-testtesttest', 1, 11, 2, 'uuid-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('old-testtesttest', 1) + "')"); + db.exec(startOfQuery + "('old-testtesttest,test', 1, 11, 2, 'uuid-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('old-testtesttest,test', 1) + "')"); }); it('Should be able to get a time', (done) => { diff --git a/test/cases/voteOnSponsorTime.js b/test/cases/voteOnSponsorTime.js index 8ecb3b3..e99c2fe 100644 --- a/test/cases/voteOnSponsorTime.js +++ b/test/cases/voteOnSponsorTime.js @@ -6,22 +6,22 @@ const getHash = require('../../src/utils/getHash.js'); describe('voteOnSponsorTime', () => { before(() => { let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES"; - db.exec(startOfQuery + "('vote-testtesttest', 1, 11, 2, 'vote-uuid-0', 'testman', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')"); - db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 2, 'vote-uuid-1', 'testman', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')"); - db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 10, 'vote-uuid-1.5', 'testman', 0, 50, 'outro', 0, '"+getHash('vote-testtesttest')+"')"); - db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 10, 'vote-uuid-1.6', 'testman', 0, 50, 'interaction', 0, '"+getHash('vote-testtesttest')+"')"); - db.exec(startOfQuery + "('vote-testtesttest3', 20, 33, 10, 'vote-uuid-2', 'testman', 0, 50, 'intro', 0, '"+getHash('vote-testtesttest')+"')"); - db.exec(startOfQuery + "('vote-testtesttest,test', 1, 11, 100, 'vote-uuid-3', 'testman', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')"); - db.exec(startOfQuery + "('vote-test3', 1, 11, 2, 'vote-uuid-4', 'testman', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')"); - db.exec(startOfQuery + "('vote-test3', 7, 22, -3, 'vote-uuid-5', 'testman', 0, 50, 'intro', 0, '"+getHash('vote-testtesttest')+"')"); - db.exec(startOfQuery + "('vote-multiple', 1, 11, 2, 'vote-uuid-6', 'testman', 0, 50, 'intro', 0, '"+getHash('vote-testtesttest')+"')"); - db.exec(startOfQuery + "('vote-multiple', 20, 33, 2, 'vote-uuid-7', 'testman', 0, 50, 'intro', 0, '"+getHash('vote-testtesttest')+"')"); - db.exec(startOfQuery + "('voter-submitter', 1, 11, 2, 'vote-uuid-8', '" + getHash("randomID") + "', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')"); - db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-9', '" + getHash("randomID2") + "', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')"); - db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-10', '" + getHash("randomID3") + "', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')"); - db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-11', '" + getHash("randomID4") + "', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')"); - db.exec(startOfQuery + "('own-submission-video', 1, 11, 500, 'own-submission-uuid', '"+ getHash('own-submission-id') +"', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')"); - db.exec(startOfQuery + "('not-own-submission-video', 1, 11, 500, 'not-own-submission-uuid', '"+ getHash('somebody-else-id') +"', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest')+"')"); + db.exec(startOfQuery + "('vote-testtesttest', 1, 11, 2, 'vote-uuid-0', 'testman', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest', 1)+"')"); + db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 2, 'vote-uuid-1', 'testman', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest2', 1)+"')"); + db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 10, 'vote-uuid-1.5', 'testman', 0, 50, 'outro', 0, '"+getHash('vote-testtesttest2', 1)+"')"); + db.exec(startOfQuery + "('vote-testtesttest2', 1, 11, 10, 'vote-uuid-1.6', 'testman', 0, 50, 'interaction', 0, '"+getHash('vote-testtesttest2', 1)+"')"); + db.exec(startOfQuery + "('vote-testtesttest3', 20, 33, 10, 'vote-uuid-2', 'testman', 0, 50, 'intro', 0, '"+getHash('vote-testtesttest3', 1)+"')"); + db.exec(startOfQuery + "('vote-testtesttest,test', 1, 11, 100, 'vote-uuid-3', 'testman', 0, 50, 'sponsor', 0, '"+getHash('vote-testtesttest,test', 1)+"')"); + db.exec(startOfQuery + "('vote-test3', 1, 11, 2, 'vote-uuid-4', 'testman', 0, 50, 'sponsor', 0, '"+getHash('vote-test3', 1)+"')"); + db.exec(startOfQuery + "('vote-test3', 7, 22, -3, 'vote-uuid-5', 'testman', 0, 50, 'intro', 0, '"+getHash('vote-test3', 1)+"')"); + db.exec(startOfQuery + "('vote-multiple', 1, 11, 2, 'vote-uuid-6', 'testman', 0, 50, 'intro', 0, '"+getHash('vote-multiple', 1)+"')"); + db.exec(startOfQuery + "('vote-multiple', 20, 33, 2, 'vote-uuid-7', 'testman', 0, 50, 'intro', 0, '"+getHash('vote-multiple', 1)+"')"); + db.exec(startOfQuery + "('voter-submitter', 1, 11, 2, 'vote-uuid-8', '" + getHash("randomID") + "', 0, 50, 'sponsor', 0, '"+getHash('voter-submitter', 1)+"')"); + db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-9', '" + getHash("randomID2") + "', 0, 50, 'sponsor', 0, '"+getHash('voter-submitter2', 1)+"')"); + db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-10', '" + getHash("randomID3") + "', 0, 50, 'sponsor', 0, '"+getHash('voter-submitter2', 1)+"')"); + db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-11', '" + getHash("randomID4") + "', 0, 50, 'sponsor', 0, '"+getHash('voter-submitter2', 1)+"')"); + db.exec(startOfQuery + "('own-submission-video', 1, 11, 500, 'own-submission-uuid', '"+ getHash('own-submission-id') +"', 0, 50, 'sponsor', 0, '"+getHash('own-submission-video', 1)+"')"); + db.exec(startOfQuery + "('not-own-submission-video', 1, 11, 500, 'not-own-submission-uuid', '"+ getHash('somebody-else-id') +"', 0, 50, 'sponsor', 0, '"+getHash('not-own-submission-video', 1)+"')"); db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("VIPUser") + "')"); privateDB.exec("INSERT INTO shadowBannedUsers (userID) VALUES ('" + getHash("randomID4") + "')"); From 84c8eeccc6172bffe5444d1b890b273b89a2b57c Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Mon, 31 Aug 2020 01:06:50 +0100 Subject: [PATCH 16/40] added hash prefix test for missing categories --- test/cases/getSegmentsByHash.js | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test/cases/getSegmentsByHash.js b/test/cases/getSegmentsByHash.js index 6b6de50..fb83526 100644 --- a/test/cases/getSegmentsByHash.js +++ b/test/cases/getSegmentsByHash.js @@ -82,4 +82,22 @@ describe('getSegmentsByHash', () => { } }); }); + + it('Should be able to get 200 for no categories (default sponsor)', (done) => { + request.get(utils.getbaseURL() + + '/api/skipSegments/fdaf', null, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode !== 200) done("non 200 status code, was " + res.statusCode); + else { + console.log(body); + body = JSON.parse(body); + if (body.length !== 2) done("expected 2 videos, got " + body.length); + else if (body[0].segments.length !== 1) done("expected 1 segments for first video, got " + body[0].segments.length); + else if (body[1].segments.length !== 1) done("expected 1 segments for second video, got " + body[1].segments.length); + else if (body[0].segments[0].category !== 'sponsor' || body[1].segments[0].category !== 'sponsor') done("both segments are not sponsor"); + else done(); + } + }); + }); }); \ No newline at end of file From 28bd24022b3792d9c96cecbafe6f60b4dc85c4e1 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Mon, 31 Aug 2020 01:08:18 +0100 Subject: [PATCH 17/40] removed schema comments and removed test log --- test/cases/getSegmentsByHash.js | 16 ---------------- test/cases/getSkipSegments.js | 13 ------------- 2 files changed, 29 deletions(-) diff --git a/test/cases/getSegmentsByHash.js b/test/cases/getSegmentsByHash.js index fb83526..1c60484 100644 --- a/test/cases/getSegmentsByHash.js +++ b/test/cases/getSegmentsByHash.js @@ -3,21 +3,6 @@ var db = require('../../src/databases/databases.js').db; var utils = require('../utils.js'); var getHash = require('../../src/utils/getHash.js'); - -/* - *CREATE TABLE IF NOT EXISTS "sponsorTimes" ( - "videoID" TEXT NOT NULL, - "startTime" REAL NOT NULL, - "endTime" REAL NOT NULL, - "votes" INTEGER NOT NULL, - "UUID" TEXT NOT NULL UNIQUE, - "userID" TEXT NOT NULL, - "timeSubmitted" INTEGER NOT NULL, - "views" INTEGER NOT NULL, - "shadowHidden" INTEGER NOT NULL -); - */ - describe('getSegmentsByHash', () => { before(() => { let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES"; @@ -90,7 +75,6 @@ describe('getSegmentsByHash', () => { if (err) done("Couldn't call endpoint"); else if (res.statusCode !== 200) done("non 200 status code, was " + res.statusCode); else { - console.log(body); body = JSON.parse(body); if (body.length !== 2) done("expected 2 videos, got " + body.length); else if (body[0].segments.length !== 1) done("expected 1 segments for first video, got " + body[0].segments.length); diff --git a/test/cases/getSkipSegments.js b/test/cases/getSkipSegments.js index db4698e..0314028 100644 --- a/test/cases/getSkipSegments.js +++ b/test/cases/getSkipSegments.js @@ -3,19 +3,6 @@ var db = require('../../src/databases/databases.js').db; var utils = require('../utils.js'); var getHash = require('../../src/utils/getHash.js'); -/* - *CREATE TABLE IF NOT EXISTS "sponsorTimes" ( - "videoID" TEXT NOT NULL, - "startTime" REAL NOT NULL, - "endTime" REAL NOT NULL, - "votes" INTEGER NOT NULL, - "UUID" TEXT NOT NULL UNIQUE, - "userID" TEXT NOT NULL, - "timeSubmitted" INTEGER NOT NULL, - "views" INTEGER NOT NULL, - "shadowHidden" INTEGER NOT NULL -); - */ describe('getSkipSegments', () => { before(() => { From 36ce803828bd2dfec60037e5a871d0f815bd4a35 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Mon, 31 Aug 2020 01:44:30 +0100 Subject: [PATCH 18/40] Update src/routes/getIsUserVIP.js Co-authored-by: Ajay Ramachandran --- src/routes/getIsUserVIP.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/routes/getIsUserVIP.js b/src/routes/getIsUserVIP.js index d38ef9d..63efd02 100644 --- a/src/routes/getIsUserVIP.js +++ b/src/routes/getIsUserVIP.js @@ -4,7 +4,7 @@ var getHash = require('../utils/getHash.js'); const logger = require('../utils/logger.js'); const isUserVIP = require('../utils/isUserVIP.js'); -module.exports = function getUsername (req, res) { +module.exports = (req, res) => { let userID = req.query.userID; if (userID == undefined) { @@ -28,4 +28,4 @@ module.exports = function getUsername (req, res) { return; } -} \ No newline at end of file +} From 1e643c1c0740d6b2c3f6849152bc28f4110c3c84 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Mon, 31 Aug 2020 01:48:41 +0100 Subject: [PATCH 19/40] categorys -> categories --- src/routes/postNoSegments.js | 20 ++++++++++---------- test/cases/noSegmentRecords.js | 26 +++++++++++++------------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/routes/postNoSegments.js b/src/routes/postNoSegments.js index 6494095..6b2b7aa 100644 --- a/src/routes/postNoSegments.js +++ b/src/routes/postNoSegments.js @@ -7,14 +7,14 @@ module.exports = (req, res) => { // Collect user input data let videoID = req.body.videoID; let userID = req.body.userID; - let categorys = req.body.categorys; + let categories = req.body.categories; // Check input data is valid if (!videoID || !userID - || !categorys - || !Array.isArray(categorys) - || categorys.length === 0 + || !categories + || !Array.isArray(categories) + || categories.length === 0 ) { res.status(400).json({ status: 400, @@ -45,20 +45,20 @@ module.exports = (req, res) => { }); } - // get user categorys not already submitted that match accepted format - let categorysToMark = categorys.filter((category) => { + // get user categories not already submitted that match accepted format + let categoriesToMark = categories.filter((category) => { return !!category.match(/^[a-zA-Z]+$/); }).filter((category) => { return noSegmentList.indexOf(category) === -1; }); // remove any duplicates - categorysToMark = categorysToMark.filter((category, index) => { - return categorysToMark.indexOf(category) === index; + categoriesToMark = categoriesToMark.filter((category, index) => { + return categoriesToMark.indexOf(category) === index; }); // create database entry - categorysToMark.forEach((category) => { + categoriesToMark.forEach((category) => { try { db.prepare('run', "INSERT INTO noSegments (videoID, userID, category) VALUES(?, ?, ?)", [videoID, userID, category]); } catch (err) { @@ -73,6 +73,6 @@ module.exports = (req, res) => { res.status(200).json({ status: 200, - submitted: categorysToMark + submitted: categoriesToMark }); }; \ No newline at end of file diff --git a/test/cases/noSegmentRecords.js b/test/cases/noSegmentRecords.js index d902478..a09264f 100644 --- a/test/cases/noSegmentRecords.js +++ b/test/cases/noSegmentRecords.js @@ -25,11 +25,11 @@ describe('noSegmentRecords', () => { else done('Version isn\'t greater that 1. Version is ' + version); }); - it('Should be able to submit categorys not in video (http response)', (done) => { + it('Should be able to submit categories not in video (http response)', (done) => { let json = { videoID: 'no-segments-video-id', userID: 'VIPUser-noSegments', - categorys: [ + categories: [ 'outro', 'shilling', 'shilling', @@ -64,11 +64,11 @@ describe('noSegmentRecords', () => { }); }); - it('Should be able to submit categorys not in video (sql check)', (done) => { + it('Should be able to submit categories not in video (sql check)', (done) => { let json = { videoID: 'no-segments-video-id-1', userID: 'VIPUser-noSegments', - categorys: [ + categories: [ 'outro', 'shilling', 'shilling', @@ -110,11 +110,11 @@ describe('noSegmentRecords', () => { }); }); - it('Should return 400 for no categorys', (done) => { + it('Should return 400 for no categories', (done) => { let json = { videoID: 'test', userID: 'test', - categorys: [] + categories: [] }; request.post(utils.getbaseURL() @@ -133,7 +133,7 @@ describe('noSegmentRecords', () => { let json = { videoID: 'test', userID: null, - categorys: ['sponsor'] + categories: ['sponsor'] }; request.post(utils.getbaseURL() @@ -152,7 +152,7 @@ describe('noSegmentRecords', () => { let json = { videoID: null, userID: 'test', - categorys: ['sponsor'] + categories: ['sponsor'] }; request.post(utils.getbaseURL() @@ -167,11 +167,11 @@ describe('noSegmentRecords', () => { }); }); - it('Should return 400 object categorys)', (done) => { + it('Should return 400 object categories)', (done) => { let json = { videoID: 'test', userID: 'test', - categorys: {} + categories: {} }; request.post(utils.getbaseURL() @@ -186,11 +186,11 @@ describe('noSegmentRecords', () => { }); }); - it('Should return 400 bad format categorys', (done) => { + it('Should return 400 bad format categories', (done) => { let json = { videoID: 'test', userID: 'test', - categorys: 'sponsor' + categories: 'sponsor' }; request.post(utils.getbaseURL() @@ -209,7 +209,7 @@ describe('noSegmentRecords', () => { let json = { videoID: 'test', userID: 'test', - categorys: [ + categories: [ 'sponsor' ] }; From 754d3762dfb81a42a0de974f1944f56f7cb7176b Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Mon, 31 Aug 2020 01:50:36 +0100 Subject: [PATCH 20/40] fixed test category name --- test/cases/getIsUserVIP.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cases/getIsUserVIP.js b/test/cases/getIsUserVIP.js index 0b25e7d..a971017 100644 --- a/test/cases/getIsUserVIP.js +++ b/test/cases/getIsUserVIP.js @@ -3,7 +3,7 @@ var utils = require('../utils.js'); var db = require('../../src/databases/databases.js').db; var getHash = require('../../src/utils/getHash.js'); -describe('getSavedTimeForUser', () => { +describe('getIsUserVIP', () => { before(() => { db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("supertestman") + "')"); }); From aa878482d31332a409deb072ff7348517f6c242c Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Mon, 31 Aug 2020 01:55:38 +0100 Subject: [PATCH 21/40] removed status from no segment responses --- src/routes/postNoSegments.js | 4 ---- test/cases/noSegmentRecords.js | 1 - 2 files changed, 5 deletions(-) diff --git a/src/routes/postNoSegments.js b/src/routes/postNoSegments.js index 6b2b7aa..6aab052 100644 --- a/src/routes/postNoSegments.js +++ b/src/routes/postNoSegments.js @@ -17,7 +17,6 @@ module.exports = (req, res) => { || categories.length === 0 ) { res.status(400).json({ - status: 400, message: 'Bad Format' }); return; @@ -29,7 +28,6 @@ module.exports = (req, res) => { if (!userIsVIP) { res.status(403).json({ - status: 403, message: 'Must be a VIP to mark videos.' }); return; @@ -65,14 +63,12 @@ module.exports = (req, res) => { logger.error("Error submitting 'noSegment' marker for category '" + category + "' for video '" + videoID + "'"); logger.error(err); res.status(500).json({ - status: 500, message: "Internal Server Error: Could not write marker to the database." }); } }); res.status(200).json({ - status: 200, submitted: categoriesToMark }); }; \ No newline at end of file diff --git a/test/cases/noSegmentRecords.js b/test/cases/noSegmentRecords.js index a09264f..3989258 100644 --- a/test/cases/noSegmentRecords.js +++ b/test/cases/noSegmentRecords.js @@ -40,7 +40,6 @@ describe('noSegmentRecords', () => { }; let expected = { - status: 200, submitted: [ 'outro', 'shilling' From 82d59e159fc5136021fbe5a62bcbd99ae42351a6 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Mon, 31 Aug 2020 02:42:25 +0100 Subject: [PATCH 22/40] added post and get test to hash prefix --- databases/_upgrade_sponsorTimes_3.sql | 2 ++ src/databases/databases.js | 5 ++-- test/cases/getHash.js | 4 +++ test/cases/getSegmentsByHash.js | 36 +++++++++++++++++++++++++++ 4 files changed, 45 insertions(+), 2 deletions(-) diff --git a/databases/_upgrade_sponsorTimes_3.sql b/databases/_upgrade_sponsorTimes_3.sql index ee2e041..1d2edbd 100644 --- a/databases/_upgrade_sponsorTimes_3.sql +++ b/databases/_upgrade_sponsorTimes_3.sql @@ -15,6 +15,8 @@ CREATE TABLE "sqlb_temp_table_3" ( "shadowHidden" INTEGER NOT NULL, "hashedVideoID" TEXT NOT NULL ); + +/* hash upgade test sha256('vid') = '1ff838dc6ca9680d88455341118157d59a055fe6d0e3870f9c002847bebe4663' */ INSERT INTO sqlb_temp_table_3 SELECT *, sha256(videoID) FROM sponsorTimes; DROP TABLE sponsorTimes; diff --git a/src/databases/databases.js b/src/databases/databases.js index 5625d6c..a6f38ab 100644 --- a/src/databases/databases.js +++ b/src/databases/databases.js @@ -5,6 +5,7 @@ var path = require('path'); var Sqlite = require('./Sqlite.js') var Mysql = require('./Mysql.js'); const logger = require('../utils/logger.js'); +const getHash = require('../utils/getHash.js'); let options = { readonly: config.readOnly, @@ -34,8 +35,8 @@ if (config.mysql) { } if (!config.readOnly) { - db.function("sha256", function (string) { - return require('crypto').createHash("sha256").update(string).digest("hex"); + db.function("sha256", (string) => { + return getHash(string, 1); }); // Upgrade database if required diff --git a/test/cases/getHash.js b/test/cases/getHash.js index 15d7cc5..896076d 100644 --- a/test/cases/getHash.js +++ b/test/cases/getHash.js @@ -13,6 +13,10 @@ describe('getHash', () => { assert.equal(getHash("test"), "2f327ef967ade1ebf4319163f7debbda9cc17bb0c8c834b00b30ca1cf1c256ee"); }); + it ('Should be able to output the same has the DB upgrade script will output', () => { + assert.equal(getHash("vid", 1), "1ff838dc6ca9680d88455341118157d59a055fe6d0e3870f9c002847bebe4663"); + }); + it ('Should take a variable number of passes', () => { assert.equal(getHash("test", 1), "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"); assert.equal(getHash("test", 2), "7b3d979ca8330a94fa7e9e1b466d8b99e0bcdea1ec90596c0dcc8d7ef6b4300c"); diff --git a/test/cases/getSegmentsByHash.js b/test/cases/getSegmentsByHash.js index 1c60484..1717e98 100644 --- a/test/cases/getSegmentsByHash.js +++ b/test/cases/getSegmentsByHash.js @@ -84,4 +84,40 @@ describe('getSegmentsByHash', () => { } }); }); + + it('Should be able to post a segment and get it using endpoint', (done) => { + let testID = 'abc123goodVideo'; + request.post(utils.getbaseURL() + + "/api/postVideoSponsorTimes", { + json: { + userID: "test", + videoID: testID, + segments: [{ + segment: [13, 17], + category: "sponsor" + }] + } + }, + (err, res, body) => { + if (err) done('(post) ' + err); + else if (res.statusCode === 200) { + request.get(utils.getbaseURL() + + '/api/skipSegments/'+getHash(testID, 1).substring(0,3), null, + (err, res, body) => { + if (err) done("(get) Couldn't call endpoint"); + else if (res.statusCode !== 200) done("(get) non 200 status code, was " + res.statusCode); + else { + body = JSON.parse(body); + if (body.length !== 1) done("(get) expected 1 video, got " + body.length); + else if (body[0].segments.length !== 1) done("(get) expected 1 segments for first video, got " + body[0].segments.length); + else if (body[0].segments[0].category !== 'sponsor') done("(get) segment should be sponsor, was "+body[0].segments[0].category); + else done(); + } + }); + } else { + done("(post) non 200 status code, was " + res.statusCode); + } + } + ); + }); }); \ No newline at end of file From 88e6c6f93c05769a7847464e294029526eaa7c5d Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Mon, 31 Aug 2020 02:52:12 +0100 Subject: [PATCH 23/40] added db upgrade tests --- databases/_upgrade_private_1.sql | 6 ++++++ test/cases/dbUpgrade.js | 12 ++++++++++++ test/cases/getSegmentsByHash.js | 6 ++++++ test/cases/noSegmentRecords.js | 2 +- 4 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 databases/_upgrade_private_1.sql create mode 100644 test/cases/dbUpgrade.js diff --git a/databases/_upgrade_private_1.sql b/databases/_upgrade_private_1.sql new file mode 100644 index 0000000..8c7ab33 --- /dev/null +++ b/databases/_upgrade_private_1.sql @@ -0,0 +1,6 @@ +BEGIN TRANSACTION; + +/* Add version to config */ +INSERT INTO config (key, value) VALUES("version", 1); + +COMMIT; \ No newline at end of file diff --git a/test/cases/dbUpgrade.js b/test/cases/dbUpgrade.js new file mode 100644 index 0000000..8228d27 --- /dev/null +++ b/test/cases/dbUpgrade.js @@ -0,0 +1,12 @@ +const databases = require('../../src/databases/databases.js'); +const db = databases.db; +const privateDB = databases.privateDB; + +describe('dbUpgrade', () => { + it('Should update the database version when starting the application', (done) => { + let dbVersion = db.prepare('get', 'SELECT key, value FROM config where key = ?', ['version']).value; + let privateVersion = privateDB.prepare('get', 'SELECT key, value FROM config where key = ?', ['version']).value; + if (dbVersion >= 1 && privateVersion >= 1) done(); + else done('Versions are not at least 1. db is ' + dbVersion + ', private is ' + privateVersion); + }); +}); \ No newline at end of file diff --git a/test/cases/getSegmentsByHash.js b/test/cases/getSegmentsByHash.js index 1717e98..191b58a 100644 --- a/test/cases/getSegmentsByHash.js +++ b/test/cases/getSegmentsByHash.js @@ -12,6 +12,12 @@ describe('getSegmentsByHash', () => { db.exec(startOfQuery + "('getSegmentsByHash-1', 60, 70, 2, 'getSegmentsByHash-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('getSegmentsByHash-1', 1) + "')"); // hash = 3272fa85ee0927f6073ef6f07ad5f3146047c1abba794cfa364d65ab9921692b }); + it('Should update the database version when starting the application', (done) => { + let version = db.prepare('get', 'SELECT key, value FROM config where key = ?', ['version']).value; + if (version > 2) done(); + else done('Version isn\'t greater than 2. Version is ' + version); + }); + it('Should be able to get a 200', (done) => { request.get(utils.getbaseURL() + '/api/skipSegments/3272f?categories=["sponsor", "intro"]', null, diff --git a/test/cases/noSegmentRecords.js b/test/cases/noSegmentRecords.js index d902478..8c2c393 100644 --- a/test/cases/noSegmentRecords.js +++ b/test/cases/noSegmentRecords.js @@ -22,7 +22,7 @@ describe('noSegmentRecords', () => { it('Should update the database version when starting the application', (done) => { let version = db.prepare('get', 'SELECT key, value FROM config where key = ?', ['version']).value; if (version > 1) done(); - else done('Version isn\'t greater that 1. Version is ' + version); + else done('Version isn\'t greater than 1. Version is ' + version); }); it('Should be able to submit categorys not in video (http response)', (done) => { From 00534d91d46c343c294c59175349489bb89d908e Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Mon, 31 Aug 2020 03:09:47 +0100 Subject: [PATCH 24/40] added underscore to category format check --- src/routes/postNoSegments.js | 2 +- test/cases/noSegmentRecords.js | 84 ++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) diff --git a/src/routes/postNoSegments.js b/src/routes/postNoSegments.js index 6aab052..cdbfce9 100644 --- a/src/routes/postNoSegments.js +++ b/src/routes/postNoSegments.js @@ -45,7 +45,7 @@ module.exports = (req, res) => { // get user categories not already submitted that match accepted format let categoriesToMark = categories.filter((category) => { - return !!category.match(/^[a-zA-Z]+$/); + return !!category.match(/^[_a-zA-Z]+$/); }).filter((category) => { return noSegmentList.indexOf(category) === -1; }); diff --git a/test/cases/noSegmentRecords.js b/test/cases/noSegmentRecords.js index 3989258..d439b13 100644 --- a/test/cases/noSegmentRecords.js +++ b/test/cases/noSegmentRecords.js @@ -96,6 +96,90 @@ describe('noSegmentRecords', () => { }); }); + it('Should be able to submit categories with _ in the category', (done) => { + let json = { + videoID: 'underscore', + userID: 'VIPUser-noSegments', + categories: [ + 'word_word', + ] + }; + + request.post(utils.getbaseURL() + + "/api/postNoSegments", {json}, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + let result = db.prepare('all', 'SELECT * FROM noSegments WHERE videoID = ?', ['underscore']); + if (result.length !== 1) { + console.log(result); + done("Expected 1 entrys in db, got " + result.length); + } else { + done(); + } + } else { + console.log(body); + done("Status code was " + res.statusCode); + } + }); + }); + + it('Should be able to submit categories with upper and lower case in the category', (done) => { + let json = { + videoID: 'bothCases', + userID: 'VIPUser-noSegments', + categories: [ + 'wordWord', + ] + }; + + request.post(utils.getbaseURL() + + "/api/postNoSegments", {json}, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + let result = db.prepare('all', 'SELECT * FROM noSegments WHERE videoID = ?', ['bothCases']); + if (result.length !== 1) { + console.log(result); + done("Expected 1 entrys in db, got " + result.length); + } else { + done(); + } + } else { + console.log(body); + done("Status code was " + res.statusCode); + } + }); + }); + + it('Should not be able to submit categories with $ in the category', (done) => { + let json = { + videoID: 'specialChar', + userID: 'VIPUser-noSegments', + categories: [ + 'word&word', + ] + }; + + request.post(utils.getbaseURL() + + "/api/postNoSegments", {json}, + (err, res, body) => { + if (err) done(err); + else if (res.statusCode === 200) { + let result = db.prepare('all', 'SELECT * FROM noSegments WHERE videoID = ?', ['specialChar']); + if (result.length !== 0) { + console.log(result); + done("Expected 0 entrys in db, got " + result.length); + } else { + done(); + } + } else { + console.log(body); + done("Status code was " + res.statusCode); + } + }); + }); + it('Should return 400 for missing params', (done) => { request.post(utils.getbaseURL() + "/api/postNoSegments", {json: {}}, From 20675998437c5f5a405eff00464ddd448b44f539 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Mon, 31 Aug 2020 03:27:12 +0100 Subject: [PATCH 25/40] added test for hash prefix length --- src/utils/hashPrefixTester.js | 1 - test/cases/getSegmentsByHash.js | 42 +++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/utils/hashPrefixTester.js b/src/utils/hashPrefixTester.js index 0a6639e..f5aec61 100644 --- a/src/utils/hashPrefixTester.js +++ b/src/utils/hashPrefixTester.js @@ -1,5 +1,4 @@ const config = require('../config.js'); -const logger = require('./logger.js'); const minimumPrefix = config.minimumPrefix || '3'; const maximumPrefix = config.maximumPrefix || '32'; // Half the hash. diff --git a/test/cases/getSegmentsByHash.js b/test/cases/getSegmentsByHash.js index 191b58a..1084ee3 100644 --- a/test/cases/getSegmentsByHash.js +++ b/test/cases/getSegmentsByHash.js @@ -58,6 +58,48 @@ describe('getSegmentsByHash', () => { }); }); + it('Should return 400 prefix too short', (done) => { + request.get(utils.getbaseURL() + + '/api/skipSegments/11?categories=["shilling"]', null, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode !== 400) done("non 400 status code, was " + res.statusCode); + else { + done(); // pass + } + }); + }); + + it('Should return 400 prefix too long', (done) => { + let prefix = new Array(50).join('1'); + if (prefix.length <= 32) { // default value, config can change this + done('failed to generate a long enough string for the test ' + prefix.length); + return; + } + + request.get(utils.getbaseURL() + + '/api/skipSegments/'+prefix+'?categories=["shilling"]', null, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode !== 400) done("non 400 status code, was " + res.statusCode); + else { + done(); // pass + } + }); + }); + + it('Should not return 400 prefix in range', (done) => { + request.get(utils.getbaseURL() + + '/api/skipSegments/11111?categories=["shilling"]', null, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode === 400) done("prefix length 5 gave 400 " + res.statusCode); + else { + done(); // pass + } + }); + }); + it('Should be able to get multiple videos', (done) => { request.get(utils.getbaseURL() + '/api/skipSegments/fdaf?categories=["sponsor","intro"]', null, From 9c9c2a23cc7efcbbeecf6fac93b3128d7aa3dc03 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Mon, 31 Aug 2020 03:31:53 +0100 Subject: [PATCH 26/40] added comment to db upgrade sql file --- databases/_upgrade_private_1.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/databases/_upgrade_private_1.sql b/databases/_upgrade_private_1.sql index 8c7ab33..c4de7e9 100644 --- a/databases/_upgrade_private_1.sql +++ b/databases/_upgrade_private_1.sql @@ -1,5 +1,7 @@ BEGIN TRANSACTION; +/* for testing the db upgrade, don't remove because it looks empty */ + /* Add version to config */ INSERT INTO config (key, value) VALUES("version", 1); From 2c32460a6e9c698c70a3ca224c5b994e8ef08356 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Mon, 31 Aug 2020 04:17:50 +0100 Subject: [PATCH 27/40] more tests --- src/routes/getSkipSegmentsByHash.js | 1 + test/cases/getSegmentsByHash.js | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/src/routes/getSkipSegmentsByHash.js b/src/routes/getSkipSegmentsByHash.js index eec0e0e..702a28d 100644 --- a/src/routes/getSkipSegmentsByHash.js +++ b/src/routes/getSkipSegmentsByHash.js @@ -2,6 +2,7 @@ const hashPrefixTester = require('../utils/hashPrefixTester.js'); const getSegments = require('./getSkipSegments.js').cleanGetSegments; const databases = require('../databases/databases.js'); +const logger = require('../utils/logger.js'); const db = databases.db; module.exports = async function (req, res) { diff --git a/test/cases/getSegmentsByHash.js b/test/cases/getSegmentsByHash.js index 1084ee3..6571fe4 100644 --- a/test/cases/getSegmentsByHash.js +++ b/test/cases/getSegmentsByHash.js @@ -100,6 +100,30 @@ describe('getSegmentsByHash', () => { }); }); + it('Should return 404 for no hash', (done) => { + request.get(utils.getbaseURL() + + '/api/skipSegments/?categories=["shilling"]', null, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode !== 404) done("expected 404, got " + res.statusCode); + else { + done(); // pass + } + }); + }); + + it('Should return 500 for bad format categories', (done) => { // should probably be 400 + request.get(utils.getbaseURL() + + '/api/skipSegments/?categories=shilling', null, + (err, res, body) => { + if (err) done("Couldn't call endpoint"); + else if (res.statusCode !== 500) done("expected 500 got " + res.statusCode); + else { + done(); // pass + } + }); + }); + it('Should be able to get multiple videos', (done) => { request.get(utils.getbaseURL() + '/api/skipSegments/fdaf?categories=["sponsor","intro"]', null, From e33062feeb95df4c6bd6bf85ffd1d70e1b01016a Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Tue, 1 Sep 2020 21:37:54 +0100 Subject: [PATCH 28/40] updated endpoints --- src/app.js | 4 ++-- test/cases/getIsUserVIP.js | 8 ++++---- test/cases/noSegmentRecords.js | 24 ++++++++++++------------ 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/app.js b/src/app.js index 8cb937f..45715b7 100644 --- a/src/app.js +++ b/src/app.js @@ -92,10 +92,10 @@ app.get('/api/getTotalStats', getTotalStats); app.get('/api/getDaysSavedFormatted', getDaysSavedFormatted); //submit video containing no segments -app.post('/api/postNoSegments', postNoSegments); +app.post('/api/noSegments', postNoSegments); //get if user is a vip -app.get('/api/getIsUserVIP', getIsUserVIP); +app.get('/api/isUserVIP', getIsUserVIP); app.get('/database.db', function (req, res) { diff --git a/test/cases/getIsUserVIP.js b/test/cases/getIsUserVIP.js index a971017..1bbfbfe 100644 --- a/test/cases/getIsUserVIP.js +++ b/test/cases/getIsUserVIP.js @@ -10,7 +10,7 @@ describe('getIsUserVIP', () => { it('Should be able to get a 200', (done) => { request.get(utils.getbaseURL() - + "/api/getIsUserVIP?userID=supertestman", null, + + "/api/isUserVIP?userID=supertestman", null, (err, res, body) => { if (err) done("couldn't call endpoint"); else if (res.statusCode !== 200) done("non 200: " + res.statusCode); @@ -21,7 +21,7 @@ describe('getIsUserVIP', () => { it('Should get a 400 if no userID', (done) => { request.get(utils.getbaseURL() - + "/api/getIsUserVIP", null, + + "/api/isUserVIP", null, (err, res, body) => { if (err) done("couldn't call endpoint"); else if (res.statusCode !== 400) done("non 400: " + res.statusCode); @@ -31,7 +31,7 @@ describe('getIsUserVIP', () => { it('Should say a VIP is a VIP', (done) => { request.get(utils.getbaseURL() - + "/api/getIsUserVIP?userID=supertestman", null, + + "/api/isUserVIP?userID=supertestman", null, (err, res, body) => { if (err) done("couldn't call endpoint"); else if (res.statusCode !== 200) done("non 200: " + res.statusCode); @@ -44,7 +44,7 @@ describe('getIsUserVIP', () => { it('Should say a normal user is not a VIP', (done) => { request.get(utils.getbaseURL() - + "/api/getIsUserVIP?userID=regulartestman", null, + + "/api/isUserVIP?userID=regulartestman", null, (err, res, body) => { if (err) done("couldn't call endpoint"); else if (res.statusCode !== 200) done("non 200: " + res.statusCode); diff --git a/test/cases/noSegmentRecords.js b/test/cases/noSegmentRecords.js index d439b13..91dea2a 100644 --- a/test/cases/noSegmentRecords.js +++ b/test/cases/noSegmentRecords.js @@ -47,7 +47,7 @@ describe('noSegmentRecords', () => { }; request.post(utils.getbaseURL() - + "/api/postNoSegments", {json}, + + "/api/noSegments", {json}, (err, res, body) => { if (err) done(err); else if (res.statusCode === 200) { @@ -78,7 +78,7 @@ describe('noSegmentRecords', () => { }; request.post(utils.getbaseURL() - + "/api/postNoSegments", {json}, + + "/api/noSegments", {json}, (err, res, body) => { if (err) done(err); else if (res.statusCode === 200) { @@ -106,7 +106,7 @@ describe('noSegmentRecords', () => { }; request.post(utils.getbaseURL() - + "/api/postNoSegments", {json}, + + "/api/noSegments", {json}, (err, res, body) => { if (err) done(err); else if (res.statusCode === 200) { @@ -134,7 +134,7 @@ describe('noSegmentRecords', () => { }; request.post(utils.getbaseURL() - + "/api/postNoSegments", {json}, + + "/api/noSegments", {json}, (err, res, body) => { if (err) done(err); else if (res.statusCode === 200) { @@ -162,7 +162,7 @@ describe('noSegmentRecords', () => { }; request.post(utils.getbaseURL() - + "/api/postNoSegments", {json}, + + "/api/noSegments", {json}, (err, res, body) => { if (err) done(err); else if (res.statusCode === 200) { @@ -182,7 +182,7 @@ describe('noSegmentRecords', () => { it('Should return 400 for missing params', (done) => { request.post(utils.getbaseURL() - + "/api/postNoSegments", {json: {}}, + + "/api/noSegments", {json: {}}, (err, res, body) => { if (err) done(err); else if (res.statusCode === 400) { @@ -201,7 +201,7 @@ describe('noSegmentRecords', () => { }; request.post(utils.getbaseURL() - + "/api/postNoSegments", {json}, + + "/api/noSegments", {json}, (err, res, body) => { if (err) done(err); else if (res.statusCode === 400) { @@ -220,7 +220,7 @@ describe('noSegmentRecords', () => { }; request.post(utils.getbaseURL() - + "/api/postNoSegments", {json}, + + "/api/noSegments", {json}, (err, res, body) => { if (err) done(err); else if (res.statusCode === 400) { @@ -239,7 +239,7 @@ describe('noSegmentRecords', () => { }; request.post(utils.getbaseURL() - + "/api/postNoSegments", {json}, + + "/api/noSegments", {json}, (err, res, body) => { if (err) done(err); else if (res.statusCode === 400) { @@ -258,7 +258,7 @@ describe('noSegmentRecords', () => { }; request.post(utils.getbaseURL() - + "/api/postNoSegments", {json}, + + "/api/noSegments", {json}, (err, res, body) => { if (err) done(err); else if (res.statusCode === 400) { @@ -277,7 +277,7 @@ describe('noSegmentRecords', () => { }; request.post(utils.getbaseURL() - + "/api/postNoSegments", {json}, + + "/api/noSegments", {json}, (err, res, body) => { if (err) done(err); else if (res.statusCode === 400) { @@ -298,7 +298,7 @@ describe('noSegmentRecords', () => { }; request.post(utils.getbaseURL() - + "/api/postNoSegments", {json}, + + "/api/noSegments", {json}, (err, res, body) => { if (err) done(err); else if (res.statusCode === 403) { From 11b0cfb5365efe5d74244f6698a0e9aaa7ae4709 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Thu, 3 Sep 2020 23:05:13 +0100 Subject: [PATCH 29/40] add db schemas to docker image --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index efca3de..ea7d751 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,8 @@ COPY package.json . RUN npm install COPY index.js . COPY src src +RUN mkdir databases +COPY databases/*.sql databases/ COPY entrypoint.sh . EXPOSE 8080 CMD ./entrypoint.sh \ No newline at end of file From 9ee5c508e4434a130781df5f4defd5a321ca1480 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Thu, 3 Sep 2020 23:54:43 -0400 Subject: [PATCH 30/40] Switch to null instead of undefined and fix indentation --- src/routes/getSkipSegments.js | 252 +++++++++++++++++----------------- 1 file changed, 126 insertions(+), 126 deletions(-) diff --git a/src/routes/getSkipSegments.js b/src/routes/getSkipSegments.js index db6ce76..4bf6a6a 100644 --- a/src/routes/getSkipSegments.js +++ b/src/routes/getSkipSegments.js @@ -9,100 +9,100 @@ var getHash = require('../utils/getHash.js'); var getIP = require('../utils/getIP.js'); function cleanGetSegments(videoID, categories) { - let userHashedIP, shadowHiddenSegments; + let userHashedIP, shadowHiddenSegments; - let segments = []; + let segments = []; - try { - for (const category of categories) { - const categorySegments = db - .prepare( - 'all', - 'SELECT startTime, endTime, votes, UUID, shadowHidden FROM sponsorTimes WHERE videoID = ? and category = ? ORDER BY startTime', - [videoID, category] - ) - .filter(segment => { - if (segment.votes < -1) { - return false; //too untrustworthy, just ignore it - } + try { + for (const category of categories) { + const categorySegments = db + .prepare( + 'all', + 'SELECT startTime, endTime, votes, UUID, shadowHidden FROM sponsorTimes WHERE videoID = ? and category = ? ORDER BY startTime', + [videoID, category] + ) + .filter(segment => { + if (segment.votes < -1) { + return false; //too untrustworthy, just ignore it + } - //check if shadowHidden - //this means it is hidden to everyone but the original ip that submitted it - if (segment.shadowHidden != 1) { - return true; - } + //check if shadowHidden + //this means it is hidden to everyone but the original ip that submitted it + if (segment.shadowHidden != 1) { + return true; + } - if (shadowHiddenSegments === undefined) { - shadowHiddenSegments = privateDB.prepare('all', 'SELECT hashedIP FROM sponsorTimes WHERE videoID = ?', [videoID]); - } + if (shadowHiddenSegments === undefined) { + shadowHiddenSegments = privateDB.prepare('all', 'SELECT hashedIP FROM sponsorTimes WHERE videoID = ?', [videoID]); + } - //if this isn't their ip, don't send it to them - return shadowHiddenSegments.some(shadowHiddenSegment => { - if (userHashedIP === undefined) { - //hash the IP only if it's strictly necessary - userHashedIP = getHash(getIP(req) + config.globalSalt); - } - return shadowHiddenSegment.hashedIP === userHashedIP; - }); - }); + //if this isn't their ip, don't send it to them + return shadowHiddenSegments.some(shadowHiddenSegment => { + if (userHashedIP === undefined) { + //hash the IP only if it's strictly necessary + userHashedIP = getHash(getIP(req) + config.globalSalt); + } + return shadowHiddenSegment.hashedIP === userHashedIP; + }); + }); - chooseSegments(categorySegments).forEach(chosenSegment => { - segments.push({ - category, - segment: [chosenSegment.startTime, chosenSegment.endTime], - UUID: chosenSegment.UUID, - }); - }); + chooseSegments(categorySegments).forEach(chosenSegment => { + segments.push({ + category, + segment: [chosenSegment.startTime, chosenSegment.endTime], + UUID: chosenSegment.UUID, + }); + }); + } + + return segments; + } catch (err) { + if (err) { + logger.error(err); + return null; + } } - - return segments; - } catch (err) { - if (err) { - logger.error('j 2 Query failed'); - return undefined; - } - } } //gets a weighted random choice from the choices array based on their `votes` property. //amountOfChoices specifies the maximum amount of choices to return, 1 or more. //choices are unique function getWeightedRandomChoice(choices, amountOfChoices) { - //trivial case: no need to go through the whole process - if (amountOfChoices >= choices.length) { - return choices; - } - - //assign a weight to each choice - let totalWeight = 0; - choices = choices.map(choice => { - //The 3 makes -2 the minimum votes before being ignored completely - //https://www.desmos.com/calculator/c1duhfrmts - //this can be changed if this system increases in popularity. - const weight = Math.exp((choice.votes + 3), 0.85); - totalWeight += weight; - - return { ...choice, weight }; - }); - - //iterate and find amountOfChoices choices - const chosen = []; - while (amountOfChoices-- > 0) { - //weighted random draw of one element of choices - const randomNumber = Math.random() * totalWeight; - let stackWeight = choices[0].weight; - let i = 0; - while (stackWeight < randomNumber) { - stackWeight += choices[++i].weight; + //trivial case: no need to go through the whole process + if (amountOfChoices >= choices.length) { + return choices; } - //add it to the chosen ones and remove it from the choices before the next iteration - chosen.push(choices[i]); - totalWeight -= choices[i].weight; - choices.splice(i, 1); - } + //assign a weight to each choice + let totalWeight = 0; + choices = choices.map(choice => { + //The 3 makes -2 the minimum votes before being ignored completely + //https://www.desmos.com/calculator/c1duhfrmts + //this can be changed if this system increases in popularity. + const weight = Math.exp((choice.votes + 3), 0.85); + totalWeight += weight; - return chosen; + return { ...choice, weight }; + }); + + //iterate and find amountOfChoices choices + const chosen = []; + while (amountOfChoices-- > 0) { + //weighted random draw of one element of choices + const randomNumber = Math.random() * totalWeight; + let stackWeight = choices[0].weight; + let i = 0; + while (stackWeight < randomNumber) { + stackWeight += choices[++i].weight; + } + + //add it to the chosen ones and remove it from the choices before the next iteration + chosen.push(choices[i]); + totalWeight -= choices[i].weight; + choices.splice(i, 1); + } + + return chosen; } //This function will find segments that are contained inside of eachother, called similar segments @@ -110,34 +110,34 @@ function getWeightedRandomChoice(choices, amountOfChoices) { //This allows new less voted items to still sometimes appear to give them a chance at getting votes. //Segments with less than -1 votes are already ignored before this function is called function chooseSegments(segments) { - //Create groups of segments that are similar to eachother - //Segments must be sorted by their startTime so that we can build groups chronologically: - //1. As long as the segments' startTime fall inside the currentGroup, we keep adding them to that group - //2. If a segment starts after the end of the currentGroup (> cursor), no other segment will ever fall - // inside that group (because they're sorted) so we can create a new one - const similarSegmentsGroups = []; - let currentGroup; - let cursor = -1; //-1 to make sure that, even if the 1st segment starts at 0, a new group is created - segments.forEach(segment => { - if (segment.startTime > cursor) { - currentGroup = { segments: [], votes: 0 }; - similarSegmentsGroups.push(currentGroup); - } + //Create groups of segments that are similar to eachother + //Segments must be sorted by their startTime so that we can build groups chronologically: + //1. As long as the segments' startTime fall inside the currentGroup, we keep adding them to that group + //2. If a segment starts after the end of the currentGroup (> cursor), no other segment will ever fall + // inside that group (because they're sorted) so we can create a new one + const similarSegmentsGroups = []; + let currentGroup; + let cursor = -1; //-1 to make sure that, even if the 1st segment starts at 0, a new group is created + segments.forEach(segment => { + if (segment.startTime > cursor) { + currentGroup = { segments: [], votes: 0 }; + similarSegmentsGroups.push(currentGroup); + } - currentGroup.segments.push(segment); - //only if it is a positive vote, otherwise it is probably just a sponsor time with slightly wrong time - if (segment.votes > 0) { - currentGroup.votes += segment.votes; - } + currentGroup.segments.push(segment); + //only if it is a positive vote, otherwise it is probably just a sponsor time with slightly wrong time + if (segment.votes > 0) { + currentGroup.votes += segment.votes; + } - cursor = Math.max(cursor, segment.endTime); - }); + cursor = Math.max(cursor, segment.endTime); + }); - //if there are too many groups, find the best 8 - return getWeightedRandomChoice(similarSegmentsGroups, 8).map( - //randomly choose 1 good segment per group and return them - group => getWeightedRandomChoice(group.segments, 1)[0] - ); + //if there are too many groups, find the best 8 + return getWeightedRandomChoice(similarSegmentsGroups, 8).map( + //randomly choose 1 good segment per group and return them + group => getWeightedRandomChoice(group.segments, 1)[0] + ); } /** @@ -151,39 +151,39 @@ function chooseSegments(segments) { * @returns */ function handleGetSegments(req, res) { - const videoID = req.query.videoID; - // Default to sponsor - // If using params instead of JSON, only one category can be pulled - const categories = req.query.categories - ? JSON.parse(req.query.categories) - : req.query.category - ? [req.query.category] - : ['sponsor']; + const videoID = req.query.videoID; + // Default to sponsor + // If using params instead of JSON, only one category can be pulled + const categories = req.query.categories + ? JSON.parse(req.query.categories) + : req.query.category + ? [req.query.category] + : ['sponsor']; - let segments = cleanGetSegments(videoID, categories); + let segments = cleanGetSegments(videoID, categories); - if (segments === undefined) { - res.sendStatus(500); - return false; + if (segments === null || segments === undefined) { + res.sendStatus(500); + return false; } if (segments.length == 0) { - res.sendStatus(404); - return false; + res.sendStatus(404); + return false; } return segments; } module.exports = { - handleGetSegments, - cleanGetSegments, - endpoint: function (req, res) { - let segments = handleGetSegments(req, res); + handleGetSegments, + cleanGetSegments, + endpoint: function (req, res) { + let segments = handleGetSegments(req, res); - if (segments) { - //send result - res.send(segments); - } - }, + if (segments) { + //send result + res.send(segments); + } + }, }; From ac80b2322bfa904b90e79c4d289f48220b9f9c01 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Fri, 4 Sep 2020 12:16:13 -0400 Subject: [PATCH 31/40] Fix broken shadow ban check --- src/routes/getSkipSegments.js | 4 ++-- src/routes/getSkipSegmentsByHash.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/routes/getSkipSegments.js b/src/routes/getSkipSegments.js index 4bf6a6a..81ea763 100644 --- a/src/routes/getSkipSegments.js +++ b/src/routes/getSkipSegments.js @@ -8,7 +8,7 @@ var logger = require('../utils/logger.js'); var getHash = require('../utils/getHash.js'); var getIP = require('../utils/getIP.js'); -function cleanGetSegments(videoID, categories) { +function cleanGetSegments(req, videoID, categories) { let userHashedIP, shadowHiddenSegments; let segments = []; @@ -160,7 +160,7 @@ function handleGetSegments(req, res) { ? [req.query.category] : ['sponsor']; - let segments = cleanGetSegments(videoID, categories); + let segments = cleanGetSegments(req, videoID, categories); if (segments === null || segments === undefined) { res.sendStatus(500); diff --git a/src/routes/getSkipSegmentsByHash.js b/src/routes/getSkipSegmentsByHash.js index 702a28d..23a9167 100644 --- a/src/routes/getSkipSegmentsByHash.js +++ b/src/routes/getSkipSegmentsByHash.js @@ -29,7 +29,7 @@ module.exports = async function (req, res) { return { videoID: video.videoID, hash: video.hashedVideoID, - segments: getSegments(video.videoID, categories) + segments: getSegments(req, video.videoID, categories) }; }); From 8de7801662dfa2cd88afb3a08b3953f0f49f88f8 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Fri, 4 Sep 2020 12:34:05 -0400 Subject: [PATCH 32/40] Removed extra test case --- test/cases/getSegmentsByHash.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/cases/getSegmentsByHash.js b/test/cases/getSegmentsByHash.js index 6571fe4..2704b61 100644 --- a/test/cases/getSegmentsByHash.js +++ b/test/cases/getSegmentsByHash.js @@ -12,12 +12,6 @@ describe('getSegmentsByHash', () => { db.exec(startOfQuery + "('getSegmentsByHash-1', 60, 70, 2, 'getSegmentsByHash-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('getSegmentsByHash-1', 1) + "')"); // hash = 3272fa85ee0927f6073ef6f07ad5f3146047c1abba794cfa364d65ab9921692b }); - it('Should update the database version when starting the application', (done) => { - let version = db.prepare('get', 'SELECT key, value FROM config where key = ?', ['version']).value; - if (version > 2) done(); - else done('Version isn\'t greater than 2. Version is ' + version); - }); - it('Should be able to get a 200', (done) => { request.get(utils.getbaseURL() + '/api/skipSegments/3272f?categories=["sponsor", "intro"]', null, From c3de215b08242ccbf57960a6529d29c283fd76f7 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Fri, 4 Sep 2020 13:49:28 -0400 Subject: [PATCH 33/40] Add index for hashed videoID --- databases/_upgrade_sponsorTimes_3.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/databases/_upgrade_sponsorTimes_3.sql b/databases/_upgrade_sponsorTimes_3.sql index 1d2edbd..8997a3f 100644 --- a/databases/_upgrade_sponsorTimes_3.sql +++ b/databases/_upgrade_sponsorTimes_3.sql @@ -22,6 +22,8 @@ INSERT INTO sqlb_temp_table_3 SELECT *, sha256(videoID) FROM sponsorTimes; DROP TABLE sponsorTimes; ALTER TABLE sqlb_temp_table_3 RENAME TO "sponsorTimes"; +CREATE INDEX IF NOT EXISTS sponsorTimes_hashedVideoID on sponsorTimes(hashedVideoID); + /* Bump version in config */ UPDATE config SET value = 3 WHERE key = "version"; From 9d94067cd0305b19860190f390ffbf76c81ca92f Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Tue, 8 Sep 2020 17:37:29 +0100 Subject: [PATCH 34/40] Added youtube api cache (optional) --- package-lock.json | 240 ++++++++++++++++++++++++++++++++ package.json | 1 + src/app.js | 1 + src/routes/postSkipSegments.js | 15 +- src/routes/voteOnSponsorTime.js | 5 +- src/utils/redis.js | 15 ++ src/utils/youtubeAPI.js | 40 ++++++ test.js | 2 +- test/youtubeMock.js | 6 + 9 files changed, 311 insertions(+), 14 deletions(-) create mode 100644 src/utils/redis.js diff --git a/package-lock.json b/package-lock.json index d314594..0878451 100644 --- a/package-lock.json +++ b/package-lock.json @@ -68,6 +68,11 @@ "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" }, + "asap": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/asap/-/asap-1.0.0.tgz", + "integrity": "sha1-sqRdpf36ILBJb8N2jMJ8EvqRan0=" + }, "asn1": { "version": "0.2.4", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", @@ -96,12 +101,39 @@ "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.9.1.tgz", "integrity": "sha512-wMHVg2EOHaMRxbzgFJ9gtjOOCrI80OHLG14rxi28XwOW8ux6IiEbRCGGGqCtdAIg4FQCbW20k9RsT4y3gJlFug==" }, + "babel-runtime": { + "version": "6.26.0", + "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz", + "integrity": "sha1-llxwWGaOgrVde/4E/yM3vItWR/4=", + "requires": { + "core-js": "^2.4.0", + "regenerator-runtime": "^0.11.0" + } + }, "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", "dev": true }, + "barrage": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/barrage/-/barrage-1.1.0.tgz", + "integrity": "sha1-b1OEdlxGP3cYAS8WDczwM/xvw5Q=", + "requires": { + "promise": "^6.0.0" + }, + "dependencies": { + "promise": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/promise/-/promise-6.1.0.tgz", + "integrity": "sha1-LOcp9rlLRcJoka0GAsXJDgTG7vY=", + "requires": { + "asap": "~1.0.0" + } + } + } + }, "base64url": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/base64url/-/base64url-1.0.6.tgz", @@ -128,6 +160,11 @@ "tar": "^4.4.10" } }, + "bignumber.js": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.0.tgz", + "integrity": "sha512-t/OYhhJ2SD+YGBQcjY8GzzDHEk9f3nerxjtfa6tlMXfe7frs/WozhvCNoGvpM0P3bNf3Gq5ZRMlGr5f3r4/N8A==" + }, "binary-extensions": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.0.0.tgz", @@ -269,6 +306,11 @@ "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=" }, + "buffer-from": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", + "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" + }, "bytes": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", @@ -471,6 +513,11 @@ "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" }, + "core-js": { + "version": "2.6.11", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.11.tgz", + "integrity": "sha512-5wjnpaT/3dV+XB4borEsnAYQchn00XSgTAWKDkEqv+K8KevjbzmofK6hfJ9TZIlpj2N0xQpazy7PiRQiWHqzWg==" + }, "core-util-is": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", @@ -559,6 +606,11 @@ "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" }, + "denque": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/denque/-/denque-1.4.1.tgz", + "integrity": "sha512-OfzPuSZKGcgr96rf1oODnfjqBFmr1DVoc/TrItj3Ohe0Ah1C5WX5Baquw/9U9KovnQ88EqmJbD66rKYUQYN1tQ==" + }, "depd": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", @@ -877,6 +929,11 @@ "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", "dev": true }, + "get-port": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/get-port/-/get-port-3.2.0.tgz", + "integrity": "sha1-3Xzn3hh8Bsi/NTeWrHHgmfCYDrw=" + }, "get-stdin": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-4.0.1.tgz", @@ -1744,6 +1801,56 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, + "mysql": { + "version": "2.18.1", + "resolved": "https://registry.npmjs.org/mysql/-/mysql-2.18.1.tgz", + "integrity": "sha512-Bca+gk2YWmqp2Uf6k5NFEurwY/0td0cpebAucFpY/3jhrwrVGuxU2uQFCHjU19SJfje0yQvi+rVWdq78hR5lig==", + "requires": { + "bignumber.js": "9.0.0", + "readable-stream": "2.3.7", + "safe-buffer": "5.1.2", + "sqlstring": "2.3.1" + }, + "dependencies": { + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, "negotiator": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", @@ -2005,6 +2112,21 @@ "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz", "integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M=" }, + "promise": { + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/promise/-/promise-7.3.1.tgz", + "integrity": "sha512-nolQXZ/4L+bP/UGlkfaIujX9BKxGwmQ9OT4mOt5yvy8iK1h3wqTEJCijzGANTCCl9nWjY41juyAn2K3Q1hLLTg==", + "requires": { + "asap": "~2.0.3" + }, + "dependencies": { + "asap": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", + "integrity": "sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY=" + } + } + }, "proxy-addr": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz", @@ -2089,6 +2211,40 @@ "picomatch": "^2.0.4" } }, + "redis": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/redis/-/redis-3.0.2.tgz", + "integrity": "sha512-PNhLCrjU6vKVuMOyFu7oSP296mwBkcE6lrAjruBYG5LgdSqtRBoVQIylrMyVZD/lkF24RSNNatzvYag6HRBHjQ==", + "requires": { + "denque": "^1.4.1", + "redis-commands": "^1.5.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0" + } + }, + "redis-commands": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.6.0.tgz", + "integrity": "sha512-2jnZ0IkjZxvguITjFTrGiLyzQZcTvaw8DAaCXxZq/dsHXz7KfMQ3OUJy7Tz9vnRtZRVz6VRCPDvruvU8Ts44wQ==" + }, + "redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60=" + }, + "redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=", + "requires": { + "redis-errors": "^1.0.0" + } + }, + "regenerator-runtime": { + "version": "0.11.1", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz", + "integrity": "sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg==" + }, "registry-auth-token": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-3.4.0.tgz", @@ -2277,6 +2433,11 @@ "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", "dev": true }, + "sqlstring": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.1.tgz", + "integrity": "sha1-R1OT/56RR5rqYtyvDKPRSYOn+0A=" + }, "sshpk": { "version": "1.16.1", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", @@ -2414,6 +2575,75 @@ "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=" }, + "sync-mysql": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/sync-mysql/-/sync-mysql-3.0.1.tgz", + "integrity": "sha1-B6ArGSk04EzZDtU+lvCfgbFM68g=", + "requires": { + "babel-runtime": "^6.18.0", + "concat-stream": "^1.6.0", + "sync-rpc": "^1.1.1", + "then-mysql": "^1.1.1" + }, + "dependencies": { + "concat-stream": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", + "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", + "requires": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^2.2.2", + "typedarray": "^0.0.6" + } + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, + "sync-rpc": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/sync-rpc/-/sync-rpc-1.3.6.tgz", + "integrity": "sha512-J8jTXuZzRlvU7HemDgHi3pGnh/rkoqR/OZSjhTyyZrEkkYQbk7Z33AXp37mkPfPpfdOuj7Ex3H/TJM1z48uPQw==", + "requires": { + "get-port": "^3.1.0" + } + }, "tar": { "version": "4.4.13", "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.13.tgz", @@ -2437,6 +2667,16 @@ "execa": "^0.7.0" } }, + "then-mysql": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/then-mysql/-/then-mysql-1.1.1.tgz", + "integrity": "sha1-p1Q2zGXyvm37t3+HIg6YKuDjYR0=", + "requires": { + "barrage": "^1.1.0", + "mysql": "^2.10.0", + "promise": "^7.1.1" + } + }, "timed-out": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/timed-out/-/timed-out-4.0.1.tgz", diff --git a/package.json b/package.json index 9c4e7b5..59acb78 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "express": "^4.17.1", "http": "0.0.0", "iso8601-duration": "^1.2.0", + "redis": "^3.0.2", "sync-mysql": "^3.0.1", "uuid": "^3.3.2", "youtube-api": "^2.0.10" diff --git a/src/app.js b/src/app.js index bf4dce8..a102a76 100644 --- a/src/app.js +++ b/src/app.js @@ -2,6 +2,7 @@ var express = require('express'); // Create a service (the app object is just a callback). var app = express(); var config = require('./config.js'); +var redis = require('./utils/redis.js'); // Middleware var corsMiddleware = require('./middleware/cors.js'); diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 48fbb76..f135495 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -48,10 +48,7 @@ function sendWebhooks(userID, videoID, UUID, segmentInfo) { if (config.youtubeAPIKey !== null) { let userSubmissionCountRow = db.prepare('get', "SELECT count(*) as submissionCount FROM sponsorTimes WHERE userID = ?", [userID]); - YouTubeAPI.videos.list({ - part: "snippet", - id: videoID - }, function (err, data) { + YouTubeAPI.listVideos(videoID, "snippet", (err, data) => { if (err || data.items.length === 0) { err && logger.error(err); return; @@ -108,10 +105,7 @@ async function autoModerateSubmission(submission, callback) { // Get the video information from the youtube API if (config.youtubeAPIKey !== null) { let {err, data} = await new Promise((resolve, reject) => { - YouTubeAPI.videos.list({ - part: "contentDetails", - id: submission.videoID - }, (err, data) => resolve({err, data})); + YouTubeAPI.listVideos(submission.videoID, "contentDetails", (err, data) => resolve({err, data})); }); if (err) { @@ -134,7 +128,6 @@ async function autoModerateSubmission(submission, callback) { } } } - } else { logger.debug("Skipped YouTube API"); @@ -214,6 +207,10 @@ module.exports = async function postSkipSegments(req, res) { if (isNaN(startTime) || isNaN(endTime) || startTime === Infinity || endTime === Infinity || startTime < 0 || startTime >= endTime) { //invalid request + logger.debug(videoID); + logger.debug(userID); + logger.debug(JSON.stringify(segments)); + logger.debug('400'); res.sendStatus(400); return; } diff --git a/src/routes/voteOnSponsorTime.js b/src/routes/voteOnSponsorTime.js index 5ebdc6d..43fb63b 100644 --- a/src/routes/voteOnSponsorTime.js +++ b/src/routes/voteOnSponsorTime.js @@ -48,10 +48,7 @@ function sendWebhooks(voteData) { } if (config.youtubeAPIKey !== null) { - YouTubeAPI.videos.list({ - part: "snippet", - id: submissionInfoRow.videoID - }, function (err, data) { + YouTubeAPI.listVideos(submissionInfoRow.videoID, "snippet", (err, data) => { if (err || data.items.length === 0) { err && logger.error(err); return; diff --git a/src/utils/redis.js b/src/utils/redis.js new file mode 100644 index 0000000..2d822c6 --- /dev/null +++ b/src/utils/redis.js @@ -0,0 +1,15 @@ +const config = require('../config.js'); +const logger = require('./logger.js'); + +if (config.redis) { + const redis = require('redis'); + logger.info('Connected to redis'); + const client = redis.createClient(config.redis); + module.exports = client; +} else { + module.exports = { + get: (key, callback) => { + callback((true)); + } + }; +} diff --git a/src/utils/youtubeAPI.js b/src/utils/youtubeAPI.js index c28f882..c1a47d6 100644 --- a/src/utils/youtubeAPI.js +++ b/src/utils/youtubeAPI.js @@ -2,6 +2,8 @@ var config = require('../config.js'); // YouTube API const YouTubeAPI = require("youtube-api"); +const redis = require('./redis.js'); +const logger = require('./logger.js'); var exportObject; // If in test mode, return a mocked youtube object @@ -14,6 +16,44 @@ if (config.mode === "test") { key: config.youtubeAPIKey }); exportObject = YouTubeAPI; + + // YouTubeAPI.videos.list wrapper with cacheing + exportObject.listVideos = (videoID, part, callback) => { + let redisKey = "youtube.video." + videoID + "." + part; + redis.get(redisKey, (getErr, result) => { + if (getErr || !result) { + logger.debug("redis: no cache for video information: " + videoID); + YouTubeAPI.videos.list({ + part, + id: videoID + }, (ytErr, data) => { + if (!ytErr) { + // Only set cache if data returned + if (data.items.length > 0) { + redis.set(redisKey, JSON.stringify(data), (setErr) => { + logger.debug("redis: video information cache set for: " + videoID); + setErr && logger.warn(setErr); + callback(false, data); // don't fail + }); + } else { + callback(false, data); // don't fail + } + } else { + callback(ytErr, data) + } + }); + } else { + logger.debug("redis: fetched video information from cache: " + videoID); + callback(getErr, JSON.parse(result)); + } + }); + }; } +/* YouTubeAPI.videos.list({ + part: "snippet", + id: videoID + }, function (err, data) {*/ + + module.exports = exportObject; \ No newline at end of file diff --git a/test.js b/test.js index 9810786..f4e54ff 100644 --- a/test.js +++ b/test.js @@ -32,7 +32,7 @@ var mockServer = createMockServer(() => { var server = createServer(() => { logger.info("Started main HTTP server"); // Run the tests. - mocha.run(function(failures) { + mocha.run((failures) => { mockServer.close(); server.close(); process.exitCode = failures ? 1 : 0; // exit with non-zero status if there were failures diff --git a/test/youtubeMock.js b/test/youtubeMock.js index dd64898..cc722d9 100644 --- a/test/youtubeMock.js +++ b/test/youtubeMock.js @@ -8,6 +8,12 @@ YouTubeAPI.videos.list({ // https://developers.google.com/youtube/v3/docs/videos const YouTubeAPI = { + listVideos: (id, part, callback) => { + YouTubeAPI.videos.list({ + part: part, + id: id + }, callback); + }, videos: { list: (obj, callback) => { if (obj.id === "knownWrongID") { From b7ea0fa681f282182aaf6bdd908fbc09f82aaee6 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Tue, 8 Sep 2020 17:39:53 +0100 Subject: [PATCH 35/40] remove logging --- src/routes/postSkipSegments.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index f135495..3d3b5b5 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -207,10 +207,6 @@ module.exports = async function postSkipSegments(req, res) { if (isNaN(startTime) || isNaN(endTime) || startTime === Infinity || endTime === Infinity || startTime < 0 || startTime >= endTime) { //invalid request - logger.debug(videoID); - logger.debug(userID); - logger.debug(JSON.stringify(segments)); - logger.debug('400'); res.sendStatus(400); return; } From bbe31149b4dbd553ece8b1d9a0a24e2bf0ef17c2 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Tue, 8 Sep 2020 17:43:51 +0100 Subject: [PATCH 36/40] cleaner logging --- src/utils/youtubeAPI.js | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/utils/youtubeAPI.js b/src/utils/youtubeAPI.js index c1a47d6..a053eb1 100644 --- a/src/utils/youtubeAPI.js +++ b/src/utils/youtubeAPI.js @@ -31,8 +31,11 @@ if (config.mode === "test") { // Only set cache if data returned if (data.items.length > 0) { redis.set(redisKey, JSON.stringify(data), (setErr) => { - logger.debug("redis: video information cache set for: " + videoID); - setErr && logger.warn(setErr); + if(setErr) { + logger.warn(setErr); + } else { + logger.debug("redis: video information cache set for: " + videoID); + } callback(false, data); // don't fail }); } else { @@ -50,10 +53,4 @@ if (config.mode === "test") { }; } -/* YouTubeAPI.videos.list({ - part: "snippet", - id: videoID - }, function (err, data) {*/ - - module.exports = exportObject; \ No newline at end of file From 0611ad54d03f0e13c21b0f13fb7ef50e67ace359 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Wed, 9 Sep 2020 23:14:20 +0100 Subject: [PATCH 37/40] fixed db upgrade filesize inflation --- databases/_upgrade_sponsorTimes_3.sql | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/databases/_upgrade_sponsorTimes_3.sql b/databases/_upgrade_sponsorTimes_3.sql index 1d2edbd..adae777 100644 --- a/databases/_upgrade_sponsorTimes_3.sql +++ b/databases/_upgrade_sponsorTimes_3.sql @@ -1,26 +1,9 @@ BEGIN TRANSACTION; +/* hash upgrade test sha256('vid') = '1ff838dc6ca9680d88455341118157d59a055fe6d0e3870f9c002847bebe4663' /* Add hash field */ -CREATE TABLE "sqlb_temp_table_3" ( - "videoID" TEXT NOT NULL, - "startTime" REAL NOT NULL, - "endTime" REAL NOT NULL, - "votes" INTEGER NOT NULL, - "incorrectVotes" INTEGER NOT NULL default '1', - "UUID" TEXT NOT NULL UNIQUE, - "userID" TEXT NOT NULL, - "timeSubmitted" INTEGER NOT NULL, - "views" INTEGER NOT NULL, - "category" TEXT NOT NULL DEFAULT "sponsor", - "shadowHidden" INTEGER NOT NULL, - "hashedVideoID" TEXT NOT NULL -); - -/* hash upgade test sha256('vid') = '1ff838dc6ca9680d88455341118157d59a055fe6d0e3870f9c002847bebe4663' */ -INSERT INTO sqlb_temp_table_3 SELECT *, sha256(videoID) FROM sponsorTimes; - -DROP TABLE sponsorTimes; -ALTER TABLE sqlb_temp_table_3 RENAME TO "sponsorTimes"; +ALTER TABLE sponsorTimes ADD hashedVideoID TEXT NOT NULL default ""; +UPDATE sponsorTimes SET hashedVideoID = sha256(videoID); /* Bump version in config */ UPDATE config SET value = 3 WHERE key = "version"; From eaaf3b881215a46cd71af24bfa675e126fcc34a9 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Tue, 15 Sep 2020 12:15:23 -0400 Subject: [PATCH 38/40] Add pollyfill for redis set --- src/utils/redis.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/utils/redis.js b/src/utils/redis.js index 2d822c6..d1ccb25 100644 --- a/src/utils/redis.js +++ b/src/utils/redis.js @@ -10,6 +10,9 @@ if (config.redis) { module.exports = { get: (key, callback) => { callback((true)); + }, + set: (key, value, callback) => { + callback((true)); } }; } From a86dc0fc7b2ee1609ff309294d8c23a3cf552985 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Tue, 15 Sep 2020 12:16:34 -0400 Subject: [PATCH 39/40] Changed redis polyfill to not throw an error --- src/utils/redis.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/redis.js b/src/utils/redis.js index d1ccb25..b278b6f 100644 --- a/src/utils/redis.js +++ b/src/utils/redis.js @@ -9,10 +9,10 @@ if (config.redis) { } else { module.exports = { get: (key, callback) => { - callback((true)); + callback(false); }, set: (key, value, callback) => { - callback((true)); + callback(false); } }; } From be86f439470c6c0353cc2928d9d2b5799ce1248f Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Tue, 15 Sep 2020 12:22:28 -0400 Subject: [PATCH 40/40] Added default options --- src/config.js | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/src/config.js b/src/config.js index 1bc3721..d85b930 100644 --- a/src/config.js +++ b/src/config.js @@ -1,6 +1,6 @@ -var fs = require('fs'); -var config = undefined; +const fs = require('fs'); +let config = {}; // Check to see if launched in test mode if (process.env.npm_lifecycle_script === 'node test.js') { @@ -9,4 +9,27 @@ if (process.env.npm_lifecycle_script === 'node test.js') { config = JSON.parse(fs.readFileSync('config.json')); } -module.exports = config; \ No newline at end of file +addDefaults(config, { + "port": 80, + "behindProxy": "X-Forwarded-For", + "db": "./databases/sponsorTimes.db", + "privateDB": "./databases/private.db", + "createDatabaseIfNotExist": true, + "schemaFolder": "./databases", + "dbSchema": "./databases/_sponsorTimes.db.sql", + "privateDBSchema": "./databases/_private.db.sql", + "readOnly": false, + "webhooks": [], + "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"] +}) + +module.exports = config; + +// Add defaults +function addDefaults(config, defaults) { + for (const key in defaults) { + if(!config.hasOwnProperty(key)) { + config[key] = defaults[key]; + } + } +}; \ No newline at end of file