external/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2024-11-10 01:02:30 +01:00
Code Issues Projects Releases Packages Wiki Activity

Merge branch 'master' into nb-mod-fetch

Browse source
This commit is contained in:
Ajay Ramachandran 2020-09-11 17:32:14 -04:00 committed by GitHub
commit 0c14560eaa
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 145 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
config.json.example
View file

@ -10,6 +10,7 @@
"discordCompletelyIncorrectReportWebhookURL": null, //URL from discord if you would like notifications when someone reports a submission as completely incorrect [optional]
"neuralBlockURL": null, // URL to check submissions against neural block. Ex. https://ai.neuralblock.app
"discordNeuralBlockRejectWebhookURL": null, //URL from discord if you would like notifications when NeuralBlock rejects a submission [optional]
"userCounterURL": null, // For user counting. URL to instance of https://github.com/ajayyy/PrivacyUserCount
"proxySubmission": null, // Base url to proxy submissions to persist // e.g. https://sponsor.ajay.app (no trailing slash)
"behindProxy": "X-Forwarded-For", //Options: "X-Forwarded-For", "Cloudflare", "X-Real-IP", anything else will mean it is not behind a proxy. True defaults to "X-Forwarded-For"
"db": "./databases/sponsorTimes.db",
@ -20,5 +21,6 @@
"privateDBSchema": "./databases/_private.db.sql",
"mode": "development",
"readOnly": false,
"webhooks": []
"webhooks": [],
"categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"] // List of supported categories any other category will be rejected
}

3
src/app.js
View file

@ -6,6 +6,7 @@ var config = require('./config.js');
// Middleware
var corsMiddleware = require('./middleware/cors.js');
var loggerMiddleware = require('./middleware/logger.js');
const userCounter = require('./middleware/userCounter.js');
// Routes
var getSkipSegments = require('./routes/getSkipSegments.js').endpoint;
@ -31,6 +32,8 @@ app.use(corsMiddleware);
app.use(loggerMiddleware);
app.use(express.json())
if (config.userCounterURL) app.use(userCounter);
// Setup pretty JSON
if (config.mode === "development") app.set('json spaces', 2);

16
src/middleware/userCounter.js Normal file
View file

@ -0,0 +1,16 @@
var request = require('request');
const config = require('../config.js');
const getIP = require('../utils/getIP.js');
const getHash = require('../utils/getHash.js');
const logger = require('../utils/logger.js');
module.exports = function userCounter(req, res, next) {
try {
request.post(config.userCounterURL + "/api/v1/addIP?hashedIP=" + getHash(getIP(req), 1));
} catch(e) {
logger.debug("Failing to connect to user counter at: " + config.userCounterURL);
}
next();
}

100
src/routes/getTotalStats.js
View file

@ -1,53 +1,69 @@
var db = require('../databases/databases.js').db;
var request = require('request');
const db = require('../databases/databases.js').db;
const request = require('request');
const config = require('../config.js');
// A cache of the number of chrome web store users
var chromeUsersCache = null;
var firefoxUsersCache = null;
var lastUserCountCheck = 0;
let chromeUsersCache = null;
let firefoxUsersCache = null;
// By the privacy friendly user counter
let apiUsersCache = null;
let lastUserCountCheck = 0;
module.exports = function getTotalStats (req, res) {
let row = db.prepare('get', "SELECT COUNT(DISTINCT userID) as userCount, COUNT(*) as totalSubmissions, " +
let row = db.prepare('get', "SELECT COUNT(DISTINCT userID) as userCount, COUNT(*) as totalSubmissions, " +
"SUM(views) as viewCount, SUM((endTime - startTime) / 60 * views) as minutesSaved FROM sponsorTimes WHERE shadowHidden != 1 AND votes >= 0", []);
if (row !== undefined) {
//send this result
res.send({
userCount: row.userCount,
activeUsers: chromeUsersCache + firefoxUsersCache,
viewCount: row.viewCount,
totalSubmissions: row.totalSubmissions,
minutesSaved: row.minutesSaved
});
if (row !== undefined) {
let extensionUsers = chromeUsersCache + firefoxUsersCache;
// Check if the cache should be updated (every ~14 hours)
let now = Date.now();
if (now - lastUserCountCheck > 5000000) {
lastUserCountCheck = now;
//send this result
res.send({
userCount: row.userCount,
activeUsers: extensionUsers,
apiUsers: Math.max(apiUsersCache, extensionUsers),
viewCount: row.viewCount,
totalSubmissions: row.totalSubmissions,
minutesSaved: row.minutesSaved
});
// Get total users
request.get("https://addons.mozilla.org/api/v3/addons/addon/sponsorblock/", function (err, firefoxResponse, body) {
try {
firefoxUsersCache = parseInt(JSON.parse(body).average_daily_users);
// Check if the cache should be updated (every ~14 hours)
let now = Date.now();
if (now - lastUserCountCheck > 5000000) {
lastUserCountCheck = now;
request.get("https://chrome.google.com/webstore/detail/sponsorblock-for-youtube/mnjggcdmjocbbbhaepdhchncahnbgone", function(err, chromeResponse, body) {
if (body !== undefined) {
try {
chromeUsersCache = parseInt(body.match(/(?<=\<span class=\"e-f-ih\" title=\").*?(?= users\">)/)[0].replace(",", ""));
} catch (error) {
// Re-check later
lastUserCountCheck = 0;
}
} else {
lastUserCountCheck = 0;
}
});
} catch (error) {
// Re-check later
lastUserCountCheck = 0;
}
});
}
}
updateExtensionUsers();
}
}
}
function updateExtensionUsers() {
if (config.userCounterURL) {
request.get(config.userCounterURL + "/api/v1/userCount", (err, response, body) => {
apiUsersCache = Math.max(apiUsersCache, JSON.parse(body).userCount);
});
}
request.get("https://addons.mozilla.org/api/v3/addons/addon/sponsorblock/", function (err, firefoxResponse, body) {
try {
firefoxUsersCache = parseInt(JSON.parse(body).average_daily_users);
request.get("https://chrome.google.com/webstore/detail/sponsorblock-for-youtube/mnjggcdmjocbbbhaepdhchncahnbgone", function(err, chromeResponse, body) {
if (body !== undefined) {
try {
chromeUsersCache = parseInt(body.match(/(?<=\<span class=\"e-f-ih\" title=\").*?(?= users\">)/)[0].replace(",", ""));
} catch (error) {
// Re-check later
lastUserCountCheck = 0;
}
} else {
lastUserCountCheck = 0;
}
});
} catch (error) {
// Re-check later
lastUserCountCheck = 0;
}
});
}

15
src/routes/postSkipSegments.js
View file

@ -64,7 +64,7 @@ function sendWebhooks(userID, videoID, UUID, segmentInfo) {
// If it is a first time submission
// Then send a notification to discord
if (config.discordFirstTimeSubmissionsWebhookURL === null) return;
if (config.discordFirstTimeSubmissionsWebhookURL === null || userSubmissionCountRow.submissionCount > 1) return;
request.post(config.discordFirstTimeSubmissionsWebhookURL, {
json: {
"embeds": [{
@ -163,7 +163,7 @@ async function autoModerateSubmission(submission) {
});
if (err) {
return "Couldn't get video information.";
return false;
} else {
// Check to see if video exists
if (data.pageInfo.totalResults === 0) {
@ -267,7 +267,7 @@ module.exports = async function postSkipSegments(req, res) {
//check if all correct inputs are here and the length is 1 second or more
if (videoID == undefined || userID == undefined || segments == undefined || segments.length < 1) {
//invalid request
res.sendStatus(400);
res.status(400).send("Parameters are not valid");
return;
}
@ -286,9 +286,14 @@ module.exports = async function postSkipSegments(req, res) {
for (let i = 0; i < segments.length; i++) {
if (segments[i] === undefined || segments[i].segment === undefined || segments[i].category === undefined) {
//invalid request
res.sendStatus(400);
res.status(400).send("One of your segments are invalid");
return;
}
if (!config.categoryList.includes(segments[i].category)) {
res.status("400").send("Category doesn't exist.");
return;
}
let startTime = parseFloat(segments[i].segment[0]);
let endTime = parseFloat(segments[i].segment[1]);
@ -296,7 +301,7 @@ module.exports = async function postSkipSegments(req, res) {
if (isNaN(startTime) || isNaN(endTime)
|| startTime === Infinity || endTime === Infinity || startTime < 0 || startTime >= endTime) {
//invalid request
res.sendStatus(400);
res.status(400).send("One of your segments times are invalid (too short, startTime before endTime, etc.)");
return;
}

2
src/routes/setUsername.js
View file

@ -12,7 +12,7 @@ module.exports = function setUsername(req, res) {
let adminUserIDInput = req.query.adminUserID;
if (userID == undefined || userName == undefined || userID === "undefined" || username.length > 50) {
if (userID == undefined || userName == undefined || userID === "undefined" || userName.length > 50) {
//invalid request
res.sendStatus(400);
return;

5
src/routes/shadowBanUser.js
View file

@ -1,5 +1,3 @@
var config = require('../config.js');
var databases = require('../databases/databases.js');
var db = databases.db;
var privateDB = databases.privateDB;
@ -30,7 +28,8 @@ module.exports = async function shadowBanUser(req, res) {
//hash the userID
adminUserIDInput = getHash(adminUserIDInput);
if (adminUserIDInput !== config.adminUserID) {
let isVIP = db.prepare("get", "SELECT count(*) as userCount FROM vipUsers WHERE userID = ?", [adminUserIDInput]).userCount > 0;
if (!isVIP) {
//not authorized
res.sendStatus(403);
return;

5
src/routes/voteOnSponsorTime.js
View file

@ -150,6 +150,11 @@ function categoryVote(UUID, userID, isVIP, category, hashedIP, res) {
res.status("400").send("Submission doesn't exist.");
return;
}
if (!config.categoryList.includes(category)) {
res.status("400").send("Category doesn't exist.");
return;
}
let timeSubmitted = Date.now();

4
src/utils/webhookUtils.js
View file

@ -17,10 +17,10 @@ function getVoteAuthorRaw(submissionCount, isVIP, isOwnSubmission) {
function getVoteAuthor(submissionCount, isVIP, isOwnSubmission) {
if (submissionCount === 0) {
return "Report by New User";
} else if (isVIP) {
return "Report by VIP User";
} else if (isOwnSubmission) {
return "Report by Submitter";
} else if (isVIP) {
return "Report by VIP User";
}
return "";

3
test.json
View file

@ -48,5 +48,6 @@
"vote.down"
]
}
]
],
"categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"]
}

43
test/cases/voteOnSponsorTime.js
View file

@ -22,6 +22,8 @@ describe('voteOnSponsorTime', () => {
db.exec(startOfQuery + "('voter-submitter2', 1, 11, 2, 'vote-uuid-11', '" + getHash("randomID4") + "', 0, 50, 'sponsor', 0)");
db.exec(startOfQuery + "('own-submission-video', 1, 11, 500, 'own-submission-uuid', '"+ getHash('own-submission-id') +"', 0, 50, 'sponsor', 0)");
db.exec(startOfQuery + "('not-own-submission-video', 1, 11, 500, 'not-own-submission-uuid', '"+ getHash('somebody-else-id') +"', 0, 50, 'sponsor', 0)");
db.exec(startOfQuery + "('incorrect-category', 1, 11, 500, 'incorrect-category', '"+ getHash('somebody-else-id') +"', 0, 50, 'sponsor', 0)");
db.exec(startOfQuery + "('incorrect-category-change', 1, 11, 500, 'incorrect-category-change', '"+ getHash('somebody-else-id') +"', 0, 50, 'sponsor', 0)");
db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("VIPUser") + "')");
privateDB.exec("INSERT INTO shadowBannedUsers (userID) VALUES ('" + getHash("randomID4") + "')");
@ -207,6 +209,24 @@ describe('voteOnSponsorTime', () => {
});
});
it('Should not able to change to an invalid category', (done) => {
request.get(utils.getbaseURL()
+ "/api/voteOnSponsorTime?userID=randomID2&UUID=incorrect-category&category=fakecategory", null,
(err, res, body) => {
if (err) done(err);
else if (res.statusCode === 400) {
let row = db.prepare('get', "SELECT category FROM sponsorTimes WHERE UUID = ?", ["incorrect-category"]);
if (row.category === "sponsor") {
done()
} else {
done("Vote did not succeed. Submission went from sponsor to " + row.category);
}
} else {
done("Status code was " + res.statusCode);
}
});
});
it('Should be able to change your vote for a category and it should immediately change (for now)', (done) => {
request.get(utils.getbaseURL()
+ "/api/voteOnSponsorTime?userID=randomID2&UUID=vote-uuid-4&category=outro", null,
@ -225,6 +245,29 @@ describe('voteOnSponsorTime', () => {
});
});
it('Should not be able to change your vote to an invalid category', (done) => {
const vote = (inputCat, assertCat, callback) => {
request.get(utils.getbaseURL()
+ "/api/voteOnSponsorTime?userID=randomID2&UUID=incorrect-category-change&category="+inputCat, null,
(err) => {
if (err) done(err);
else{
let row = db.prepare('get', "SELECT category FROM sponsorTimes WHERE UUID = ?", ["incorrect-category-change"]);
if (row.category === assertCat) {
callback();
} else {
done("Vote did not succeed. Submission went from sponsor to " + row.category);
}
}
});
};
vote("sponsor", "sponsor", () => {
vote("fakeCategory", "sponsor", done);
});
});
it('VIP should be able to vote for a category and it should immediately change', (done) => {
request.get(utils.getbaseURL()
+ "/api/voteOnSponsorTime?userID=VIPUser&UUID=vote-uuid-5&category=outro", null,