external/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2024-09-20 04:54:00 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #204 from ajayyy/segment-locking

Browse source 
Replace VIP starting with 1000 votes with locked submissions
This commit is contained in:
Ajay Ramachandran 2021-02-18 20:50:13 -05:00 committed by GitHub
commit c4c596bbf4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 105 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/app.ts
View file

@ -18,7 +18,7 @@ import {shadowBanUser} from './routes/shadowBanUser';
import {getUsername} from './routes/getUsername';
import {setUsername} from './routes/setUsername';
import {viewedVideoSponsorTime} from './routes/viewedVideoSponsorTime';
import {voteOnSponsorTime} from './routes/voteOnSponsorTime';
import {voteOnSponsorTime, getUserID as voteGetUserID} from './routes/voteOnSponsorTime';
import {getSkipSegmentsByHash} from './routes/getSkipSegmentsByHash';
import {postSkipSegments} from './routes/postSkipSegments';
import {endpoint as getSkipSegments} from './routes/getSkipSegments';
@ -55,7 +55,7 @@ function setupRoutes(app: Express) {
const voteEndpoints: RequestHandler[] = [voteOnSponsorTime];
const viewEndpoints: RequestHandler[] = [viewedVideoSponsorTime];
if (config.rateLimit) {
if (config.rateLimit.vote) voteEndpoints.unshift(rateLimitMiddleware(config.rateLimit.vote));
if (config.rateLimit.vote) voteEndpoints.unshift(rateLimitMiddleware(config.rateLimit.vote, voteGetUserID));
if (config.rateLimit.view) viewEndpoints.unshift(rateLimitMiddleware(config.rateLimit.view));
}

12
src/middleware/requestRateLimit.ts
View file

@ -2,8 +2,11 @@ import {getIP} from '../utils/getIP';
import {getHash} from '../utils/getHash';
import rateLimit from 'express-rate-limit';
import {RateLimitConfig} from '../types/config.model';
import {Request} from 'express';
import { isUserVIP } from '../utils/isUserVIP';
import { UserID } from '../types/user.model';
export function rateLimitMiddleware(limitConfig: RateLimitConfig): rateLimit.RateLimit {
export function rateLimitMiddleware(limitConfig: RateLimitConfig, getUserID?: (req: Request) => UserID): rateLimit.RateLimit {
return rateLimit({
windowMs: limitConfig.windowMs,
max: limitConfig.max,
@ -13,5 +16,12 @@ export function rateLimitMiddleware(limitConfig: RateLimitConfig): rateLimit.Rat
keyGenerator: (req) => {
return getHash(getIP(req), 1);
},
handler: (req, res, next) => {
if (getUserID === undefined || !isUserVIP(getHash(getUserID(req)))) {
return res.status(limitConfig.statusCode).send(limitConfig.message);
} else {
return next();
}
}
});
}

16
src/routes/getSkipSegments.ts
View file

@ -50,7 +50,7 @@ function getSegmentsByVideoID(req: Request, videoID: string, categories: Categor
const segmentsByCategory: SBRecord<Category, DBSegment[]> = db
.prepare(
'all',
`SELECT startTime, endTime, votes, UUID, category, shadowHidden FROM sponsorTimes WHERE videoID = ? AND category IN (${Array(categories.length).fill('?').join()}) ORDER BY startTime`,
`SELECT startTime, endTime, votes, locked, UUID, category, shadowHidden FROM sponsorTimes WHERE videoID = ? AND category IN (${Array(categories.length).fill('?').join()}) ORDER BY startTime`,
[videoID, categories]
).reduce((acc: SBRecord<Category, DBSegment[]>, segment: DBSegment) => {
acc[segment.category] = acc[segment.category] || [];
@ -82,7 +82,7 @@ function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash, categ
const segmentPerVideoID: SegmentWithHashPerVideoID = db
.prepare(
'all',
`SELECT videoID, startTime, endTime, votes, UUID, category, shadowHidden, hashedVideoID FROM sponsorTimes WHERE hashedVideoID LIKE ? AND category IN (${Array(categories.length).fill('?').join()}) ORDER BY startTime`,
`SELECT videoID, startTime, endTime, votes, locked, UUID, category, shadowHidden, hashedVideoID FROM sponsorTimes WHERE hashedVideoID LIKE ? AND category IN (${Array(categories.length).fill('?').join()}) ORDER BY startTime`,
[hashedVideoIDPrefix + '%', categories]
).reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => {
acc[segment.videoID] = acc[segment.videoID] || {
@ -176,7 +176,7 @@ function chooseSegments(segments: DBSegment[]): DBSegment[] {
let cursor = -1; //-1 to make sure that, even if the 1st segment starts at 0, a new group is created
segments.forEach(segment => {
if (segment.startTime > cursor) {
currentGroup = {segments: [], votes: 0};
currentGroup = {segments: [], votes: 0, locked: false};
overlappingSegmentsGroups.push(currentGroup);
}
@ -186,9 +186,19 @@ function chooseSegments(segments: DBSegment[]): DBSegment[] {
currentGroup.votes += segment.votes;
}
if (segment.locked) {
currentGroup.locked = true;
}
cursor = Math.max(cursor, segment.endTime);
});
overlappingSegmentsGroups.forEach((group) => {
if (group.locked) {
group.segments = group.segments.filter((segment) => segment.locked);
}
});
//if there are too many groups, find the best 8
return getWeightedRandomChoice(overlappingSegmentsGroups, 32).map(
//randomly choose 1 good segment per group and return them

11
src/routes/postSkipSegments.ts
View file

@ -433,10 +433,6 @@ export async function postSkipSegments(req: Request, res: Response) {
}
let startingVotes = 0 + decreaseVotes;
if (isVIP) {
//this user is a vip, start them at a higher approval rating
startingVotes = 10000;
}
if (config.youtubeAPIKey !== null) {
let {err, data} = await new Promise((resolve) => {
@ -489,11 +485,12 @@ export async function postSkipSegments(req: Request, res: Response) {
//also better for duplication checking
const UUID = getSubmissionUUID(videoID, segmentInfo.category, userID, segmentInfo.segment[0], segmentInfo.segment[1]);
const startingLocked = isVIP ? 1 : 0;
try {
db.prepare('run', "INSERT INTO sponsorTimes " +
"(videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID)" +
"VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", [
videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, UUID, userID, timeSubmitted, 0, segmentInfo.category, shadowBanned, getHash(videoID, 1),
"(videoID, startTime, endTime, votes, locked, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID)" +
"VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", [
videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, shadowBanned, getHash(videoID, 1),
],
);

21
src/routes/voteOnSponsorTime.ts
View file

@ -10,6 +10,7 @@ import {getFormattedTime} from '../utils/getFormattedTime';
import {getIP} from '../utils/getIP';
import {getHash} from '../utils/getHash';
import {config} from '../config';
import { UserID } from '../types/user.model';
const voteTypes = {
normal: 0,
@ -214,21 +215,25 @@ function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnSubmiss
res.sendStatus(200);
}
async function voteOnSponsorTime(req: Request, res: Response) {
export function getUserID(req: Request): UserID {
return req.query.userID as UserID;
}
export async function voteOnSponsorTime(req: Request, res: Response) {
const UUID = req.query.UUID as string;
let userID = req.query.userID as string;
const paramUserID = getUserID(req);
let type = req.query.type !== undefined ? parseInt(req.query.type as string) : undefined;
const category = req.query.category as string;
if (UUID === undefined || userID === undefined || (type === undefined && category === undefined)) {
if (UUID === undefined || paramUserID === undefined || (type === undefined && category === undefined)) {
//invalid request
res.sendStatus(400);
return;
}
//hash the userID
const nonAnonUserID = getHash(userID);
userID = getHash(userID + UUID);
const nonAnonUserID = getHash(paramUserID);
const userID = getHash(paramUserID + UUID);
//x-forwarded-for if this server is behind a proxy
const ip = getIP(req);
@ -421,8 +426,4 @@ async function voteOnSponsorTime(req: Request, res: Response) {
res.status(500).json({error: 'Internal error creating segment vote'});
}
}
export {
voteOnSponsorTime,
};
}

2
src/types/segments.model.ts
View file

@ -25,6 +25,7 @@ export interface DBSegment {
endTime: number;
UUID: SegmentUUID;
votes: number;
locked: boolean;
shadowHidden: Visibility;
videoID: VideoID;
hashedVideoID: VideoIDHash;
@ -33,6 +34,7 @@ export interface DBSegment {
export interface OverlappingSegmentGroup {
segments: DBSegment[],
votes: number;
locked: boolean; // Contains a locked segment
}
export interface VotableObject {

35
test/cases/getSkipSegments.ts
View file

@ -5,14 +5,16 @@ import {getHash} from '../../src/utils/getHash';
describe('getSkipSegments', () => {
before(() => {
let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES";
db.exec(startOfQuery + "('testtesttest', 1, 11, 2, '1-uuid-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('testtesttest', 1) + "')");
db.exec(startOfQuery + "('testtesttest', 20, 33, 2, '1-uuid-2', 'testman', 0, 50, 'intro', 0, '" + getHash('testtesttest', 1) + "')");
db.exec(startOfQuery + "('testtesttest,test', 1, 11, 2, '1-uuid-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('testtesttest,test', 1) + "')");
db.exec(startOfQuery + "('test3', 1, 11, 2, '1-uuid-4', 'testman', 0, 50, 'sponsor', 0, '" + getHash('test3', 1) + "')");
db.exec(startOfQuery + "('test3', 7, 22, -3, '1-uuid-5', 'testman', 0, 50, 'sponsor', 0, '" + getHash('test3', 1) + "')");
db.exec(startOfQuery + "('multiple', 1, 11, 2, '1-uuid-6', 'testman', 0, 50, 'intro', 0, '" + getHash('multiple', 1) + "')");
db.exec(startOfQuery + "('multiple', 20, 33, 2, '1-uuid-7', 'testman', 0, 50, 'intro', 0, '" + getHash('multiple', 1) + "')");
let startOfQuery = "INSERT INTO sponsorTimes (videoID, startTime, endTime, votes, locked, UUID, userID, timeSubmitted, views, category, shadowHidden, hashedVideoID) VALUES";
db.exec(startOfQuery + "('testtesttest', 1, 11, 2, 0, '1-uuid-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('testtesttest', 1) + "')");
db.exec(startOfQuery + "('testtesttest', 20, 33, 2, 0, '1-uuid-2', 'testman', 0, 50, 'intro', 0, '" + getHash('testtesttest', 1) + "')");
db.exec(startOfQuery + "('testtesttest,test', 1, 11, 2, 0, '1-uuid-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('testtesttest,test', 1) + "')");
db.exec(startOfQuery + "('test3', 1, 11, 2, 0, '1-uuid-4', 'testman', 0, 50, 'sponsor', 0, '" + getHash('test3', 1) + "')");
db.exec(startOfQuery + "('test3', 7, 22, -3, 0, '1-uuid-5', 'testman', 0, 50, 'sponsor', 0, '" + getHash('test3', 1) + "')");
db.exec(startOfQuery + "('multiple', 1, 11, 2, 0, '1-uuid-6', 'testman', 0, 50, 'intro', 0, '" + getHash('multiple', 1) + "')");
db.exec(startOfQuery + "('multiple', 20, 33, 2, 0, '1-uuid-7', 'testman', 0, 50, 'intro', 0, '" + getHash('multiple', 1) + "')");
db.exec(startOfQuery + "('locked', 20, 33, 2, 1, '1-uuid-locked-8', 'testman', 0, 50, 'intro', 0, '" + getHash('locked', 1) + "')");
db.exec(startOfQuery + "('locked', 20, 34, 100000, 0, '1-uuid-9', 'testman', 0, 50, 'intro', 0, '" + getHash('locked', 1) + "')");
});
@ -207,4 +209,21 @@ describe('getSkipSegments', () => {
.catch(err => done("Couldn't call endpoint"));
});
it('Should always get locked segment', (done: Done) => {
fetch(getbaseURL() + "/api/skipSegments?videoID=locked&category=intro")
.then(async res => {
if (res.status !== 200) done("Status code was: " + res.status);
else {
const data = await res.json();
if (data.length === 1 && data[0].segment[0] === 20 && data[0].segment[1] === 33
&& data[0].category === "intro" && data[0].UUID === "1-uuid-locked-8") {
done();
} else {
done("Received incorrect body: " + (await res.text()));
}
}
})
.catch(err => done("Couldn't call endpoint"));
});
});

37
test/cases/postSkipSegments.ts
View file

@ -39,6 +39,8 @@ describe('postSkipSegments', () => {
db.exec(startOfWarningQuery + "('" + warnUser03Hash + "', '" + (now - 1000) + "', '" + warnVip01Hash + "', 0)");
db.exec(startOfWarningQuery + "('" + warnUser03Hash + "', '" + (now - 2000) + "', '" + warnVip01Hash + "', 1)");
db.exec(startOfWarningQuery + "('" + warnUser03Hash + "', '" + (now - 3601000) + "', '" + warnVip01Hash + "', 1)");
db.exec("INSERT INTO vipUsers (userID) VALUES ('" + getHash("VIPUserSubmission") + "')");
});
it('Should be able to submit a single time (Params method)', (done: Done) => {
@ -82,8 +84,8 @@ describe('postSkipSegments', () => {
})
.then(res => {
if (res.status === 200) {
const row = db.prepare('get', "SELECT startTime, endTime, category FROM sponsorTimes WHERE videoID = ?", ["dQw4w9WgXcF"]);
if (row.startTime === 0 && row.endTime === 10 && row.category === "sponsor") {
const row = db.prepare('get', "SELECT startTime, endTime, locked, category FROM sponsorTimes WHERE videoID = ?", ["dQw4w9WgXcF"]);
if (row.startTime === 0 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor") {
done();
} else {
done("Submitted times were not saved. Actual submission: " + JSON.stringify(row));
@ -95,6 +97,37 @@ describe('postSkipSegments', () => {
.catch(err => done(err));
});
it('VIP submission should start locked', (done: Done) => {
fetch(getbaseURL()
+ "/api/postVideoSponsorTimes", {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
userID: "VIPUserSubmission",
videoID: "vipuserIDSubmission",
segments: [{
segment: [0, 10],
category: "sponsor",
}],
}),
})
.then(res => {
if (res.status === 200) {
const row = db.prepare('get', "SELECT startTime, endTime, locked, category FROM sponsorTimes WHERE videoID = ?", ["vipuserIDSubmission"]);
if (row.startTime === 0 && row.endTime === 10 && row.locked === 1 && row.category === "sponsor") {
done();
} else {
done("Submitted times were not saved. Actual submission: " + JSON.stringify(row));
}
} else {
done("Status code was " + res.status);
}
})
.catch(err => done(err));
});
it('Should be able to submit multiple times (JSON method)', (done: Done) => {
fetch(getbaseURL()
+ "/api/postVideoSponsorTimes", {