external/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2024-09-20 04:54:00 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #236 from ajayyy/segment-sort

Browse source 
Segment sorting
This commit is contained in:
Ajay Ramachandran 2021-05-23 23:20:42 -04:00 committed by GitHub
commit 9990e0b807
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 1036 additions and 1110 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
databases/_upgrade_sponsorTimes_12.sql Normal file
View file

@ -0,0 +1,31 @@
BEGIN TRANSACTION;
/* Add Service field */
CREATE TABLE "sqlb_temp_table_12" (
"videoID" TEXT NOT NULL,
"startTime" REAL NOT NULL,
"endTime" REAL NOT NULL,
"votes" INTEGER NOT NULL,
"locked" INTEGER NOT NULL default '0',
"incorrectVotes" INTEGER NOT NULL default '1',
"UUID" TEXT NOT NULL UNIQUE,
"userID" TEXT NOT NULL,
"timeSubmitted" INTEGER NOT NULL,
"views" INTEGER NOT NULL,
"category" TEXT NOT NULL DEFAULT 'sponsor',
"service" TEXT NOT NULL DEFAULT 'YouTube',
"videoDuration" REAL NOT NULL DEFAULT '0',
"hidden" INTEGER NOT NULL DEFAULT '0',
"reputation" REAL NOT NULL DEFAULT 0,
"shadowHidden" INTEGER NOT NULL,
"hashedVideoID" TEXT NOT NULL default ''
);
INSERT INTO sqlb_temp_table_12 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category","service","videoDuration","hidden",0,"shadowHidden","hashedVideoID" FROM "sponsorTimes";
DROP TABLE "sponsorTimes";
ALTER TABLE sqlb_temp_table_12 RENAME TO "sponsorTimes";
UPDATE "config" SET value = 12 WHERE key = 'version';
COMMIT;

1757
package-lock.json generated
View file

File diff suppressed because it is too large Load diff

4
package.json
View file

@ -30,13 +30,13 @@
"@types/better-sqlite3": "^5.4.0",
"@types/express": "^4.17.8",
"@types/express-rate-limit": "^5.1.0",
"@types/mocha": "^8.0.3",
"@types/mocha": "^8.2.2",
"@types/node": "^14.11.9",
"@types/node-fetch": "^2.5.7",
"@types/pg": "^7.14.10",
"@types/redis": "^2.8.28",
"@types/request": "^2.48.5",
"mocha": "^7.1.1",
"mocha": "^8.4.0",
"nodemon": "^2.0.2",
"sinon": "^9.2.0",
"ts-mock-imports": "^1.3.0",

93
src/routes/getSkipSegments.ts
View file

@ -1,15 +1,15 @@
import { Request, Response } from 'express';
import { RedisClient } from 'redis';
import { config } from '../config';
import { db, privateDB } from '../databases/databases';
import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys';
import { skipSegmentsHashKey, skipSegmentsKey } from '../utils/redisKeys';
import { SBRecord } from '../types/lib.model';
import { Category, CategoryActionType, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, Service, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model";
import { getCategoryActionType } from '../utils/categoryInfo';
import { getHash } from '../utils/getHash';
import { getIP } from '../utils/getIP';
import { Logger } from '../utils/logger';
import redis from '../utils/redis';
import { QueryCacher } from '../utils/queryCacher'
import { getReputation } from '../utils/reputation';
async function prepareCategorySegments(req: Request, videoID: VideoID, category: Category, segments: DBSegment[], cache: SegmentCache = {shadowHiddenSegmentIPs: {}}): Promise<Segment[]> {
@ -42,7 +42,7 @@ async function prepareCategorySegments(req: Request, videoID: VideoID, category:
const filteredSegments = segments.filter((_, index) => shouldFilter[index]);
const maxSegments = getCategoryActionType(category) === CategoryActionType.Skippable ? 32 : 1
return chooseSegments(filteredSegments, maxSegments).map((chosenSegment) => ({
return (await chooseSegments(filteredSegments, maxSegments)).map((chosenSegment) => ({
category,
segment: [chosenSegment.startTime, chosenSegment.endTime],
UUID: chosenSegment.UUID,
@ -50,7 +50,7 @@ async function prepareCategorySegments(req: Request, videoID: VideoID, category:
}));
}
async function getSegmentsByVideoID(req: Request, videoID: string, categories: Category[], service: Service): Promise<Segment[]> {
async function getSegmentsByVideoID(req: Request, videoID: VideoID, categories: Category[], service: Service): Promise<Segment[]> {
const cache: SegmentCache = {shadowHiddenSegmentIPs: {}};
const segments: Segment[] = [];
@ -58,13 +58,9 @@ async function getSegmentsByVideoID(req: Request, videoID: string, categories: C
categories = categories.filter((category) => !/[^a-z|_|-]/.test(category));
if (categories.length === 0) return null;
const segmentsByCategory: SBRecord<Category, DBSegment[]> = (await db
.prepare(
'all',
`SELECT "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden" FROM "sponsorTimes"
WHERE "videoID" = ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`,
[videoID, service]
)).reduce((acc: SBRecord<Category, DBSegment[]>, segment: DBSegment) => {
const segmentsByCategory: SBRecord<Category, DBSegment[]> = (await getSegmentsFromDBByVideoID(videoID, service))
.filter((segment: DBSegment) => categories.includes(segment?.category))
.reduce((acc: SBRecord<Category, DBSegment[]>, segment: DBSegment) => {
acc[segment.category] = acc[segment.category] || [];
acc[segment.category].push(segment);
@ -72,7 +68,7 @@ async function getSegmentsByVideoID(req: Request, videoID: string, categories: C
}, {});
for (const [category, categorySegments] of Object.entries(segmentsByCategory)) {
segments.push(...(await prepareCategorySegments(req, videoID as VideoID, category as Category, categorySegments, cache)));
segments.push(...(await prepareCategorySegments(req, videoID, category as Category, categorySegments, cache)));
}
return segments;
@ -94,7 +90,7 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash,
categories = categories.filter((category) => !(/[^a-z|_|-]/.test(category)));
if (categories.length === 0) return null;
const segmentPerVideoID: SegmentWithHashPerVideoID = (await getSegmentsFromDB(hashedVideoIDPrefix, service))
const segmentPerVideoID: SegmentWithHashPerVideoID = (await getSegmentsFromDBByHash(hashedVideoIDPrefix, service))
.filter((segment: DBSegment) => categories.includes(segment?.category))
.reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => {
acc[segment.videoID] = acc[segment.videoID] || {
@ -129,37 +125,34 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash,
}
}
async function getSegmentsFromDB(hashedVideoIDPrefix: VideoIDHash, service: Service): Promise<DBSegment[]> {
async function getSegmentsFromDBByHash(hashedVideoIDPrefix: VideoIDHash, service: Service): Promise<DBSegment[]> {
const fetchFromDB = () => db
.prepare(
'all',
`SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden", "hashedVideoID" FROM "sponsorTimes"
`SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "category", "videoDuration", "reputation", "shadowHidden", "hashedVideoID" FROM "sponsorTimes"
WHERE "hashedVideoID" LIKE ? AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`,
[hashedVideoIDPrefix + '%', service]
);
) as Promise<DBSegment[]>;
if (hashedVideoIDPrefix.length === 4) {
const key = skipSegmentsHashKey(hashedVideoIDPrefix, service);
const {err, reply} = await redis.getAsync(key);
if (!err && reply) {
try {
Logger.debug("Got data from redis: " + reply);
return JSON.parse(reply);
} catch (e) {
// If all else, continue on to fetching from the database
}
}
const data = await fetchFromDB();
redis.setAsync(key, JSON.stringify(data));
return data;
return await QueryCacher.get(fetchFromDB, skipSegmentsHashKey(hashedVideoIDPrefix, service))
}
return await fetchFromDB();
}
async function getSegmentsFromDBByVideoID(videoID: VideoID, service: Service): Promise<DBSegment[]> {
const fetchFromDB = () => db
.prepare(
'all',
`SELECT "startTime", "endTime", "votes", "locked", "UUID", "userID", "category", "videoDuration", "reputation", "shadowHidden" FROM "sponsorTimes"
WHERE "videoID" = ? AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`,
[videoID, service]
) as Promise<DBSegment[]>;
return await QueryCacher.get(fetchFromDB, skipSegmentsKey(videoID, service))
}
//gets a weighted random choice from the choices array based on their `votes` property.
//amountOfChoices specifies the maximum amount of choices to return, 1 or more.
//choices are unique
@ -176,9 +169,11 @@ function getWeightedRandomChoice<T extends VotableObject>(choices: T[], amountOf
//assign a weight to each choice
let totalWeight = 0;
let choicesWithWeights: TWithWeight[] = choices.map(choice => {
const boost = Math.min(choice.reputation, Math.max(0, choice.votes * 2));
//The 3 makes -2 the minimum votes before being ignored completely
//this can be changed if this system increases in popularity.
const weight = Math.exp((choice.votes + 3));
const weight = Math.exp(choice.votes * Math.min(1, choice.reputation + 1) + 3 + boost);
totalWeight += weight;
return {...choice, weight};
@ -208,7 +203,7 @@ function getWeightedRandomChoice<T extends VotableObject>(choices: T[], amountOf
//Only one similar time will be returned, randomly generated based on the sqrt of votes.
//This allows new less voted items to still sometimes appear to give them a chance at getting votes.
//Segments with less than -1 votes are already ignored before this function is called
function chooseSegments(segments: DBSegment[], max: number): DBSegment[] {
async function chooseSegments(segments: DBSegment[], max: number): Promise<DBSegment[]> {
//Create groups of segments that are similar to eachother
//Segments must be sorted by their startTime so that we can build groups chronologically:
//1. As long as the segments' startTime fall inside the currentGroup, we keep adding them to that group
@ -217,9 +212,9 @@ function chooseSegments(segments: DBSegment[], max: number): DBSegment[] {
const overlappingSegmentsGroups: OverlappingSegmentGroup[] = [];
let currentGroup: OverlappingSegmentGroup;
let cursor = -1; //-1 to make sure that, even if the 1st segment starts at 0, a new group is created
segments.forEach(segment => {
for (const segment of segments) {
if (segment.startTime > cursor) {
currentGroup = {segments: [], votes: 0, locked: false};
currentGroup = {segments: [], votes: 0, reputation: 0, locked: false};
overlappingSegmentsGroups.push(currentGroup);
}
@ -229,17 +224,24 @@ function chooseSegments(segments: DBSegment[], max: number): DBSegment[] {
currentGroup.votes += segment.votes;
}
if (segment.userID) segment.reputation = Math.min(segment.reputation, await getReputation(segment.userID));
if (segment.reputation > 0) {
currentGroup.reputation += segment.reputation;
}
if (segment.locked) {
currentGroup.locked = true;
}
cursor = Math.max(cursor, segment.endTime);
});
};
overlappingSegmentsGroups.forEach((group) => {
if (group.locked) {
group.segments = group.segments.filter((segment) => segment.locked);
}
group.reputation = group.reputation / group.segments.length;
});
//if there are too many groups, find the best ones
@ -278,18 +280,6 @@ async function handleGetSegments(req: Request, res: Response): Promise<Segment[]
service = Service.YouTube;
}
// Only 404s are cached at the moment
const redisResult = await redis.getAsync(skipSegmentsKey(videoID));
if (redisResult.reply) {
const redisSegments = JSON.parse(redisResult.reply);
if (redisSegments?.length === 0) {
res.sendStatus(404);
Logger.debug("Using segments from cache for " + videoID);
return false;
}
}
const segments = await getSegmentsByVideoID(req, videoID, categories, service);
if (segments === null || segments === undefined) {
@ -300,9 +290,6 @@ async function handleGetSegments(req: Request, res: Response): Promise<Segment[]
if (segments.length === 0) {
res.sendStatus(404);
// Save in cache
if (categories.length == 7) redis.setAsync(skipSegmentsKey(videoID), JSON.stringify(segments));
return false;
}

23
src/routes/postSkipSegments.ts
View file

@ -11,11 +11,13 @@ import {getFormattedTime} from '../utils/getFormattedTime';
import {isUserTrustworthy} from '../utils/isUserTrustworthy';
import {dispatchEvent} from '../utils/webhookUtils';
import {Request, Response} from 'express';
import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys';
import { skipSegmentsHashKey, skipSegmentsKey } from '../utils/redisKeys';
import redis from '../utils/redis';
import { Category, CategoryActionType, IncomingSegment, Segment, SegmentUUID, Service, VideoDuration, VideoID } from '../types/segments.model';
import { deleteLockCategories } from './deleteLockCategories';
import { getCategoryActionType } from '../utils/categoryInfo';
import { QueryCacher } from '../utils/queryCacher';
import { getReputation } from '../utils/reputation';
interface APIVideoInfo {
err: string | boolean,
@ -290,14 +292,10 @@ function proxySubmission(req: Request) {
body: req.body,
})
.then(async res => {
if (config.mode === 'development') {
Logger.debug('Proxy Submission: ' + res.status + ' (' + (await res.text()) + ')');
}
})
.catch(err => {
if (config.mode === 'development') {
Logger.error("Proxy Submission: Failed to make call");
}
});
}
@ -511,6 +509,7 @@ export async function postSkipSegments(req: Request, res: Response) {
}
let startingVotes = 0 + decreaseVotes;
const reputation = await getReputation(userID);
for (const segmentInfo of segments) {
//this can just be a hash of the data
@ -522,9 +521,9 @@ export async function postSkipSegments(req: Request, res: Response) {
const startingLocked = isVIP ? 1 : 0;
try {
await db.prepare('run', `INSERT INTO "sponsorTimes"
("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "service", "videoDuration", "shadowHidden", "hashedVideoID")
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, service, videoDuration, shadowBanned, hashedVideoID,
("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "service", "videoDuration", "reputation", "shadowHidden", "hashedVideoID")
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, service, videoDuration, reputation, shadowBanned, hashedVideoID,
],
);
@ -532,8 +531,12 @@ export async function postSkipSegments(req: Request, res: Response) {
await privateDB.prepare('run', `INSERT INTO "sponsorTimes" VALUES(?, ?, ?)`, [videoID, hashedIP, timeSubmitted]);
// Clear redis cache for this video
redis.delAsync(skipSegmentsKey(videoID));
redis.delAsync(skipSegmentsHashKey(hashedVideoID, service));
QueryCacher.clearVideoCache({
videoID,
hashedVideoID,
service,
userID
});
} catch (err) {
//a DB change probably occurred
res.sendStatus(500);

51
src/routes/voteOnSponsorTime.ts
View file

@ -5,16 +5,14 @@ import fetch from 'node-fetch';
import {YouTubeAPI} from '../utils/youtubeApi';
import {db, privateDB} from '../databases/databases';
import {dispatchEvent, getVoteAuthor, getVoteAuthorRaw} from '../utils/webhookUtils';
import {isUserTrustworthy} from '../utils/isUserTrustworthy';
import {getFormattedTime} from '../utils/getFormattedTime';
import {getIP} from '../utils/getIP';
import {getHash} from '../utils/getHash';
import {config} from '../config';
import { UserID } from '../types/user.model';
import redis from '../utils/redis';
import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys';
import { Category, CategoryActionType, HashedIP, IPAddress, SegmentUUID, Service, VideoID, VideoIDHash } from '../types/segments.model';
import { getCategoryActionType } from '../utils/categoryInfo';
import { QueryCacher } from '../utils/queryCacher';
const voteTypes = {
normal: 0,
@ -158,8 +156,8 @@ async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, i
return;
}
const videoInfo = (await db.prepare('get', `SELECT "category", "videoID", "hashedVideoID", "service" FROM "sponsorTimes" WHERE "UUID" = ?`,
[UUID])) as {category: Category, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service};
const videoInfo = (await db.prepare('get', `SELECT "category", "videoID", "hashedVideoID", "service", "userID" FROM "sponsorTimes" WHERE "UUID" = ?`,
[UUID])) as {category: Category, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, userID: UserID};
if (!videoInfo) {
// Submission doesn't exist
res.status(400).send("Submission doesn't exist.");
@ -230,7 +228,7 @@ async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, i
}
}
clearRedisCache(videoInfo);
QueryCacher.clearVideoCache(videoInfo);
res.sendStatus(finalResponse.finalStatus);
}
@ -366,8 +364,8 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
}
//check if the increment amount should be multiplied (downvotes have more power if there have been many views)
const videoInfo = await db.prepare('get', `SELECT "videoID", "hashedVideoID", "service", "votes", "views" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]) as
{videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, votes: number, views: number};
const videoInfo = await db.prepare('get', `SELECT "videoID", "hashedVideoID", "service", "votes", "views", "userID" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]) as
{videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, votes: number, views: number, userID: UserID};
if (voteTypeEnum === voteTypes.normal) {
if ((isVIP || isOwnSubmission) && incrementAmount < 0) {
@ -385,7 +383,8 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
// Only change the database if they have made a submission before and haven't voted recently
const ableToVote = isVIP
|| ((await db.prepare("get", `SELECT "userID" FROM "sponsorTimes" WHERE "userID" = ?`, [nonAnonUserID])) !== undefined
|| (!(isOwnSubmission && incrementAmount > 0)
&& (await db.prepare("get", `SELECT "userID" FROM "sponsorTimes" WHERE "userID" = ?`, [nonAnonUserID])) !== undefined
&& (await privateDB.prepare("get", `SELECT "userID" FROM "shadowBannedUsers" WHERE "userID" = ?`, [nonAnonUserID])) === undefined
&& (await privateDB.prepare("get", `SELECT "UUID" FROM "votes" WHERE "UUID" = ? AND "hashedIP" = ? AND "userID" != ?`, [UUID, hashedIP, userID])) === undefined)
&& finalResponse.finalStatus === 200;
@ -416,32 +415,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
await db.prepare('run', 'UPDATE "sponsorTimes" SET locked = 0 WHERE "UUID" = ?', [UUID]);
}
clearRedisCache(videoInfo);
//for each positive vote, see if a hidden submission can be shown again
if (incrementAmount > 0 && voteTypeEnum === voteTypes.normal) {
//find the UUID that submitted the submission that was voted on
const submissionUserIDInfo = await db.prepare('get', 'SELECT "userID" FROM "sponsorTimes" WHERE "UUID" = ?', [UUID]);
if (!submissionUserIDInfo) {
// They are voting on a non-existent submission
res.status(400).send("Voting on a non-existent submission");
return;
}
const submissionUserID = submissionUserIDInfo.userID;
//check if any submissions are hidden
const hiddenSubmissionsRow = await db.prepare('get', 'SELECT count(*) as "hiddenSubmissions" FROM "sponsorTimes" WHERE "userID" = ? AND "shadowHidden" > 0', [submissionUserID]);
if (hiddenSubmissionsRow.hiddenSubmissions > 0) {
//see if some of this users submissions should be visible again
if (await isUserTrustworthy(submissionUserID)) {
//they are trustworthy again, show 2 of their submissions again, if there are two to show
await db.prepare('run', 'UPDATE "sponsorTimes" SET "shadowHidden" = 0 WHERE ROWID IN (SELECT ROWID FROM "sponsorTimes" WHERE "userID" = ? AND "shadowHidden" = 1 LIMIT 2)', [submissionUserID]);
}
}
}
QueryCacher.clearVideoCache(videoInfo);
}
res.status(finalResponse.finalStatus).send(finalResponse.finalMessage ?? undefined);
@ -466,10 +440,3 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
res.status(500).json({error: 'Internal error creating segment vote'});
}
}
function clearRedisCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; }) {
if (videoInfo) {
redis.delAsync(skipSegmentsKey(videoInfo.videoID));
redis.delAsync(skipSegmentsHashKey(videoInfo.hashedVideoID, videoInfo.service));
}
}

5
src/types/segments.model.ts
View file

@ -1,5 +1,6 @@
import { HashedValue } from "./hash.model";
import { SBRecord } from "./lib.model";
import { UserID } from "./user.model";
export type SegmentUUID = string & { __segmentUUIDBrand: unknown };
export type VideoID = string & { __videoIDBrand: unknown };
@ -42,11 +43,13 @@ export interface DBSegment {
startTime: number;
endTime: number;
UUID: SegmentUUID;
userID: UserID;
votes: number;
locked: boolean;
shadowHidden: Visibility;
videoID: VideoID;
videoDuration: VideoDuration;
reputation: number;
hashedVideoID: VideoIDHash;
}
@ -54,10 +57,12 @@ export interface OverlappingSegmentGroup {
segments: DBSegment[],
votes: number;
locked: boolean; // Contains a locked segment
reputation: number;
}
export interface VotableObject {
votes: number;
reputation: number;
}
export interface VotableObjectWithWeight extends VotableObject {

37
src/utils/queryCacher.ts Normal file
View file

@ -0,0 +1,37 @@
import redis from "../utils/redis";
import { Logger } from "../utils/logger";
import { skipSegmentsHashKey, skipSegmentsKey, reputationKey } from "./redisKeys";
import { Service, VideoID, VideoIDHash } from "../types/segments.model";
import { UserID } from "../types/user.model";
async function get<T>(fetchFromDB: () => Promise<T>, key: string): Promise<T> {
const {err, reply} = await redis.getAsync(key);
if (!err && reply) {
try {
Logger.debug("Got data from redis: " + reply);
return JSON.parse(reply);
} catch (e) {
// If all else, continue on to fetching from the database
}
}
const data = await fetchFromDB();
redis.setAsync(key, JSON.stringify(data));
return data;
}
function clearVideoCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; userID: UserID; }) {
if (videoInfo) {
redis.delAsync(skipSegmentsKey(videoInfo.videoID, videoInfo.service));
redis.delAsync(skipSegmentsHashKey(videoInfo.hashedVideoID, videoInfo.service));
redis.delAsync(reputationKey(videoInfo.userID));
}
}
export const QueryCacher = {
get,
clearVideoCache
}

11
src/middleware/redisKeys.ts → src/utils/redisKeys.ts
View file

@ -1,8 +1,9 @@
import { Service, VideoID, VideoIDHash } from "../types/segments.model";
import { Logger } from "../utils/logger";
import { UserID } from "../types/user.model";
import { Logger } from "./logger";
export function skipSegmentsKey(videoID: VideoID): string {
return "segments-" + videoID;
export function skipSegmentsKey(videoID: VideoID, service: Service): string {
return "segments." + service + ".videoID." + videoID;
}
export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string {
@ -11,3 +12,7 @@ export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: S
return "segments." + service + "." + hashedVideoIDPrefix;
}
export function reputationKey(userID: UserID): string {
return "reputation.user." + userID;
}

45
src/utils/reputation.ts Normal file
View file

@ -0,0 +1,45 @@
import { db } from "../databases/databases";
import { UserID } from "../types/user.model";
import { QueryCacher } from "./queryCacher";
import { reputationKey } from "./redisKeys";
interface ReputationDBResult {
totalSubmissions: number,
downvotedSubmissions: number,
upvotedSum: number,
oldUpvotedSubmissions: number
}
export async function getReputation(userID: UserID): Promise<number> {
const pastDate = Date.now() - 1000 * 60 * 60 * 24 * 45; // 45 days ago
const fetchFromDB = () => db.prepare("get",
`SELECT COUNT(*) AS "totalSubmissions",
SUM(CASE WHEN "votes" < 0 THEN 1 ELSE 0 END) AS "downvotedSubmissions",
SUM(CASE WHEN "votes" > 0 THEN "votes" ELSE 0 END) AS "upvotedSum",
SUM(CASE WHEN "timeSubmitted" < ? AND "votes" > 0 THEN 1 ELSE 0 END) AS "oldUpvotedSubmissions"
FROM "sponsorTimes" WHERE "userID" = ?`, [pastDate, userID]) as Promise<ReputationDBResult>;
const result = await QueryCacher.get(fetchFromDB, reputationKey(userID));
// Grace period
if (result.totalSubmissions < 5) {
return 0;
}
const downvoteRatio = result.downvotedSubmissions / result.totalSubmissions;
if (downvoteRatio > 0.3) {
return convertRange(downvoteRatio, 0.3, 1, -0.5, -1.5);
}
if (result.oldUpvotedSubmissions < 3 || result.upvotedSum < 5) {
return 0;
}
return convertRange(Math.min(result.upvotedSum, 50), 5, 50, 0, 15);
}
function convertRange(value: number, currentMin: number, currentMax: number, targetMin: number, targetMax: number): number {
const currentRange = currentMax - currentMin;
const targetRange = targetMax - targetMin;
return ((value - currentMin) / currentRange) * targetRange + targetMin;
}

79
test/cases/reputation.ts Normal file
View file

@ -0,0 +1,79 @@
import assert from 'assert';
import { db } from '../../src/databases/databases';
import { UserID } from '../../src/types/user.model';
import { getHash } from '../../src/utils/getHash';
import { getReputation } from '../../src/utils/reputation';
const userIDLowSubmissions = "reputation-lowsubmissions" as UserID;
const userIDHighDownvotes = "reputation-highdownvotes" as UserID;
const userIDNewSubmissions = "reputation-newsubmissions" as UserID;
const userIDLowSum = "reputation-lowsum" as UserID;
const userIDHighRep = "reputation-highrep" as UserID;
describe('reputation', () => {
before(async () => {
const videoID = "reputation-videoID";
let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "service", "videoDuration", "hidden", "shadowHidden", "hashedVideoID") VALUES';
await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-0-uuid-0', '${getHash(userIDLowSubmissions)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-0-uuid-1', '${getHash(userIDLowSubmissions)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 100, 0, 'reputation-0-uuid-2', '${getHash(userIDLowSubmissions)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-1-uuid-0', '${getHash(userIDHighDownvotes)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -2, 0, 'reputation-1-uuid-1', '${getHash(userIDHighDownvotes)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -2, 0, 'reputation-1-uuid-2', '${getHash(userIDHighDownvotes)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -2, 0, 'reputation-1-uuid-3', '${getHash(userIDHighDownvotes)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -2, 0, 'reputation-1-uuid-4', '${getHash(userIDHighDownvotes)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -1, 0, 'reputation-1-uuid-5', '${getHash(userIDHighDownvotes)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-1-uuid-6', '${getHash(userIDHighDownvotes)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-1-uuid-7', '${getHash(userIDHighDownvotes)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-2-uuid-0', '${getHash(userIDNewSubmissions)}', ${Date.now()}, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-2-uuid-1', '${getHash(userIDNewSubmissions)}', ${Date.now()}, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-2-uuid-2', '${getHash(userIDNewSubmissions)}', ${Date.now()}, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-2-uuid-3', '${getHash(userIDNewSubmissions)}', ${Date.now()}, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-2-uuid-4', '${getHash(userIDNewSubmissions)}', ${Date.now()}, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -1, 0, 'reputation-2-uuid-5', '${getHash(userIDNewSubmissions)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-2-uuid-6', '${getHash(userIDNewSubmissions)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-2-uuid-7', '${getHash(userIDNewSubmissions)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-3-uuid-0', '${getHash(userIDLowSum)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 1, 0, 'reputation-3-uuid-1', '${getHash(userIDLowSum)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-3-uuid-2', '${getHash(userIDLowSum)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-3-uuid-3', '${getHash(userIDLowSum)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 1, 0, 'reputation-3-uuid-4', '${getHash(userIDLowSum)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -1, 0, 'reputation-3-uuid-5', '${getHash(userIDLowSum)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-3-uuid-6', '${getHash(userIDLowSum)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-3-uuid-7', '${getHash(userIDLowSum)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-4-uuid-0', '${getHash(userIDHighRep)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-4-uuid-1', '${getHash(userIDHighRep)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-4-uuid-2', '${getHash(userIDHighRep)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-4-uuid-3', '${getHash(userIDHighRep)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-4-uuid-4', '${getHash(userIDHighRep)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -1, 0, 'reputation-4-uuid-5', '${getHash(userIDHighRep)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-4-uuid-6', '${getHash(userIDHighRep)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-4-uuid-7', '${getHash(userIDHighRep)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`);
});
it("user in grace period", async () => {
assert.strictEqual(await getReputation(getHash(userIDLowSubmissions)), 0);
});
it("user with high downvote ratio", async () => {
assert.strictEqual(await getReputation(getHash(userIDHighDownvotes)), -0.9642857142857144);
});
it("user with mostly new submissions", async () => {
assert.strictEqual(await getReputation(getHash(userIDNewSubmissions)), 0);
});
it("user with not enough vote sum", async () => {
assert.strictEqual(await getReputation(getHash(userIDLowSum)), 0);
});
it("user with high reputation", async () => {
assert.strictEqual(await getReputation(getHash(userIDHighRep)), 1.6666666666666665);
});
});