external/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2024-11-10 01:02:30 +01:00
Code Issues Projects Releases Packages Wiki Activity

Use reputation when sorting segments

Browse source
This commit is contained in:
Ajay Ramachandran 2021-05-23 17:54:51 -04:00
parent 194c657ba7
commit 5c2ab9087a
5 changed files with 59 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
databases/_upgrade_sponsorTimes_12.sql Normal file
View file

@ -0,0 +1,31 @@
BEGIN TRANSACTION;
/* Add Service field */
CREATE TABLE "sqlb_temp_table_12" (
"videoID" TEXT NOT NULL,
"startTime" REAL NOT NULL,
"endTime" REAL NOT NULL,
"votes" INTEGER NOT NULL,
"locked" INTEGER NOT NULL default '0',
"incorrectVotes" INTEGER NOT NULL default '1',
"UUID" TEXT NOT NULL UNIQUE,
"userID" TEXT NOT NULL,
"timeSubmitted" INTEGER NOT NULL,
"views" INTEGER NOT NULL,
"category" TEXT NOT NULL DEFAULT 'sponsor',
"service" TEXT NOT NULL DEFAULT 'YouTube',
"videoDuration" REAL NOT NULL DEFAULT '0',
"hidden" INTEGER NOT NULL DEFAULT '0',
"reputation" REAL NOT NULL DEFAULT 0,
"shadowHidden" INTEGER NOT NULL,
"hashedVideoID" TEXT NOT NULL default ''
);
INSERT INTO sqlb_temp_table_12 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category","service","videoDuration","hidden",0,"shadowHidden","hashedVideoID" FROM "sponsorTimes";
DROP TABLE "sponsorTimes";
ALTER TABLE sqlb_temp_table_12 RENAME TO "sponsorTimes";
UPDATE "config" SET value = 12 WHERE key = 'version';
COMMIT;

4
src/middleware/reputation.ts
View file

@ -10,7 +10,7 @@ interface ReputationDBResult {
oldUpvotedSubmissions: number
}
export async function getReputation(userID: UserID) {
export async function getReputation(userID: UserID): Promise<number> {
const pastDate = Date.now() - 1000 * 1000 * 60 * 60 * 24 * 45; // 45 days ago
const fetchFromDB = () => db.prepare("get",
`SELECT COUNT(*) AS "totalSubmissions",
@ -32,7 +32,7 @@ export async function getReputation(userID: UserID) {
}
if (result.oldUpvotedSubmissions < 3 || result.upvotedSum < 5) {
return 0
return 0;
}
return convertRange(Math.min(result.upvotedSum, 50), 5, 50, 0, 15);

24
src/routes/getSkipSegments.ts
View file

@ -9,6 +9,7 @@ import { getHash } from '../utils/getHash';
import { getIP } from '../utils/getIP';
import { Logger } from '../utils/logger';
import { QueryCacher } from '../middleware/queryCacher'
import { getReputation } from '../middleware/reputation';
async function prepareCategorySegments(req: Request, videoID: VideoID, category: Category, segments: DBSegment[], cache: SegmentCache = {shadowHiddenSegmentIPs: {}}): Promise<Segment[]> {
@ -41,7 +42,7 @@ async function prepareCategorySegments(req: Request, videoID: VideoID, category:
const filteredSegments = segments.filter((_, index) => shouldFilter[index]);
const maxSegments = getCategoryActionType(category) === CategoryActionType.Skippable ? 32 : 1
return chooseSegments(filteredSegments, maxSegments).map((chosenSegment) => ({
return (await chooseSegments(filteredSegments, maxSegments)).map((chosenSegment) => ({
category,
segment: [chosenSegment.startTime, chosenSegment.endTime],
UUID: chosenSegment.UUID,
@ -128,7 +129,7 @@ async function getSegmentsFromDBByHash(hashedVideoIDPrefix: VideoIDHash, service
const fetchFromDB = () => db
.prepare(
'all',
`SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden", "hashedVideoID" FROM "sponsorTimes"
`SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "category", "videoDuration", "reputation", "shadowHidden", "hashedVideoID" FROM "sponsorTimes"
WHERE "hashedVideoID" LIKE ? AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`,
[hashedVideoIDPrefix + '%', service]
) as Promise<DBSegment[]>;
@ -144,7 +145,7 @@ async function getSegmentsFromDBByVideoID(videoID: VideoID, service: Service): P
const fetchFromDB = () => db
.prepare(
'all',
`SELECT "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden" FROM "sponsorTimes"
`SELECT "startTime", "endTime", "votes", "locked", "UUID", "userID", "category", "videoDuration", "reputation", "shadowHidden" FROM "sponsorTimes"
WHERE "videoID" = ? AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`,
[videoID, service]
) as Promise<DBSegment[]>;
@ -170,7 +171,7 @@ function getWeightedRandomChoice<T extends VotableObject>(choices: T[], amountOf
let choicesWithWeights: TWithWeight[] = choices.map(choice => {
//The 3 makes -2 the minimum votes before being ignored completely
//this can be changed if this system increases in popularity.
const weight = Math.exp((choice.votes + 3));
const weight = Math.exp((choice.votes + 3 + choice.reputation));
totalWeight += weight;
return {...choice, weight};
@ -200,7 +201,7 @@ function getWeightedRandomChoice<T extends VotableObject>(choices: T[], amountOf
//Only one similar time will be returned, randomly generated based on the sqrt of votes.
//This allows new less voted items to still sometimes appear to give them a chance at getting votes.
//Segments with less than -1 votes are already ignored before this function is called
function chooseSegments(segments: DBSegment[], max: number): DBSegment[] {
async function chooseSegments(segments: DBSegment[], max: number): Promise<DBSegment[]> {
//Create groups of segments that are similar to eachother
//Segments must be sorted by their startTime so that we can build groups chronologically:
//1. As long as the segments' startTime fall inside the currentGroup, we keep adding them to that group
@ -209,9 +210,9 @@ function chooseSegments(segments: DBSegment[], max: number): DBSegment[] {
const overlappingSegmentsGroups: OverlappingSegmentGroup[] = [];
let currentGroup: OverlappingSegmentGroup;
let cursor = -1; //-1 to make sure that, even if the 1st segment starts at 0, a new group is created
segments.forEach(segment => {
for (const segment of segments) {
if (segment.startTime > cursor) {
currentGroup = {segments: [], votes: 0, locked: false};
currentGroup = {segments: [], votes: 0, reputation: 0, locked: false};
overlappingSegmentsGroups.push(currentGroup);
}
@ -221,17 +222,24 @@ function chooseSegments(segments: DBSegment[], max: number): DBSegment[] {
currentGroup.votes += segment.votes;
}
segment.reputation = Math.min(segment.reputation, await getReputation(segment.userID));
if (segment.reputation > 0) {
currentGroup.reputation += segment.reputation;
}
if (segment.locked) {
currentGroup.locked = true;
}
cursor = Math.max(cursor, segment.endTime);
});
};
overlappingSegmentsGroups.forEach((group) => {
if (group.locked) {
group.segments = group.segments.filter((segment) => segment.locked);
}
group.reputation = group.reputation / group.segments.length;
});
//if there are too many groups, find the best ones

8
src/routes/postSkipSegments.ts
View file

@ -17,6 +17,7 @@ import { Category, CategoryActionType, IncomingSegment, Segment, SegmentUUID, Se
import { deleteLockCategories } from './deleteLockCategories';
import { getCategoryActionType } from '../utils/categoryInfo';
import { QueryCacher } from '../middleware/queryCacher';
import { getReputation } from '../middleware/reputation';
interface APIVideoInfo {
err: string | boolean,
@ -508,6 +509,7 @@ export async function postSkipSegments(req: Request, res: Response) {
}
let startingVotes = 0 + decreaseVotes;
const reputation = await getReputation(userID);
for (const segmentInfo of segments) {
//this can just be a hash of the data
@ -519,9 +521,9 @@ export async function postSkipSegments(req: Request, res: Response) {
const startingLocked = isVIP ? 1 : 0;
try {
await db.prepare('run', `INSERT INTO "sponsorTimes"
("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "service", "videoDuration", "shadowHidden", "hashedVideoID")
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, service, videoDuration, shadowBanned, hashedVideoID,
("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "service", "videoDuration", "reputation", "shadowHidden", "hashedVideoID")
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, service, videoDuration, reputation, shadowBanned, hashedVideoID,
],
);

5
src/types/segments.model.ts
View file

@ -1,5 +1,6 @@
import { HashedValue } from "./hash.model";
import { SBRecord } from "./lib.model";
import { UserID } from "./user.model";
export type SegmentUUID = string & { __segmentUUIDBrand: unknown };
export type VideoID = string & { __videoIDBrand: unknown };
@ -42,11 +43,13 @@ export interface DBSegment {
startTime: number;
endTime: number;
UUID: SegmentUUID;
userID: UserID;
votes: number;
locked: boolean;
shadowHidden: Visibility;
videoID: VideoID;
videoDuration: VideoDuration;
reputation: number;
hashedVideoID: VideoIDHash;
}
@ -54,10 +57,12 @@ export interface OverlappingSegmentGroup {
segments: DBSegment[],
votes: number;
locked: boolean; // Contains a locked segment
reputation: number;
}
export interface VotableObject {
votes: number;
reputation: number;
}
export interface VotableObjectWithWeight extends VotableObject {