external/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2024-09-19 20:51:19 +02:00
Code Issues Projects Releases Packages Wiki Activity

Add endpoints for rating endpoint (dislikes)

Browse source 
https://github.com/ajayyy/SponsorBlock/issues/1039
This commit is contained in:
Ajay Ramachandran 2021-11-14 23:09:12 -05:00
parent 7590047c6d
commit bc6db0d109
17 changed files with 415 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.eslintrc.js
View file

@ -27,6 +27,6 @@ module.exports = {
"indent": ["warn", 4, { "SwitchCase": 1 }],
"object-curly-spacing": ["warn", "always"],
"require-await": "warn",
"no-console": "error"
"no-console": "warn"
},
};

33
DatabaseSchema.md
View file

@ -185,6 +185,22 @@
| hashedVideoID | TEXT | not null, default '', sha256 |
| userAgent | TEXT | not null, default '' |
### ratings
| Name | Type | |
| -- | :--: | -- |
| videoID | TEXT | not null |
| service | TEXT | not null, default 'YouTube' |
| type | INTEGER | not null |
| count | INTEGER | not null |
| hashedVideoID | TEXT | not null |
| index | field |
| -- | :--: |
| ratings_hashedVideoID_gin | hashedVideoID |
| ratings_hashedVideoID | hashedVideoID, service |
| ratings_videoID | videoID, service |
# Private
[vote](#vote)
@ -238,4 +254,19 @@
| Name | Type | |
| -- | :--: | -- |
| key | TEXT | not null |
| value | TEXT | not null |
| value | TEXT | not null |
### ratings
| Name | Type | |
| -- | :--: | -- |
| videoID | TEXT | not null |
| service | TEXT | not null, default 'YouTube' |
| userID | TEXT | not null |
| type | INTEGER | not null |
| timeSubmitted | INTEGER | not null |
| hashedIP | TEXT | not null |
| index | field |
| -- | :--: |
| ratings_videoID | videoID, service, userID, timeSubmitted |

7
databases/_private_indexes.sql
View file

@ -22,4 +22,11 @@ CREATE INDEX IF NOT EXISTS "votes_userID"
CREATE INDEX IF NOT EXISTS "categoryVotes_UUID"
ON public."categoryVotes" USING btree
("UUID" COLLATE pg_catalog."default" ASC NULLS LAST, "userID" COLLATE pg_catalog."default" ASC NULLS LAST, "hashedIP" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST)
TABLESPACE pg_default;
-- ratings
CREATE INDEX IF NOT EXISTS "ratings_videoID"
ON public."ratings" USING btree
("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST, "userID" COLLATE pg_catalog."default" ASC NULLS LAST, "timeSubmitted" ASC NULLS LAST)
TABLESPACE pg_default;

17
databases/_sponsorTimes_indexes.sql
View file

@ -86,4 +86,21 @@ CREATE INDEX IF NOT EXISTS "videoInfo_videoID"
CREATE INDEX IF NOT EXISTS "videoInfo_channelID"
ON public."videoInfo" USING btree
("channelID" COLLATE pg_catalog."default" ASC NULLS LAST)
TABLESPACE pg_default;
-- ratings
CREATE INDEX IF NOT EXISTS "ratings_hashedVideoID_gin"
ON public."ratings" USING gin
("hashedVideoID" COLLATE pg_catalog."default" gin_trgm_ops, category COLLATE pg_catalog."default" gin_trgm_ops)
TABLESPACE pg_default;
CREATE INDEX IF NOT EXISTS "ratings_hashedVideoID"
ON public."ratings" USING btree
("hashedVideoID" COLLATE pg_catalog."default" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST)
TABLESPACE pg_default;
CREATE INDEX IF NOT EXISTS "ratings_videoID"
ON public."ratings" USING btree
("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST)
TABLESPACE pg_default;

14
databases/_upgrade_private_4.sql Normal file
View file

@ -0,0 +1,14 @@
BEGIN TRANSACTION;
CREATE TABLE IF NOT EXISTS "ratings" (
"videoID" TEXT NOT NULL,
"service" TEXT NOT NULL default 'YouTube',
"type" INTEGER NOT NULL,
"userID" TEXT NOT NULL,
"timeSubmitted" INTEGER NOT NULL,
"hashedIP" TEXT NOT NULL
);
UPDATE "config" SET value = 4 WHERE key = 'version';
COMMIT;

13
databases/_upgrade_sponsorTimes_28.sql Normal file
View file

@ -0,0 +1,13 @@
BEGIN TRANSACTION;
CREATE TABLE IF NOT EXISTS "ratings" (
"videoID" TEXT NOT NULL,
"service" TEXT NOT NULL default 'YouTube',
"type" INTEGER NOT NULL,
"count" INTEGER NOT NULL,
"hashedVideoID" TEXT NOT NULL
);
UPDATE "config" SET value = 28 WHERE key = 'version';
COMMIT;

8
src/app.ts
View file

@ -43,6 +43,8 @@ import ExpressPromiseRouter from "express-promise-router";
import { Server } from "http";
import { youtubeApiProxy } from "./routes/youtubeApiProxy";
import { getChapterNames } from "./routes/getChapterNames";
import { postRating } from "./routes/ratings/postRating";
import { getRating } from "./routes/ratings/getRating";
export function createServer(callback: () => void): Server {
// Create a service (the app object is just a callback).
@ -74,9 +76,11 @@ function setupRoutes(router: Router) {
// Rate limit endpoint lists
const voteEndpoints: RequestHandler[] = [voteOnSponsorTime];
const viewEndpoints: RequestHandler[] = [viewedVideoSponsorTime];
const postRateEndpoints: RequestHandler[] = [postRating];
if (config.rateLimit) {
if (config.rateLimit.vote) voteEndpoints.unshift(rateLimitMiddleware(config.rateLimit.vote, voteGetUserID));
if (config.rateLimit.view) viewEndpoints.unshift(rateLimitMiddleware(config.rateLimit.view));
if (config.rateLimit.rate) postRateEndpoints.unshift(rateLimitMiddleware(config.rateLimit.rate));
}
//add the get function
@ -186,6 +190,10 @@ function setupRoutes(router: Router) {
router.get("/api/lockReason", getLockReason);
// ratings
router.get("/api/ratings/rate/:prefix", getRating);
router.post("/api/ratings/rate", postRateEndpoints);
if (config.postgres) {
router.get("/database", (req, res) => dumpDatabase(req, res, true));
router.get("/database.json", (req, res) => dumpDatabase(req, res, false));

6
src/config.ts
View file

@ -58,6 +58,12 @@ addDefaults(config, {
statusCode: 200,
message: "Too many views, please try again later",
},
rate: {
windowMs: 900000,
max: 20,
statusCode: 200,
message: "Success",
}
},
userCounterURL: null,
newLeafURLs: null,

82
src/routes/ratings/getRating.ts Normal file
View file

@ -0,0 +1,82 @@
import { Request, Response } from "express";
import { db } from "../../databases/databases";
import { RatingType } from "../../types/ratings.model";
import { Service, VideoID, VideoIDHash } from "../../types/segments.model";
import { getService } from "../../utils/getService";
import { hashPrefixTester } from "../../utils/hashPrefixTester";
import { Logger } from "../../utils/logger";
import { QueryCacher } from "../../utils/queryCacher";
import { ratingHashKey } from "../../utils/redisKeys";
interface DBRating {
videoID: VideoID,
hashedVideoID: VideoIDHash,
service: Service,
type: RatingType,
count: number
}
export async function getRating(req: Request, res: Response): Promise<Response> {
let hashPrefix = req.params.prefix as VideoIDHash;
if (!hashPrefix || !hashPrefixTester(hashPrefix)) {
return res.status(400).send("Hash prefix does not match format requirements."); // Exit early on faulty prefix
}
hashPrefix = hashPrefix.toLowerCase() as VideoIDHash;
let types: RatingType[] = [];
try {
types = req.query.types
? JSON.parse(req.query.types as string)
: req.query.type
? Array.isArray(req.query.type)
? req.query.type
: [req.query.type]
: [RatingType.Upvote, RatingType.Downvote];
if (!Array.isArray(types)) {
return res.status(400).send("Categories parameter does not match format requirements.");
}
types = types.map((type) => parseInt(type as unknown as string, 10));
} catch(error) {
return res.status(400).send("Bad parameter: categories (invalid JSON)");
}
const service: Service = getService(req.query.service, req.body.service);
try {
const ratings = (await getRatings(hashPrefix, service))
.filter((rating) => types.includes(rating.type))
.map((rating) => ({
videoID: rating.videoID,
hash: rating.hashedVideoID,
service: rating.service,
type: rating.type,
count: rating.count
}));
if (ratings) {
res.status(200);
} else {
res.status(404);
}
return res.send(ratings ?? []);
} catch (err) {
Logger.error(err as string);
return res.sendStatus(500);
}
}
async function getRatings(hashPrefix: VideoIDHash, service: Service): Promise<DBRating[]> {
const fetchFromDB = () => db
.prepare(
"all",
`SELECT "videoID", "hashedVideoID", "type", "count" FROM "ratings" WHERE "hashedVideoID" LIKE ? AND "service" = ? ORDER BY "hashedVideoID"`,
[`${hashPrefix}%`, service]
) as Promise<DBRating[]>;
if (hashPrefix.length === 4) {
return await QueryCacher.get(fetchFromDB, ratingHashKey(hashPrefix, service));
}
return fetchFromDB();
}

62
src/routes/ratings/postRating.ts Normal file
View file

@ -0,0 +1,62 @@
import { db, privateDB } from "../../databases/databases";
import { getHash } from "../../utils/getHash";
import { Logger } from "../../utils/logger";
import { Request, Response } from "express";
import { HashedUserID, UserID } from "../../types/user.model";
import { HashedIP, IPAddress, VideoID } from "../../types/segments.model";
import { getIP } from "../../utils/getIP";
import { getService } from "../../utils/getService";
import { RatingType, RatingTypes } from "../../types/ratings.model";
import { config } from "../../config";
export async function postRating(req: Request, res: Response): Promise<Response> {
const privateUserID = req.body.userID as UserID;
const videoID = req.body.videoID as VideoID;
const service = getService(req.query.service, req.body.service);
const type = req.body.type as RatingType;
const enabled = req.body.enabled ?? true;
if (privateUserID == undefined || videoID == undefined || service == undefined || type == undefined
|| (typeof privateUserID !== "string") || (typeof videoID !== "string") || (typeof service !== "string")
|| (typeof type !== "number") || (enabled && (typeof enabled !== "boolean")) || !RatingTypes.includes(type)) {
//invalid request
return res.sendStatus(400);
}
const hashedIP: HashedIP = getHash(getIP(req) + config.globalSalt as IPAddress, 1);
const hashedUserID: HashedUserID = getHash(privateUserID);
const hashedVideoID = getHash(videoID, 1);
try {
// Check if this user has voted before
const existingVote = await privateDB.prepare("get", `SELECT count(*) as "count" FROM "ratings" WHERE "videoID" = ? AND "service" = ? AND "type" = ? AND "userID" = ?`, [videoID, service, type, hashedUserID]);
if (existingVote.count > 0 && !enabled) {
// Undo the vote
await db.prepare("run", `UPDATE "ratings" SET "count" = "count" - 1 WHERE "videoID" = ? AND "service" = ? AND type = ?`, [videoID, service, type]);
await privateDB.prepare("run", `DELETE FROM "ratings" WHERE "videoID" = ? AND "service" = ? AND "type" = ? AND "userID" = ?`, [videoID, service, type, hashedUserID]);
return res.sendStatus(200);
} else if (existingVote.count === 0 && enabled) {
// Make sure there hasn't been another vote from this IP
const existingIPVote = (await privateDB.prepare("get", `SELECT count(*) as "count" FROM "ratings" WHERE "videoID" = ? AND "service" = ? AND "type" = ? AND "hashedIP" = ?`, [videoID, service, type, hashedIP]))
.count > 0;
if (!existingIPVote) {
// Check if general rating already exists, if so increase it
const rating = await db.prepare("get", `SELECT count(*) as "count" FROM "ratings" WHERE "videoID" = ? AND "service" = ? AND type = ?`, [videoID, service, type]);
if (rating.count > 0) {
await db.prepare("run", `UPDATE "ratings" SET "count" = "count" + 1 WHERE "videoID" = ? AND "service" = ? AND type = ?`, [videoID, service, type]);
} else {
await db.prepare("run", `INSERT INTO "ratings" ("videoID", "service", "type", "count", "hashedVideoID") VALUES (?, ?, ?, 1, ?)`, [videoID, service, type, hashedVideoID]);
}
// Create entry in privateDB
await privateDB.prepare("run", `INSERT INTO "ratings" ("videoID", "service", "type", "userID", "timeSubmitted", "hashedIP") VALUES (?, ?, ?, ?, ?, ?)`, [videoID, service, type, hashedUserID, Date.now(), hashedIP]);
}
}
return res.sendStatus(200);
} catch (err) {
Logger.error(err as string);
return res.sendStatus(500);
}
}

1
src/types/config.model.ts
View file

@ -34,6 +34,7 @@ export interface SBSConfig {
rateLimit: {
vote: RateLimitConfig;
view: RateLimitConfig;
rate: RateLimitConfig;
};
mysql?: any;
privateMysql?: any;

6
src/types/ratings.model.ts Normal file
View file

@ -0,0 +1,6 @@
export enum RatingType {
Downvote = 0,
Upvote = 1
}
export const RatingTypes = [RatingType.Downvote, RatingType.Upvote];

2
src/utils/getIP.ts
View file

@ -15,6 +15,6 @@ export function getIP(req: Request): IPAddress {
case "X-Real-IP":
return req.headers["x-real-ip"] as IPAddress;
default:
return req.connection.remoteAddress as IPAddress;
return (req.connection?.remoteAddress || req.socket?.remoteAddress) as IPAddress;
}
}

7
src/utils/redisKeys.ts
View file

@ -16,3 +16,10 @@ export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: S
export function reputationKey(userID: UserID): string {
return `reputation.user.${userID}`;
}
export function ratingHashKey(hashPrefix: VideoIDHash, service: Service): string {
hashPrefix = hashPrefix.substring(0, 4) as VideoIDHash;
if (hashPrefix.length !== 4) Logger.warn(`Redis rating hash-prefix key is not length 4! ${hashPrefix}`);
return `rating.${service}.${hashPrefix}`;
}

49
test/cases/ratings/getRating.ts Normal file
View file

@ -0,0 +1,49 @@
import { db } from "../../../src/databases/databases";
import { getHash } from "../../../src/utils/getHash";
import assert from "assert";
import { client } from "../../utils/httpClient";
import { AxiosResponse } from "axios";
import { partialDeepEquals } from "../../utils/partialDeepEquals";
const endpoint = "/api/ratings/rate/";
const getRating = (hash: string, params?: unknown): Promise<AxiosResponse> => client.get(endpoint + hash, { params });
describe("getRating", () => {
before(async () => {
const insertUserNameQuery = 'INSERT INTO "ratings" ("videoID", "service", "type", "count", "hashedVideoID") VALUES (?, ?, ?, ?, ?)';
await db.prepare("run", insertUserNameQuery, ["some-likes-and-dislikes", "YouTube", 0, 5, getHash("some-likes-and-dislikes", 1)]); //b3f0
await db.prepare("run", insertUserNameQuery, ["some-likes-and-dislikes", "YouTube", 1, 10, getHash("some-likes-and-dislikes", 1)]);
});
it("Should be able to get dislikes and likes by default", (done) => {
getRating("b3f0")
.then(res => {
assert.strictEqual(res.status, 200);
const expected = [{
type: 0,
count: 5,
}, {
type: 1,
count: 10,
}];
assert.ok(partialDeepEquals(res.data, expected));
done();
})
.catch(err => done(err));
});
it("Should be able to filter for only dislikes", (done) => {
getRating("b3f0", { type: 0 })
.then(res => {
assert.strictEqual(res.status, 200);
const expected = [{
type: 0,
count: 5,
}];
assert.ok(partialDeepEquals(res.data, expected));
done();
})
.catch(err => done(err));
});
});

96
test/cases/ratings/postRating.ts Normal file
View file

@ -0,0 +1,96 @@
import { db } from "../../../src/databases/databases";
import { getHash } from "../../../src/utils/getHash";
import assert from "assert";
import { client } from "../../utils/httpClient";
import { AxiosResponse } from "axios";
import { partialDeepEquals } from "../../utils/partialDeepEquals";
const endpoint = "/api/ratings/rate/";
const postRating = (body: unknown): Promise<AxiosResponse> => client.post(endpoint, body);
const queryDatabase = (videoID: string) => db.prepare("all", `SELECT * FROM "ratings" WHERE "videoID" = ?`, [videoID]);
describe("postRating", () => {
before(async () => {
const insertUserNameQuery = 'INSERT INTO "ratings" ("videoID", "service", "type", "count", "hashedVideoID") VALUES (?, ?, ?, ?, ?)';
await db.prepare("run", insertUserNameQuery, ["multiple-rates", "YouTube", 0, 3, getHash("multiple-rates", 1)]);
});
it("Should be able to vote on a video", (done) => {
postRating({
userID: "rating-testman",
videoID: "normal-video",
type: 0
})
.then(async res => {
assert.strictEqual(res.status, 200);
const expected = [{
hashedVideoID: getHash("normal-video", 1),
videoID: "normal-video",
type: 0,
count: 1,
service: "YouTube"
}];
assert.ok(partialDeepEquals(await queryDatabase("normal-video"), expected));
done();
})
.catch(err => done(err));
});
it("Should be able to undo a vote on a video", (done) => {
postRating({
userID: "rating-testman",
videoID: "normal-video",
type: 0,
enabled: false
})
.then(async res => {
assert.strictEqual(res.status, 200);
const expected = [{
type: 0,
count: 0
}];
assert.ok(partialDeepEquals(await queryDatabase("normal-video"), expected));
done();
})
.catch(err => done(err));
});
it("Should be able to vote after someone else on a video", (done) => {
postRating({
userID: "rating-testman",
videoID: "multiple-rates",
type: 0
})
.then(async res => {
assert.strictEqual(res.status, 200);
const expected = [{
type: 0,
count: 4
}];
assert.ok(partialDeepEquals(await queryDatabase("multiple-rates"), expected));
done();
})
.catch(err => done(err));
});
it("Should be able to vote a different type than existing votes on a video", (done) => {
postRating({
userID: "rating-testman",
videoID: "multiple-rates",
type: 1
})
.then(async res => {
assert.strictEqual(res.status, 200);
const expected = [{
type: 0,
count: 4
}, {
type: 1,
count: 1
}];
assert.ok(partialDeepEquals(await queryDatabase("multiple-rates"), expected));
done();
})
.catch(err => done(err));
});
});

24
test/test.ts
View file

@ -32,19 +32,21 @@ async function init() {
// Instantiate a Mocha instance.
const mocha = new Mocha();
const testDir = "./test/cases";
const testDirs = ["./test/cases", "./test/cases/ratings"];
// Add each .ts file to the mocha instance
fs.readdirSync(testDir)
.filter((file) =>
// Only keep the .ts files
file.substr(-3) === ".ts"
)
.forEach(function(file) {
mocha.addFile(
path.join(testDir, file)
);
});
testDirs.forEach(testDir => {
fs.readdirSync(testDir)
.filter((file) =>
// Only keep the .ts files
file.substr(-3) === ".ts"
)
.forEach(function(file) {
mocha.addFile(
path.join(testDir, file)
);
});
});
const mockServer = createMockServer(() => {
Logger.info("Started mock HTTP Server");