external/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2024-11-10 01:02:30 +01:00
Code Issues Projects Releases Packages Wiki Activity

Merge branch 'master' of https://github.com/ajayyy/SponsorBlockServer

Browse source 
# Conflicts:
#	src/routes/voteOnSponsorTime.ts
This commit is contained in:
Ajay Ramachandran 2021-04-17 16:25:10 -04:00
commit 346485da8c
26 changed files with 988 additions and 280 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -99,6 +99,7 @@ test/databases/sponsorTimes.db
test/databases/sponsorTimes.db-shm
test/databases/sponsorTimes.db-wal
test/databases/private.db
docker/database-export
# Config files
config.json

2
README.MD
View file

@ -8,7 +8,7 @@ This is the server backend for it
This uses a Postgres or Sqlite database to hold all the timing data.
To make sure that this project doesn't die, I have made the database publicly downloadable at https://sponsor.ajay.app/database.db. You can download a backup or get archive.org to take a backup if you do desire. The database is under [this license](https://creativecommons.org/licenses/by-nc-sa/4.0/) unless you get explicit permission from me.
To make sure that this project doesn't die, I have made the database publicly downloadable at https://sponsor.ajay.app/database. You can download a backup or get archive.org to take a backup if you do desire. The database is under [this license](https://creativecommons.org/licenses/by-nc-sa/4.0/) unless you get explicit permission from me.
Hopefully this project can be combined with projects like [this](https://github.com/Sponsoff/sponsorship_remover) and use this data to create a neural network to predict when sponsored segments happen. That project is sadly abandoned now, so I have decided to attempt to revive this idea.

32
databases/_private_indexes.sql Normal file
View file

@ -0,0 +1,32 @@
-- sponsorTimes
CREATE INDEX IF NOT EXISTS "sponsorTimes_hashedIP"
ON public."sponsorTimes" USING btree
("hashedIP" COLLATE pg_catalog."default" ASC NULLS LAST)
TABLESPACE pg_default;
CREATE INDEX IF NOT EXISTS "privateDB_sponsorTimes_videoID"
ON public."sponsorTimes" USING btree
("videoID" ASC NULLS LAST)
;
-- votes
CREATE INDEX IF NOT EXISTS "votes_userID"
ON public.votes USING btree
("UUID" COLLATE pg_catalog."default" ASC NULLS LAST)
TABLESPACE pg_default;
-- shadowBannedUsers
CREATE INDEX IF NOT EXISTS "shadowBannedUsers_index"
ON public."shadowBannedUsers" USING btree
("userID" COLLATE pg_catalog."default" ASC NULLS LAST)
TABLESPACE pg_default;
-- categoryVotes
CREATE INDEX IF NOT EXISTS "categoryVotes_UUID"
ON public."categoryVotes" USING btree
("UUID" COLLATE pg_catalog."default" ASC NULLS LAST, "userID" COLLATE pg_catalog."default" ASC NULLS LAST, "hashedIP" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST)
TABLESPACE pg_default;

66
databases/_sponsorTimes_indexes.sql Normal file
View file

@ -0,0 +1,66 @@
-- sponsorTimes
CREATE INDEX IF NOT EXISTS "sponsorTime_timeSubmitted"
ON public."sponsorTimes" USING btree
("timeSubmitted" ASC NULLS LAST)
TABLESPACE pg_default;
CREATE INDEX IF NOT EXISTS "sponsorTime_userID"
ON public."sponsorTimes" USING btree
("userID" COLLATE pg_catalog."default" ASC NULLS LAST)
TABLESPACE pg_default;
CREATE INDEX IF NOT EXISTS "sponsorTimes_UUID"
ON public."sponsorTimes" USING btree
("UUID" COLLATE pg_catalog."default" ASC NULLS LAST)
TABLESPACE pg_default;
CREATE INDEX IF NOT EXISTS "sponsorTimes_hashedVideoID_gin"
ON public."sponsorTimes" USING gin
("hashedVideoID" COLLATE pg_catalog."default" gin_trgm_ops, category COLLATE pg_catalog."default" gin_trgm_ops)
TABLESPACE pg_default;
CREATE INDEX IF NOT EXISTS "sponsorTimes_videoID"
ON public."sponsorTimes" USING btree
("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST, "timeSubmitted" ASC NULLS LAST)
TABLESPACE pg_default;
-- userNames
CREATE INDEX IF NOT EXISTS "userNames_userID"
ON public."userNames" USING btree
("userID" COLLATE pg_catalog."default" ASC NULLS LAST)
TABLESPACE pg_default;
-- vipUsers
CREATE INDEX IF NOT EXISTS "vipUsers_index"
ON public."vipUsers" USING btree
("userID" COLLATE pg_catalog."default" ASC NULLS LAST)
TABLESPACE pg_default;
-- warnings
CREATE INDEX IF NOT EXISTS "warnings_index"
ON public.warnings USING btree
("userID" COLLATE pg_catalog."default" ASC NULLS LAST, "issueTime" DESC NULLS LAST, enabled DESC NULLS LAST)
TABLESPACE pg_default;
CREATE INDEX IF NOT EXISTS "warnings_issueTime"
ON public.warnings USING btree
("issueTime" ASC NULLS LAST)
TABLESPACE pg_default;
-- noSegments
CREATE INDEX IF NOT EXISTS "noSegments_videoID"
ON public."noSegments" USING btree
("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST)
TABLESPACE pg_default;
-- categoryVotes
CREATE INDEX IF NOT EXISTS "categoryVotes_UUID_public"
ON public."categoryVotes" USING btree
("UUID" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST)
TABLESPACE pg_default;

30
databases/_upgrade_sponsorTimes_10.sql Normal file
View file

@ -0,0 +1,30 @@
BEGIN TRANSACTION;
/* Add Hidden field */
CREATE TABLE "sqlb_temp_table_10" (
"videoID" TEXT NOT NULL,
"startTime" REAL NOT NULL,
"endTime" REAL NOT NULL,
"votes" INTEGER NOT NULL,
"locked" INTEGER NOT NULL default '0',
"incorrectVotes" INTEGER NOT NULL default '1',
"UUID" TEXT NOT NULL UNIQUE,
"userID" TEXT NOT NULL,
"timeSubmitted" INTEGER NOT NULL,
"views" INTEGER NOT NULL,
"category" TEXT NOT NULL DEFAULT 'sponsor',
"service" TEXT NOT NULL DEFAULT 'YouTube',
"videoDuration" REAL NOT NULL DEFAULT '0',
"hidden" INTEGER NOT NULL DEFAULT '0',
"shadowHidden" INTEGER NOT NULL,
"hashedVideoID" TEXT NOT NULL default ''
);
INSERT INTO sqlb_temp_table_10 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category","service","videoDuration",0,"shadowHidden","hashedVideoID" FROM "sponsorTimes";
DROP TABLE "sponsorTimes";
ALTER TABLE sqlb_temp_table_10 RENAME TO "sponsorTimes";
UPDATE "config" SET value = 10 WHERE key = 'version';
COMMIT;

28
databases/_upgrade_sponsorTimes_7.sql Normal file
View file

@ -0,0 +1,28 @@
BEGIN TRANSACTION;
/* Add Service field */
CREATE TABLE "sqlb_temp_table_7" (
"videoID" TEXT NOT NULL,
"startTime" REAL NOT NULL,
"endTime" REAL NOT NULL,
"votes" INTEGER NOT NULL,
"locked" INTEGER NOT NULL default '0',
"incorrectVotes" INTEGER NOT NULL default '1',
"UUID" TEXT NOT NULL UNIQUE,
"userID" TEXT NOT NULL,
"timeSubmitted" INTEGER NOT NULL,
"views" INTEGER NOT NULL,
"category" TEXT NOT NULL DEFAULT 'sponsor',
"service" TEXT NOT NULL DEFAULT 'YouTube',
"shadowHidden" INTEGER NOT NULL,
"hashedVideoID" TEXT NOT NULL default ''
);
INSERT INTO sqlb_temp_table_7 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category",'YouTube', "shadowHidden","hashedVideoID" FROM "sponsorTimes";
DROP TABLE "sponsorTimes";
ALTER TABLE sqlb_temp_table_7 RENAME TO "sponsorTimes";
UPDATE "config" SET value = 7 WHERE key = 'version';
COMMIT;

29
databases/_upgrade_sponsorTimes_8.sql Normal file
View file

@ -0,0 +1,29 @@
BEGIN TRANSACTION;
/* Add Service field */
CREATE TABLE "sqlb_temp_table_8" (
"videoID" TEXT NOT NULL,
"startTime" REAL NOT NULL,
"endTime" REAL NOT NULL,
"votes" INTEGER NOT NULL,
"locked" INTEGER NOT NULL default '0',
"incorrectVotes" INTEGER NOT NULL default '1',
"UUID" TEXT NOT NULL UNIQUE,
"userID" TEXT NOT NULL,
"timeSubmitted" INTEGER NOT NULL,
"views" INTEGER NOT NULL,
"category" TEXT NOT NULL DEFAULT 'sponsor',
"service" TEXT NOT NULL DEFAULT 'YouTube',
"videoDuration" INTEGER NOT NULL DEFAULT '0',
"shadowHidden" INTEGER NOT NULL,
"hashedVideoID" TEXT NOT NULL default ''
);
INSERT INTO sqlb_temp_table_8 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category","service",'0', "shadowHidden","hashedVideoID" FROM "sponsorTimes";
DROP TABLE "sponsorTimes";
ALTER TABLE sqlb_temp_table_8 RENAME TO "sponsorTimes";
UPDATE "config" SET value = 8 WHERE key = 'version';
COMMIT;

29
databases/_upgrade_sponsorTimes_9.sql Normal file
View file

@ -0,0 +1,29 @@
BEGIN TRANSACTION;
/* Add Service field */
CREATE TABLE "sqlb_temp_table_9" (
"videoID" TEXT NOT NULL,
"startTime" REAL NOT NULL,
"endTime" REAL NOT NULL,
"votes" INTEGER NOT NULL,
"locked" INTEGER NOT NULL default '0',
"incorrectVotes" INTEGER NOT NULL default '1',
"UUID" TEXT NOT NULL UNIQUE,
"userID" TEXT NOT NULL,
"timeSubmitted" INTEGER NOT NULL,
"views" INTEGER NOT NULL,
"category" TEXT NOT NULL DEFAULT 'sponsor',
"service" TEXT NOT NULL DEFAULT 'YouTube',
"videoDuration" REAL NOT NULL DEFAULT '0',
"shadowHidden" INTEGER NOT NULL,
"hashedVideoID" TEXT NOT NULL default ''
);
INSERT INTO sqlb_temp_table_9 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category","service",'0', "shadowHidden","hashedVideoID" FROM "sponsorTimes";
DROP TABLE "sponsorTimes";
ALTER TABLE sqlb_temp_table_9 RENAME TO "sponsorTimes";
UPDATE "config" SET value = 9 WHERE key = 'version';
COMMIT;

1
docker/docker-compose.yml
View file

@ -7,6 +7,7 @@ services:
- database.env
volumes:
- database-data:/var/lib/postgresql/data
- ./database-export/:/opt/exports
ports:
- 127.0.0.1:5432:5432
redis:

87
nginx/nginx.conf
View file

@ -2,7 +2,7 @@ worker_processes 8;
worker_rlimit_nofile 8192;
events {
worker_connections 32768; ## Default: 1024
worker_connections 132768; ## Default: 1024
}
http {
@ -12,19 +12,35 @@ http {
upstream backend_GET {
least_conn;
server localhost:4442;
server localhost:4443;
server localhost:4444;
server localhost:4445;
server localhost:4446;
server localhost:4441;
server localhost:4442;
#server localhost:4443;
#server localhost:4444;
#server localhost:4445;
#server localhost:4446;
#server localhost:4447;
#server localhost:4448;
server 10.0.0.3:4441;
server 10.0.0.3:4442;
#server 134.209.69.251:80 backup;
server 116.203.32.253:80 backup;
#server 116.203.32.253:80;
}
upstream backend_POST {
server localhost:4441;
#server localhost:4441;
#server localhost:4442;
server 10.0.0.3:4441;
#server 10.0.0.3:4442;
}
upstream backend_db {
#server localhost:4441;
server 10.0.0.3:4441;
}
proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=CACHEZONE:10m inactive=60m max_size=40m;
proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=CACHEZONE:10m inactive=60m max_size=400m;
proxy_cache_key "$scheme$request_method$host$request_uri";
add_header X-Cache $upstream_cache_status;
@ -43,14 +59,16 @@ http {
# internal;
#}
#proxy_send_timeout 120s;
location @myerrordirective_500 {
return 502 "Internal Server Error";
return 400 "Internal Server Error";
}
location @myerrordirective_502 {
return 502 "Bad Gateway";
return 400 "Bad Gateway";
}
location @myerrordirective_504 {
return 502 "Gateway Timeout";
return 400 "Gateway Timeout";
}
@ -62,17 +80,16 @@ http {
return 301 https://sb.ltn.fi;
}
location /invidious/ {
proxy_pass https://invidious.fdn.fr/;
}
location /test/ {
proxy_pass http://localhost:4440/;
#proxy_pass https://sbtest.etcinit.com/;
}
location /api/skipSegments {
proxy_pass http://backend_$request_method;
#return 200 "[]";
proxy_pass http://backend_$request_method;
#proxy_cache CACHEZONE;
#proxy_cache_valid 2m;
}
location /api/getTopUsers {
@ -83,24 +100,43 @@ http {
location /api/getTotalStats {
proxy_pass http://backend_GET;
}
#return 200 "";
}
location /api/getVideoSponsorTimes {
proxy_pass http://backend_GET;
}
location = /database.db {
alias /home/sbadmin/sponsor/databases/sponsorTimes.db;
location /database {
proxy_pass http://backend_db;
}
location = /database.db {
#return 404 "Sqlite database has been replaced with csv exports at https://sponsor.ajay.app/database. Sqlite exports might come back soon, but exported at longer intervals.";
#alias /home/sbadmin/sponsor/databases/sponsorTimes.db;
alias /home/sbadmin/test-db/database.db;
}
location = /database/sponsorTimes.csv {
alias /home/sbadmin/sponsorTimes.csv;
}
#location /api/voteOnSponsorTime {
# return 200 "Success";
#}
#location /api/viewedVideoSponsorTime {
# return 200 "Success";
#}
location /api {
proxy_pass http://backend_POST;
}
location / {
root /home/sbadmin/caddy/SponsorBlockSite/public-prod;
root /home/sbadmin/SponsorBlockSite/public-prod;
### CORS
if ($request_method = 'OPTIONS') {
add_header 'Access-Control-Allow-Origin' '*';
@ -132,12 +168,15 @@ http {
}
listen 443 ssl; # managed by Certbot
listen 443 default_server ssl; # managed by Certbot
#listen 80;
ssl_certificate /etc/letsencrypt/live/sponsor.ajay.app/fullchain.pem; # managed by Certbot
ssl_certificate_key /etc/letsencrypt/live/sponsor.ajay.app/privkey.pem; # managed by Certbot
include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
}

6
package-lock.json generated
View file

@ -2989,9 +2989,9 @@
"integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ=="
},
"y18n": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz",
"integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==",
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.1.tgz",
"integrity": "sha512-wNcy4NvjMYL8gogWWYAO7ZFWFfHcbdbE57tZO8e4cbpj8tfUcwrwqSl3ad8HxpYWCdXcJUCeKKZS62Av1affwQ==",
"dev": true
},
"yallist": {

16
src/app.ts
View file

@ -5,7 +5,7 @@ import {oldGetVideoSponsorTimes} from './routes/oldGetVideoSponsorTimes';
import {postSegmentShift} from './routes/postSegmentShift';
import {postWarning} from './routes/postWarning';
import {getIsUserVIP} from './routes/getIsUserVIP';
import {deleteNoSegments} from './routes/deleteNoSegments';
import {deleteNoSegmentsEndpoint} from './routes/deleteNoSegments';
import {postNoSegments} from './routes/postNoSegments';
import {getUserInfo} from './routes/getUserInfo';
import {getDaysSavedFormatted} from './routes/getDaysSavedFormatted';
@ -26,6 +26,7 @@ import {userCounter} from './middleware/userCounter';
import {loggerMiddleware} from './middleware/logger';
import {corsMiddleware} from './middleware/cors';
import {rateLimitMiddleware} from './middleware/requestRateLimit';
import dumpDatabase from './routes/dumpDatabase';
export function createServer(callback: () => void) {
@ -116,7 +117,7 @@ function setupRoutes(app: Express) {
//submit video containing no segments
app.post('/api/noSegments', postNoSegments);
app.delete('/api/noSegments', deleteNoSegments);
app.delete('/api/noSegments', deleteNoSegmentsEndpoint);
//get if user is a vip
app.get('/api/isUserVIP', getIsUserVIP);
@ -127,7 +128,12 @@ function setupRoutes(app: Express) {
//get if user is a vip
app.post('/api/segmentShift', postSegmentShift);
app.get('/database.db', function (req: Request, res: Response) {
res.sendFile("./databases/sponsorTimes.db", {root: "./"});
});
if (config.postgres) {
app.get('/database', (req, res) => dumpDatabase(req, res, true));
app.get('/database.json', (req, res) => dumpDatabase(req, res, false));
} else {
app.get('/database.db', function (req: Request, res: Response) {
res.sendFile("./databases/sponsorTimes.db", {root: "./"});
});
}
}

2
src/config.ts
View file

@ -16,7 +16,7 @@ addDefaults(config, {
privateDBSchema: "./databases/_private.db.sql",
readOnly: false,
webhooks: [],
categoryList: ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"],
categoryList: ["sponsor", "selfpromo", "interaction", "intro", "outro", "preview", "music_offtopic"],
maxNumberOfActiveWarnings: 3,
hoursAfterWarningExpires: 24,
adminUserID: "",

18
src/databases/Postgres.ts
View file

@ -23,6 +23,13 @@ export class Postgres implements IDatabase {
// Upgrade database if required
await this.upgradeDB(this.config.fileNamePrefix, this.config.dbSchemaFolder);
try {
await this.applyIndexes(this.config.fileNamePrefix, this.config.dbSchemaFolder);
} catch (e) {
Logger.warn("Applying indexes failed. See https://github.com/ajayyy/SponsorBlockServer/wiki/Postgres-Extensions for more information.");
Logger.warn(e);
}
}
}
@ -118,9 +125,18 @@ export class Postgres implements IDatabase {
Logger.debug('db update: no file ' + path);
}
private async applyIndexes(fileNamePrefix: string, schemaFolder: string) {
const path = schemaFolder + "/_" + fileNamePrefix + "_indexes.sql";
if (fs.existsSync(path)) {
await this.pool.query(fs.readFileSync(path).toString());
} else {
Logger.debug('failed to apply indexes to ' + fileNamePrefix);
}
}
private processUpgradeQuery(query: string): string {
let result = query;
result = result.replace(/sha256\((.*?)\)/gm, "digest($1, 'sha256')");
result = result.replace(/sha256\((.*?)\)/gm, "encode(digest($1, 'sha256'), 'hex')");
result = result.replace(/integer/gmi, "NUMERIC");
return result;

10
src/middleware/redisKeys.ts
View file

@ -1,5 +1,13 @@
import { Category, VideoID } from "../types/segments.model";
import { Service, VideoID, VideoIDHash } from "../types/segments.model";
import { Logger } from "../utils/logger";
export function skipSegmentsKey(videoID: VideoID): string {
return "segments-" + videoID;
}
export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string {
hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 4) as VideoIDHash;
if (hashedVideoIDPrefix.length !== 4) Logger.warn("Redis skip segment hash-prefix key is not length 4! " + hashedVideoIDPrefix);
return "segments." + service + "." + hashedVideoIDPrefix;
}

29
src/routes/deleteNoSegments.ts
View file

@ -2,12 +2,14 @@ import {Request, Response} from 'express';
import {isUserVIP} from '../utils/isUserVIP';
import {getHash} from '../utils/getHash';
import {db} from '../databases/databases';
import { Category, VideoID } from '../types/segments.model';
import { UserID } from '../types/user.model';
export async function deleteNoSegments(req: Request, res: Response) {
export async function deleteNoSegmentsEndpoint(req: Request, res: Response) {
// Collect user input data
const videoID = req.body.videoID;
let userID = req.body.userID;
const categories = req.body.categories;
const videoID = req.body.videoID as VideoID;
const userID = req.body.userID as UserID;
const categories = req.body.categories as Category[];
// Check input data is valid
if (!videoID
@ -23,8 +25,8 @@ export async function deleteNoSegments(req: Request, res: Response) {
}
// Check if user is VIP
userID = getHash(userID);
const userIsVIP = await isUserVIP(userID);
const hashedUserID = getHash(userID);
const userIsVIP = await isUserVIP(hashedUserID);
if (!userIsVIP) {
res.status(403).json({
@ -33,13 +35,22 @@ export async function deleteNoSegments(req: Request, res: Response) {
return;
}
deleteNoSegments(videoID, categories);
res.status(200).json({message: 'Removed no segments entrys for video ' + videoID});
}
/**
*
* @param videoID
* @param categories If null, will remove all
*/
export async function deleteNoSegments(videoID: VideoID, categories: Category[]): Promise<void> {
const entries = (await db.prepare("all", 'SELECT * FROM "noSegments" WHERE "videoID" = ?', [videoID])).filter((entry: any) => {
return (categories.indexOf(entry.category) !== -1);
return categories === null || categories.indexOf(entry.category) !== -1;
});
for (const entry of entries) {
await db.prepare('run', 'DELETE FROM "noSegments" WHERE "videoID" = ? AND "category" = ?', [videoID, entry.category]);
}
res.status(200).json({message: 'Removed no segments entrys for video ' + videoID});
}

78
src/routes/dumpDatabase.ts Normal file
View file

@ -0,0 +1,78 @@
import {db} from '../databases/databases';
import {Logger} from '../utils/logger';
import {Request, Response} from 'express';
import { config } from '../config';
const ONE_MINUTE = 1000 * 60;
const styleHeader = `<style>body{font-family: sans-serif}</style>`
const licenseHeader = `<p>The API and database follow <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" rel="nofollow">CC BY-NC-SA 4.0</a> unless you have explicit permission.</p>
<p><a href="https://gist.github.com/ajayyy/4b27dfc66e33941a45aeaadccb51de71">Attribution Template</a></p>
<p>If you need to use the database or API in a way that violates this license, contact me with your reason and I may grant you access under a different license.</p></a></p>`;
const tables = [{
name: "sponsorTimes",
order: "timeSubmitted"
},
{
name: "userNames"
},
{
name: "categoryVotes"
},
{
name: "noSegments",
},
{
name: "warnings",
order: "issueTime"
},
{
name: "vipUsers"
}];
const links: string[] = tables.map((table) => `/database/${table.name}.csv`);
const linksHTML: string = tables.map((table) => `<p><a href="/database/${table.name}.csv">${table.name}.csv</a></p>`)
.reduce((acc, url) => acc + url, "");
let lastUpdate = 0;
export default function dumpDatabase(req: Request, res: Response, showPage: boolean) {
if (!config.postgres) {
res.status(404).send("Not supported on this instance");
return;
}
const now = Date.now();
const updateQueued = now - lastUpdate > ONE_MINUTE;
res.status(200)
if (showPage) {
res.send(`${styleHeader}
<h1>SponsorBlock database dumps</h1>${licenseHeader}
<h3>How this works</h3>
Send a request to <code>https://sponsor.ajay.app/database.json</code>, or visit this page to trigger the database dump to run.
Then, you can download the csv files below, or use the links returned from the JSON request.
<h3>Links</h3>
${linksHTML}<br/>
${updateQueued ? `Update queued.` : ``} Last updated: ${lastUpdate ? new Date(lastUpdate).toUTCString() : `Unknown`}`);
} else {
res.send({
lastUpdated: lastUpdate,
updateQueued,
links
})
}
if (updateQueued) {
lastUpdate = Date.now();
for (const table of tables) {
db.prepare('run', `COPY (SELECT * FROM "${table.name}"${table.order ? ` ORDER BY "${table.order}"` : ``})
TO '/opt/exports/${table.name}.csv' WITH (FORMAT CSV, HEADER true);`);
}
}
}

65
src/routes/getSkipSegments.ts
View file

@ -2,9 +2,9 @@ import { Request, Response } from 'express';
import { RedisClient } from 'redis';
import { config } from '../config';
import { db, privateDB } from '../databases/databases';
import { skipSegmentsKey } from '../middleware/redisKeys';
import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys';
import { SBRecord } from '../types/lib.model';
import { Category, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model";
import { Category, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, Service, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model";
import { getHash } from '../utils/getHash';
import { getIP } from '../utils/getIP';
import { Logger } from '../utils/logger';
@ -43,11 +43,12 @@ async function prepareCategorySegments(req: Request, videoID: VideoID, category:
return chooseSegments(filteredSegments).map((chosenSegment) => ({
category,
segment: [chosenSegment.startTime, chosenSegment.endTime],
UUID: chosenSegment.UUID
UUID: chosenSegment.UUID,
videoDuration: chosenSegment.videoDuration
}));
}
async function getSegmentsByVideoID(req: Request, videoID: string, categories: Category[]): Promise<Segment[]> {
async function getSegmentsByVideoID(req: Request, videoID: string, categories: Category[], service: Service): Promise<Segment[]> {
const cache: SegmentCache = {shadowHiddenSegmentIPs: {}};
const segments: Segment[] = [];
@ -58,9 +59,9 @@ async function getSegmentsByVideoID(req: Request, videoID: string, categories: C
const segmentsByCategory: SBRecord<Category, DBSegment[]> = (await db
.prepare(
'all',
`SELECT "startTime", "endTime", "votes", "locked", "UUID", "category", "shadowHidden" FROM "sponsorTimes"
WHERE "videoID" = ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) ORDER BY "startTime"`,
[videoID]
`SELECT "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden" FROM "sponsorTimes"
WHERE "videoID" = ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`,
[videoID, service]
)).reduce((acc: SBRecord<Category, DBSegment[]>, segment: DBSegment) => {
acc[segment.category] = acc[segment.category] || [];
acc[segment.category].push(segment);
@ -81,7 +82,7 @@ async function getSegmentsByVideoID(req: Request, videoID: string, categories: C
}
}
async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash, categories: Category[]): Promise<SBRecord<VideoID, VideoData>> {
async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash, categories: Category[], service: Service): Promise<SBRecord<VideoID, VideoData>> {
const cache: SegmentCache = {shadowHiddenSegmentIPs: {}};
const segments: SBRecord<VideoID, VideoData> = {};
@ -91,13 +92,9 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash,
categories = categories.filter((category) => !(/[^a-z|_|-]/.test(category)));
if (categories.length === 0) return null;
const segmentPerVideoID: SegmentWithHashPerVideoID = (await db
.prepare(
'all',
`SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "category", "shadowHidden", "hashedVideoID" FROM "sponsorTimes"
WHERE "hashedVideoID" LIKE ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) ORDER BY "startTime"`,
[hashedVideoIDPrefix + '%']
)).reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => {
const segmentPerVideoID: SegmentWithHashPerVideoID = (await getSegmentsFromDB(hashedVideoIDPrefix, service))
.filter((segment: DBSegment) => categories.includes(segment?.category))
.reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => {
acc[segment.videoID] = acc[segment.videoID] || {
hash: segment.hashedVideoID,
segmentPerCategory: {},
@ -130,6 +127,37 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash,
}
}
async function getSegmentsFromDB(hashedVideoIDPrefix: VideoIDHash, service: Service): Promise<DBSegment[]> {
const fetchFromDB = () => db
.prepare(
'all',
`SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden", "hashedVideoID" FROM "sponsorTimes"
WHERE "hashedVideoID" LIKE ? AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`,
[hashedVideoIDPrefix + '%', service]
);
if (hashedVideoIDPrefix.length === 4) {
const key = skipSegmentsHashKey(hashedVideoIDPrefix, service);
const {err, reply} = await redis.getAsync(key);
if (!err && reply) {
try {
Logger.debug("Got data from redis: " + reply);
return JSON.parse(reply);
} catch (e) {
// If all else, continue on to fetching from the database
}
}
const data = await fetchFromDB();
redis.setAsync(key, JSON.stringify(data));
return data;
}
return await fetchFromDB();
}
//gets a weighted random choice from the choices array based on their `votes` property.
//amountOfChoices specifies the maximum amount of choices to return, 1 or more.
//choices are unique
@ -239,6 +267,11 @@ async function handleGetSegments(req: Request, res: Response): Promise<Segment[]
? [req.query.category]
: ['sponsor'];
let service: Service = req.query.service ?? req.body.service ?? Service.YouTube;
if (!Object.values(Service).some((val) => val == service)) {
service = Service.YouTube;
}
// Only 404s are cached at the moment
const redisResult = await redis.getAsync(skipSegmentsKey(videoID));
@ -251,7 +284,7 @@ async function handleGetSegments(req: Request, res: Response): Promise<Segment[]
}
}
const segments = await getSegmentsByVideoID(req, videoID, categories);
const segments = await getSegmentsByVideoID(req, videoID, categories, service);
if (segments === null || segments === undefined) {
res.sendStatus(500);

9
src/routes/getSkipSegmentsByHash.ts
View file

@ -1,7 +1,7 @@
import {hashPrefixTester} from '../utils/hashPrefixTester';
import {getSegmentsByHash} from './getSkipSegments';
import {Request, Response} from 'express';
import { Category, VideoIDHash } from '../types/segments.model';
import { Category, Service, VideoIDHash } from '../types/segments.model';
export async function getSkipSegmentsByHash(req: Request, res: Response) {
let hashPrefix = req.params.prefix as VideoIDHash;
@ -25,12 +25,17 @@ export async function getSkipSegmentsByHash(req: Request, res: Response) {
catch(error) {
return res.status(400).send("Bad parameter: categories (invalid JSON)");
}
let service: Service = req.query.service ?? req.body.service ?? Service.YouTube;
if (!Object.values(Service).some((val) => val == service)) {
service = Service.YouTube;
}
// filter out none string elements, only flat array with strings is valid
categories = categories.filter((item: any) => typeof item === "string");
// Get all video id's that match hash prefix
const segments = await getSegmentsByHash(req, hashPrefix, categories);
const segments = await getSegmentsByHash(req, hashPrefix, categories, service);
if (!segments) return res.status(404).json([]);

375
src/routes/postSkipSegments.ts
View file

@ -11,9 +11,15 @@ import {getFormattedTime} from '../utils/getFormattedTime';
import {isUserTrustworthy} from '../utils/isUserTrustworthy';
import {dispatchEvent} from '../utils/webhookUtils';
import {Request, Response} from 'express';
import { skipSegmentsKey } from '../middleware/redisKeys';
import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys';
import redis from '../utils/redis';
import { Category, IncomingSegment, Segment, SegmentUUID, Service, VideoDuration, VideoID } from '../types/segments.model';
import { deleteNoSegments } from './deleteNoSegments';
interface APIVideoInfo {
err: string | boolean,
data: any
}
async function sendWebhookNotification(userID: string, videoID: string, UUID: string, submissionCount: number, youtubeData: any, {submissionStart, submissionEnd}: { submissionStart: number; submissionEnd: number; }, segmentInfo: any) {
const row = await db.prepare('get', `SELECT "userName" FROM "userNames" WHERE "userID" = ?`, [userID]);
@ -45,62 +51,58 @@ async function sendWebhookNotification(userID: string, videoID: string, UUID: st
});
}
async function sendWebhooks(userID: string, videoID: string, UUID: string, segmentInfo: any) {
if (config.youtubeAPIKey !== null) {
async function sendWebhooks(apiVideoInfo: APIVideoInfo, userID: string, videoID: string, UUID: string, segmentInfo: any, service: Service) {
if (apiVideoInfo && service == Service.YouTube) {
const userSubmissionCountRow = await db.prepare('get', `SELECT count(*) as "submissionCount" FROM "sponsorTimes" WHERE "userID" = ?`, [userID]);
YouTubeAPI.listVideos(videoID, (err: any, data: any) => {
if (err || data.items.length === 0) {
err && Logger.error(err);
return;
const {data, err} = apiVideoInfo;
if (err) return;
const startTime = parseFloat(segmentInfo.segment[0]);
const endTime = parseFloat(segmentInfo.segment[1]);
sendWebhookNotification(userID, videoID, UUID, userSubmissionCountRow.submissionCount, data, {
submissionStart: startTime,
submissionEnd: endTime,
}, segmentInfo);
// If it is a first time submission
// Then send a notification to discord
if (config.discordFirstTimeSubmissionsWebhookURL === null || userSubmissionCountRow.submissionCount > 1) return;
fetch(config.discordFirstTimeSubmissionsWebhookURL, {
method: 'POST',
body: JSON.stringify({
"embeds": [{
"title": data.items[0].snippet.title,
"url": "https://www.youtube.com/watch?v=" + videoID + "&t=" + (parseInt(startTime.toFixed(0)) - 2),
"description": "Submission ID: " + UUID +
"\n\nTimestamp: " +
getFormattedTime(startTime) + " to " + getFormattedTime(endTime) +
"\n\nCategory: " + segmentInfo.category,
"color": 10813440,
"author": {
"name": userID,
},
"thumbnail": {
"url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "",
},
}],
}),
headers: {
'Content-Type': 'application/json'
}
const startTime = parseFloat(segmentInfo.segment[0]);
const endTime = parseFloat(segmentInfo.segment[1]);
sendWebhookNotification(userID, videoID, UUID, userSubmissionCountRow.submissionCount, data, {
submissionStart: startTime,
submissionEnd: endTime,
}, segmentInfo);
// If it is a first time submission
// Then send a notification to discord
if (config.discordFirstTimeSubmissionsWebhookURL === null || userSubmissionCountRow.submissionCount > 1) return;
fetch(config.discordFirstTimeSubmissionsWebhookURL, {
method: 'POST',
body: JSON.stringify({
"embeds": [{
"title": data.items[0].snippet.title,
"url": "https://www.youtube.com/watch?v=" + videoID + "&t=" + (parseInt(startTime.toFixed(0)) - 2),
"description": "Submission ID: " + UUID +
"\n\nTimestamp: " +
getFormattedTime(startTime) + " to " + getFormattedTime(endTime) +
"\n\nCategory: " + segmentInfo.category,
"color": 10813440,
"author": {
"name": userID,
},
"thumbnail": {
"url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "",
},
}],
}),
headers: {
'Content-Type': 'application/json'
}
})
.then(res => {
if (res.status >= 400) {
Logger.error("Error sending first time submission Discord hook");
Logger.error(JSON.stringify(res));
Logger.error("\n");
}
})
.catch(err => {
Logger.error("Failed to send first time submission Discord hook.");
Logger.error(JSON.stringify(err));
})
.then(res => {
if (res.status >= 400) {
Logger.error("Error sending first time submission Discord hook");
Logger.error(JSON.stringify(res));
Logger.error("\n");
});
}
})
.catch(err => {
Logger.error("Failed to send first time submission Discord hook.");
Logger.error(JSON.stringify(err));
Logger.error("\n");
});
}
}
@ -166,73 +168,98 @@ async function sendWebhooksNB(userID: string, videoID: string, UUID: string, sta
// Looks like this was broken for no defined youtube key - fixed but IMO we shouldn't return
// false for a pass - it was confusing and lead to this bug - any use of this function in
// the future could have the same problem.
async function autoModerateSubmission(submission: { videoID: any; userID: any; segments: any }) {
// Get the video information from the youtube API
if (config.youtubeAPIKey !== null) {
const {err, data} = await new Promise((resolve) => {
YouTubeAPI.listVideos(submission.videoID, (err: any, data: any) => resolve({err, data}));
});
async function autoModerateSubmission(apiVideoInfo: APIVideoInfo,
submission: { videoID: any; userID: any; segments: any }) {
if (apiVideoInfo) {
const {err, data} = apiVideoInfo;
if (err) return false;
if (err) {
return false;
} else {
// Check to see if video exists
if (data.pageInfo.totalResults === 0) {
return "No video exists with id " + submission.videoID;
// Check to see if video exists
if (data.pageInfo.totalResults === 0) return "No video exists with id " + submission.videoID;
const duration = getYouTubeVideoDuration(apiVideoInfo);
const segments = submission.segments;
let nbString = "";
for (let i = 0; i < segments.length; i++) {
const startTime = parseFloat(segments[i].segment[0]);
const endTime = parseFloat(segments[i].segment[1]);
if (duration == 0) {
// Allow submission if the duration is 0 (bug in youtube api)
return false;
} else {
const segments = submission.segments;
let nbString = "";
for (let i = 0; i < segments.length; i++) {
if (segments[i].category === "sponsor") {
//Prepare timestamps to send to NB all at once
nbString = nbString + segments[i].segment[0] + "," + segments[i].segment[1] + ";";
}
}
}
// Get all submissions for this user
const allSubmittedByUser = await db.prepare('all', `SELECT "startTime", "endTime" FROM "sponsorTimes" WHERE "userID" = ? and "videoID" = ? and "votes" > -1`, [submission.userID, submission.videoID]);
const allSegmentTimes = [];
if (allSubmittedByUser !== undefined) {
//add segments the user has previously submitted
for (const segmentInfo of allSubmittedByUser) {
allSegmentTimes.push([parseFloat(segmentInfo.startTime), parseFloat(segmentInfo.endTime)]);
}
}
//add segments they are trying to add in this submission
for (let i = 0; i < segments.length; i++) {
let startTime = parseFloat(segments[i].segment[0]);
let endTime = parseFloat(segments[i].segment[1]);
allSegmentTimes.push([startTime, endTime]);
}
//merge all the times into non-overlapping arrays
const allSegmentsSorted = mergeTimeSegments(allSegmentTimes.sort(function (a, b) {
return a[0] - b[0] || a[1] - b[1];
}));
let videoDuration = data.items[0].contentDetails.duration;
videoDuration = isoDurations.toSeconds(isoDurations.parse(videoDuration));
if (videoDuration != 0) {
let allSegmentDuration = 0;
//sum all segment times together
allSegmentsSorted.forEach(segmentInfo => allSegmentDuration += segmentInfo[1] - segmentInfo[0]);
if (allSegmentDuration > (videoDuration / 100) * 80) {
// Reject submission if all segments combine are over 80% of the video
return "Total length of your submitted segments are over 80% of the video.";
}
}
// Check NeuralBlock
const neuralBlockURL = config.neuralBlockURL;
if (!neuralBlockURL) return false;
const response = await fetch(neuralBlockURL + "/api/checkSponsorSegments?vid=" + submission.videoID +
"&segments=" + nbString.substring(0, nbString.length - 1));
if (!response.ok) return false;
const nbPredictions = await response.json();
let nbDecision = false;
let predictionIdx = 0; //Keep track because only sponsor categories were submitted
for (let i = 0; i < segments.length; i++) {
if (segments[i].category === "sponsor") {
if (nbPredictions.probabilities[predictionIdx] < 0.70) {
nbDecision = true; // At least one bad entry
const startTime = parseFloat(segments[i].segment[0]);
const endTime = parseFloat(segments[i].segment[1]);
let duration = data.items[0].contentDetails.duration;
duration = isoDurations.toSeconds(isoDurations.parse(duration));
if (duration == 0) {
// Allow submission if the duration is 0 (bug in youtube api)
return false;
} else if ((endTime - startTime) > (duration / 100) * 80) {
// Reject submission if over 80% of the video
return "One of your submitted segments is over 80% of the video.";
} else {
if (segments[i].category === "sponsor") {
//Prepare timestamps to send to NB all at once
nbString = nbString + segments[i].segment[0] + "," + segments[i].segment[1] + ";";
}
}
}
// Check NeuralBlock
const neuralBlockURL = config.neuralBlockURL;
if (!neuralBlockURL) return false;
const response = await fetch(neuralBlockURL + "/api/checkSponsorSegments?vid=" + submission.videoID +
"&segments=" + nbString.substring(0, nbString.length - 1));
if (!response.ok) return false;
const nbPredictions = await response.json();
let nbDecision = false;
let predictionIdx = 0; //Keep track because only sponsor categories were submitted
for (let i = 0; i < segments.length; i++) {
if (segments[i].category === "sponsor") {
if (nbPredictions.probabilities[predictionIdx] < 0.70) {
nbDecision = true; // At least one bad entry
const startTime = parseFloat(segments[i].segment[0]);
const endTime = parseFloat(segments[i].segment[1]);
const UUID = getSubmissionUUID(submission.videoID, segments[i].category, submission.userID, startTime, endTime);
// Send to Discord
// Note, if this is too spammy. Consider sending all the segments as one Webhook
sendWebhooksNB(submission.userID, submission.videoID, UUID, startTime, endTime, segments[i].category, nbPredictions.probabilities[predictionIdx], data);
}
predictionIdx++;
}
}
if (nbDecision) {
return "Rejected based on NeuralBlock predictions.";
} else {
return false;
const UUID = getSubmissionUUID(submission.videoID, segments[i].category, submission.userID, startTime, endTime);
// Send to Discord
// Note, if this is too spammy. Consider sending all the segments as one Webhook
sendWebhooksNB(submission.userID, submission.videoID, UUID, startTime, endTime, segments[i].category, nbPredictions.probabilities[predictionIdx], data);
}
predictionIdx++;
}
}
if (nbDecision) {
return "Rejected based on NeuralBlock predictions.";
} else {
return false;
}
} else {
Logger.debug("Skipped YouTube API");
@ -243,6 +270,21 @@ async function autoModerateSubmission(submission: { videoID: any; userID: any; s
}
}
function getYouTubeVideoDuration(apiVideoInfo: APIVideoInfo): VideoDuration {
const duration = apiVideoInfo?.data?.items[0]?.contentDetails?.duration;
return duration ? isoDurations.toSeconds(isoDurations.parse(duration)) as VideoDuration : null;
}
async function getYouTubeVideoInfo(videoID: VideoID): Promise<APIVideoInfo> {
if (config.youtubeAPIKey !== null) {
return new Promise((resolve) => {
YouTubeAPI.listVideos(videoID, (err: any, data: any) => resolve({err, data}));
});
} else {
return null;
}
}
function proxySubmission(req: Request) {
fetch(config.proxySubmission + '/api/skipSegments?userID=' + req.query.userID + '&videoID=' + req.query.videoID, {
method: 'POST',
@ -267,14 +309,18 @@ export async function postSkipSegments(req: Request, res: Response) {
const videoID = req.query.videoID || req.body.videoID;
let userID = req.query.userID || req.body.userID;
let service: Service = req.query.service ?? req.body.service ?? Service.YouTube;
if (!Object.values(Service).some((val) => val == service)) {
service = Service.YouTube;
}
let videoDuration: VideoDuration = (parseFloat(req.query.videoDuration || req.body.videoDuration) || 0) as VideoDuration;
let segments = req.body.segments;
let segments = req.body.segments as IncomingSegment[];
if (segments === undefined) {
// Use query instead
segments = [{
segment: [req.query.startTime, req.query.endTime],
category: req.query.category,
segment: [req.query.startTime as string, req.query.endTime as string],
category: req.query.category as Category
}];
}
@ -312,7 +358,7 @@ export async function postSkipSegments(req: Request, res: Response) {
return res.status(403).send('Submission rejected due to a warning from a moderator. This means that we noticed you were making some common mistakes that are not malicious, and we just want to clarify the rules. Could you please send a message in Discord or Matrix so we can further help you?');
}
const noSegmentList = (await db.prepare('all', 'SELECT category from "noSegments" where "videoID" = ?', [videoID])).map((list: any) => {
let noSegmentList = (await db.prepare('all', 'SELECT category from "noSegments" where "videoID" = ?', [videoID])).map((list: any) => {
return list.category;
});
@ -321,6 +367,32 @@ export async function postSkipSegments(req: Request, res: Response) {
const decreaseVotes = 0;
let apiVideoInfo: APIVideoInfo = null;
if (service == Service.YouTube) {
apiVideoInfo = await getYouTubeVideoInfo(videoID);
}
const apiVideoDuration = getYouTubeVideoDuration(apiVideoInfo);
if (!videoDuration || (apiVideoDuration && Math.abs(videoDuration - apiVideoDuration) > 2)) {
// If api duration is far off, take that one instead (it is only precise to seconds, not millis)
videoDuration = apiVideoDuration || 0 as VideoDuration;
}
const previousSubmissions = await db.prepare('all', `SELECT "videoDuration", "UUID" FROM "sponsorTimes" WHERE "videoID" = ? AND "service" = ? AND "hidden" = 0
AND "shadowHidden" = 0 AND "votes" >= 0 AND "videoDuration" != 0`, [videoID, service]) as
{videoDuration: VideoDuration, UUID: SegmentUUID}[];
// If the video's duration is changed, then the video should be unlocked and old submissions should be hidden
const videoDurationChanged = previousSubmissions.length > 0 && !previousSubmissions.some((e) => Math.abs(videoDuration - e.videoDuration) < 2);
if (videoDurationChanged) {
// Hide all previous submissions
for (const submission of previousSubmissions) {
await db.prepare('run', `UPDATE "sponsorTimes" SET "hidden" = 1 WHERE "UUID" = ?`, [submission.UUID]);
}
// Reset no segments
noSegmentList = [];
deleteNoSegments(videoID, null);
}
// Check if all submissions are correct
for (let i = 0; i < segments.length; i++) {
if (segments[i] === undefined || segments[i].segment === undefined || segments[i].category === undefined) {
@ -343,7 +415,7 @@ export async function postSkipSegments(req: Request, res: Response) {
+ segments[i].category + "'. A moderator has decided that no new segments are needed and that all current segments of this category are timed perfectly.\n\n "
+ (segments[i].category === "sponsor" ? "Maybe the segment you are submitting is a different category that you have not enabled and is not a sponsor. " +
"Categories that aren't sponsor, such as self-promotion can be enabled in the options.\n\n " : "")
+ "If you believe this is incorrect, please contact someone on Discord.",
+ "If you believe this is incorrect, please contact someone on discord.gg/SponsorBlock or matrix.to/#/+sponsorblock:ajay.app",
);
return;
}
@ -367,7 +439,7 @@ export async function postSkipSegments(req: Request, res: Response) {
//check if this info has already been submitted before
const duplicateCheck2Row = await db.prepare('get', `SELECT COUNT(*) as count FROM "sponsorTimes" WHERE "startTime" = ?
and "endTime" = ? and "category" = ? and "videoID" = ?`, [startTime, endTime, segments[i].category, videoID]);
and "endTime" = ? and "category" = ? and "videoID" = ? and "service" = ?`, [startTime, endTime, segments[i].category, videoID, service]);
if (duplicateCheck2Row.count > 0) {
res.sendStatus(409);
return;
@ -375,8 +447,8 @@ export async function postSkipSegments(req: Request, res: Response) {
}
// Auto moderator check
if (!isVIP) {
const autoModerateResult = await autoModerateSubmission({userID, videoID, segments});//startTime, endTime, category: segments[i].category});
if (!isVIP && service == Service.YouTube) {
const autoModerateResult = await autoModerateSubmission(apiVideoInfo, {userID, videoID, segments});//startTime, endTime, category: segments[i].category});
if (autoModerateResult == "Rejected based on NeuralBlock predictions.") {
// If NB automod rejects, the submission will start with -2 votes.
// Note, if one submission is bad all submissions will be affected.
@ -437,63 +509,19 @@ export async function postSkipSegments(req: Request, res: Response) {
let startingVotes = 0 + decreaseVotes;
if (config.youtubeAPIKey !== null) {
let {err, data} = await new Promise((resolve) => {
YouTubeAPI.listVideos(videoID, (err: any, data: any) => resolve({err, data}));
});
if (err) {
Logger.error("Error while submitting when connecting to YouTube API: " + err);
} else {
//get all segments for this video and user
const allSubmittedByUser = await db.prepare('all', `SELECT "startTime", "endTime" FROM "sponsorTimes" WHERE "userID" = ? and "videoID" = ? and "votes" > -1`, [userID, videoID]);
const allSegmentTimes = [];
if (allSubmittedByUser !== undefined) {
//add segments the user has previously submitted
for (const segmentInfo of allSubmittedByUser) {
allSegmentTimes.push([parseFloat(segmentInfo.startTime), parseFloat(segmentInfo.endTime)]);
}
}
//add segments they are trying to add in this submission
for (let i = 0; i < segments.length; i++) {
let startTime = parseFloat(segments[i].segment[0]);
let endTime = parseFloat(segments[i].segment[1]);
allSegmentTimes.push([startTime, endTime]);
}
//merge all the times into non-overlapping arrays
const allSegmentsSorted = mergeTimeSegments(allSegmentTimes.sort(function (a, b) {
return a[0] - b[0] || a[1] - b[1];
}));
let videoDuration = data.items[0].contentDetails.duration;
videoDuration = isoDurations.toSeconds(isoDurations.parse(videoDuration));
if (videoDuration != 0) {
let allSegmentDuration = 0;
//sum all segment times together
allSegmentsSorted.forEach(segmentInfo => allSegmentDuration += segmentInfo[1] - segmentInfo[0]);
if (allSegmentDuration > (videoDuration / 100) * 80) {
// Reject submission if all segments combine are over 80% of the video
res.status(400).send("Total length of your submitted segments are over 80% of the video.");
return;
}
}
}
}
for (const segmentInfo of segments) {
//this can just be a hash of the data
//it's better than generating an actual UUID like what was used before
//also better for duplication checking
const UUID = getSubmissionUUID(videoID, segmentInfo.category, userID, segmentInfo.segment[0], segmentInfo.segment[1]);
const UUID = getSubmissionUUID(videoID, segmentInfo.category, userID, parseFloat(segmentInfo.segment[0]), parseFloat(segmentInfo.segment[1]));
const hashedVideoID = getHash(videoID, 1);
const startingLocked = isVIP ? 1 : 0;
try {
await db.prepare('run', `INSERT INTO "sponsorTimes"
("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "shadowHidden", "hashedVideoID")
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, shadowBanned, getHash(videoID, 1),
("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "service", "videoDuration", "shadowHidden", "hashedVideoID")
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, service, videoDuration, shadowBanned, hashedVideoID,
],
);
@ -502,9 +530,10 @@ export async function postSkipSegments(req: Request, res: Response) {
// Clear redis cache for this video
redis.delAsync(skipSegmentsKey(videoID));
redis.delAsync(skipSegmentsHashKey(hashedVideoID, service));
} catch (err) {
//a DB change probably occurred
res.sendStatus(502);
res.sendStatus(500);
Logger.error("Error when putting sponsorTime in the DB: " + videoID + ", " + segmentInfo.segment[0] + ", " +
segmentInfo.segment[1] + ", " + userID + ", " + segmentInfo.category + ". " + err);
@ -529,7 +558,7 @@ export async function postSkipSegments(req: Request, res: Response) {
res.json(newSegments);
for (let i = 0; i < segments.length; i++) {
sendWebhooks(userID, videoID, UUIDs[i], segments[i]);
sendWebhooks(apiVideoInfo, userID, videoID, UUIDs[i], segments[i], service);
}
}

47
src/routes/voteOnSponsorTime.ts
View file

@ -12,8 +12,8 @@ import {getHash} from '../utils/getHash';
import {config} from '../config';
import { UserID } from '../types/user.model';
import redis from '../utils/redis';
import { skipSegmentsKey } from '../middleware/redisKeys';
import { VideoID } from '../types/segments.model';
import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys';
import { Category, HashedIP, IPAddress, SegmentUUID, Service, VideoID, VideoIDHash } from '../types/segments.model';
const voteTypes = {
normal: 0,
@ -147,8 +147,8 @@ async function sendWebhooks(voteData: VoteData) {
}
}
async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnSubmission: boolean, category: string
, hashedIP: string, finalResponse: FinalResponse, res: Response) {
async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, isOwnSubmission: boolean, category: Category
, hashedIP: HashedIP, finalResponse: FinalResponse, res: Response) {
// Check if they've already made a vote
const usersLastVoteInfo = await privateDB.prepare('get', `select count(*) as votes, category from "categoryVotes" where "UUID" = ? and "userID" = ? group by category`, [UUID, userID]);
@ -158,8 +158,9 @@ async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnS
return;
}
const currentCategory = await db.prepare('get', `select category from "sponsorTimes" where "UUID" = ?`, [UUID]);
if (!currentCategory) {
const videoInfo = (await db.prepare('get', `SELECT "category", "videoID", "hashedVideoID", "service" FROM "sponsorTimes" WHERE "UUID" = ?`,
[UUID])) as {category: Category, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service};
if (!videoInfo) {
// Submission doesn't exist
res.status(400).send("Submission doesn't exist.");
return;
@ -198,7 +199,7 @@ async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnS
}
// See if the submissions category is ready to change
const currentCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, currentCategory.category]);
const currentCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, videoInfo.category]);
const submissionInfo = await db.prepare("get", `SELECT "userID", "timeSubmitted", "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]);
const isSubmissionVIP = submissionInfo && await isUserVIP(submissionInfo.userID);
@ -210,9 +211,9 @@ async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnS
// Add submission as vote
if (!currentCategoryInfo && submissionInfo) {
await db.prepare("run", `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, currentCategory.category, currentCategoryCount]);
await db.prepare("run", `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, videoInfo.category, currentCategoryCount]);
await privateDB.prepare("run", `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, submissionInfo.userID, "unknown", currentCategory.category, submissionInfo.timeSubmitted]);
await privateDB.prepare("run", `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, submissionInfo.userID, "unknown", videoInfo.category, submissionInfo.timeSubmitted]);
}
const nextCategoryCount = (nextCategoryInfo?.votes || 0) + voteAmount;
@ -225,6 +226,8 @@ async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnS
}
}
clearRedisCache(videoInfo);
res.sendStatus(finalResponse.finalStatus);
}
@ -233,10 +236,10 @@ export function getUserID(req: Request): UserID {
}
export async function voteOnSponsorTime(req: Request, res: Response) {
const UUID = req.query.UUID as string;
const UUID = req.query.UUID as SegmentUUID;
const paramUserID = getUserID(req);
let type = req.query.type !== undefined ? parseInt(req.query.type as string) : undefined;
const category = req.query.category as string;
const category = req.query.category as Category;
if (UUID === undefined || paramUserID === undefined || (type === undefined && category === undefined)) {
//invalid request
@ -258,7 +261,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
const ip = getIP(req);
//hash the ip 5000 times so no one can get it from the database
const hashedIP = getHash(ip + config.globalSalt);
const hashedIP: HashedIP = getHash((ip + config.globalSalt) as IPAddress);
//check if this user is on the vip list
const isVIP = (await db.prepare('get', `SELECT count(*) as "userCount" FROM "vipUsers" WHERE "userID" = ?`, [nonAnonUserID])).userCount > 0;
@ -353,13 +356,13 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
}
//check if the increment amount should be multiplied (downvotes have more power if there have been many views)
const row = await db.prepare('get', `SELECT "videoID", votes, views FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]) as
{videoID: VideoID, votes: number, views: number};
const videoInfo = await db.prepare('get', `SELECT "videoID", "hashedVideoID", "service", "votes", "views" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]) as
{videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, votes: number, views: number};
if (voteTypeEnum === voteTypes.normal) {
if ((isVIP || isOwnSubmission) && incrementAmount < 0) {
//this user is a vip and a downvote
incrementAmount = -(row.votes + 2 - oldIncrementAmount);
incrementAmount = -(videoInfo.votes + 2 - oldIncrementAmount);
type = incrementAmount;
}
} else if (voteTypeEnum == voteTypes.incorrect) {
@ -403,8 +406,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
await db.prepare('run', 'UPDATE "sponsorTimes" SET locked = 0 WHERE "UUID" = ?', [UUID]);
}
// Clear redis cache for this video
redis.delAsync(skipSegmentsKey(row?.videoID));
clearRedisCache(videoInfo);
//for each positive vote, see if a hidden submission can be shown again
if (incrementAmount > 0 && voteTypeEnum === voteTypes.normal) {
@ -441,7 +443,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
voteTypeEnum,
isVIP,
isOwnSubmission,
row,
row: videoInfo,
category,
incrementAmount,
oldIncrementAmount,
@ -453,4 +455,11 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
res.status(500).json({error: 'Internal error creating segment vote'});
}
}
}
function clearRedisCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; }) {
if (videoInfo) {
redis.delAsync(skipSegmentsKey(videoInfo.videoID));
redis.delAsync(skipSegmentsHashKey(videoInfo.hashedVideoID, videoInfo.service));
}
}

19
src/types/segments.model.ts
View file

@ -3,15 +3,33 @@ import { SBRecord } from "./lib.model";
export type SegmentUUID = string & { __segmentUUIDBrand: unknown };
export type VideoID = string & { __videoIDBrand: unknown };
export type VideoDuration = number & { __videoDurationBrand: unknown };
export type Category = string & { __categoryBrand: unknown };
export type VideoIDHash = VideoID & HashedValue;
export type IPAddress = string & { __ipAddressBrand: unknown };
export type HashedIP = IPAddress & HashedValue;
// Uncomment as needed
export enum Service {
YouTube = 'YouTube',
PeerTube = 'PeerTube',
// Twitch = 'Twitch',
// Nebula = 'Nebula',
// RSS = 'RSS',
// Corridor = 'Corridor',
// Lbry = 'Lbry'
}
export interface IncomingSegment {
category: Category;
segment: string[];
}
export interface Segment {
category: Category;
segment: number[];
UUID: SegmentUUID;
videoDuration: VideoDuration;
}
export enum Visibility {
@ -28,6 +46,7 @@ export interface DBSegment {
locked: boolean;
shadowHidden: Visibility;
videoID: VideoID;
videoDuration: VideoDuration;
hashedVideoID: VideoIDHash;
}

2
test.json
View file

@ -49,7 +49,7 @@
]
}
],
"categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"],
"categoryList": ["sponsor", "selfpromo", "interaction", "intro", "outro", "preview", "music_offtopic"],
"maxNumberOfActiveWarnings": 3,
"hoursAfterWarningExpires": 24,
"rateLimit": {

67
test/cases/getSkipSegments.ts
View file

@ -5,17 +5,19 @@ import {getHash} from '../../src/utils/getHash';
describe('getSkipSegments', () => {
before(async () => {
let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "shadowHidden", "hashedVideoID") VALUES';
await db.prepare("run", startOfQuery + "('testtesttest', 1, 11, 2, 0, '1-uuid-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('testtesttest', 1) + "')");
await db.prepare("run", startOfQuery + "('testtesttest', 20, 33, 2, 0, '1-uuid-2', 'testman', 0, 50, 'intro', 0, '" + getHash('testtesttest', 1) + "')");
await db.prepare("run", startOfQuery + "('testtesttest,test', 1, 11, 2, 0, '1-uuid-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('testtesttest,test', 1) + "')");
await db.prepare("run", startOfQuery + "('test3', 1, 11, 2, 0, '1-uuid-4', 'testman', 0, 50, 'sponsor', 0, '" + getHash('test3', 1) + "')");
await db.prepare("run", startOfQuery + "('test3', 7, 22, -3, 0, '1-uuid-5', 'testman', 0, 50, 'sponsor', 0, '" + getHash('test3', 1) + "')");
await db.prepare("run", startOfQuery + "('multiple', 1, 11, 2, 0, '1-uuid-6', 'testman', 0, 50, 'intro', 0, '" + getHash('multiple', 1) + "')");
await db.prepare("run", startOfQuery + "('multiple', 20, 33, 2, 0, '1-uuid-7', 'testman', 0, 50, 'intro', 0, '" + getHash('multiple', 1) + "')");
await db.prepare("run", startOfQuery + "('locked', 20, 33, 2, 1, '1-uuid-locked-8', 'testman', 0, 50, 'intro', 0, '" + getHash('locked', 1) + "')");
await db.prepare("run", startOfQuery + "('locked', 20, 34, 100000, 0, '1-uuid-9', 'testman', 0, 50, 'intro', 0, '" + getHash('locked', 1) + "')");
let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "service", "videoDuration", "hidden", "shadowHidden", "hashedVideoID") VALUES';
await db.prepare("run", startOfQuery + "('testtesttest', 1, 11, 2, 0, '1-uuid-0', 'testman', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '" + getHash('testtesttest', 1) + "')");
await db.prepare("run", startOfQuery + "('testtesttest2', 1, 11, 2, 0, '1-uuid-0-1', 'testman', 0, 50, 'sponsor', 'PeerTube', 120, 0, 0, '" + getHash('testtesttest2', 1) + "')");
await db.prepare("run", startOfQuery + "('testtesttest', 20, 33, 2, 0, '1-uuid-2', 'testman', 0, 50, 'intro', 'YouTube', 101, 0, 0, '" + getHash('testtesttest', 1) + "')");
await db.prepare("run", startOfQuery + "('testtesttest,test', 1, 11, 2, 0, '1-uuid-1', 'testman', 0, 50, 'sponsor', 'YouTube', 140, 0, 0, '" + getHash('testtesttest,test', 1) + "')");
await db.prepare("run", startOfQuery + "('test3', 1, 11, 2, 0, '1-uuid-4', 'testman', 0, 50, 'sponsor', 'YouTube', 200, 0, 0, '" + getHash('test3', 1) + "')");
await db.prepare("run", startOfQuery + "('test3', 7, 22, -3, 0, '1-uuid-5', 'testman', 0, 50, 'sponsor', 'YouTube', 300, 0, 0, '" + getHash('test3', 1) + "')");
await db.prepare("run", startOfQuery + "('multiple', 1, 11, 2, 0, '1-uuid-6', 'testman', 0, 50, 'intro', 'YouTube', 400, 0, 0, '" + getHash('multiple', 1) + "')");
await db.prepare("run", startOfQuery + "('multiple', 20, 33, 2, 0, '1-uuid-7', 'testman', 0, 50, 'intro', 'YouTube', 500, 0, 0, '" + getHash('multiple', 1) + "')");
await db.prepare("run", startOfQuery + "('locked', 20, 33, 2, 1, '1-uuid-locked-8', 'testman', 0, 50, 'intro', 'YouTube', 230, 0, 0, '" + getHash('locked', 1) + "')");
await db.prepare("run", startOfQuery + "('locked', 20, 34, 100000, 0, '1-uuid-9', 'testman', 0, 50, 'intro', 'YouTube', 190, 0, 0, '" + getHash('locked', 1) + "')");
await db.prepare("run", startOfQuery + "('onlyHiddenSegments', 20, 34, 100000, 0, 'onlyHiddenSegments', 'testman', 0, 50, 'sponsor', 'YouTube', 190, 1, 0, '" + getHash('onlyHiddenSegments', 1) + "')");
return;
});
@ -27,7 +29,24 @@ describe('getSkipSegments', () => {
else {
const data = await res.json();
if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11
&& data[0].category === "sponsor" && data[0].UUID === "1-uuid-0") {
&& data[0].category === "sponsor" && data[0].UUID === "1-uuid-0" && data[0].videoDuration === 100) {
return;
} else {
return ("Received incorrect body: " + (await res.text()));
}
}
})
.catch(err => "Couldn't call endpoint");
});
it('Should be able to get a time by category for a different service 1', () => {
fetch(getbaseURL() + "/api/skipSegments?videoID=testtesttest&category=sponsor&service=PeerTube")
.then(async res => {
if (res.status !== 200) return ("Status code was: " + res.status);
else {
const data = await res.json();
if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11
&& data[0].category === "sponsor" && data[0].UUID === "1-uuid-0-1" && data[0].videoDuration === 120) {
return;
} else {
return ("Received incorrect body: " + (await res.text()));
@ -61,7 +80,7 @@ describe('getSkipSegments', () => {
else {
const data = await res.json();
if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11
&& data[0].category === "sponsor" && data[0].UUID === "1-uuid-0") {
&& data[0].category === "sponsor" && data[0].UUID === "1-uuid-0" && data[0].videoDuration === 100) {
return;
} else {
return ("Received incorrect body: " + (await res.text()));
@ -78,7 +97,23 @@ describe('getSkipSegments', () => {
else {
const data = await res.json();
if (data.length === 1 && data[0].segment[0] === 20 && data[0].segment[1] === 33
&& data[0].category === "intro" && data[0].UUID === "1-uuid-2") {
&& data[0].category === "intro" && data[0].UUID === "1-uuid-2" && data[0].videoDuration === 101) {
return;
} else {
return ("Received incorrect body: " + (await res.text()));
}
}
})
.catch(err => ("Couldn't call endpoint"));
});
it('Should be empty if all submissions are hidden', () => {
fetch(getbaseURL() + "/api/skipSegments?videoID=onlyHiddenSegments")
.then(async res => {
if (res.status !== 200) return ("Status code was: " + res.status);
else {
const data = await res.json();
if (data.length === 0) {
return;
} else {
return ("Received incorrect body: " + (await res.text()));
@ -99,9 +134,9 @@ describe('getSkipSegments', () => {
let success = true;
for (const segment of data) {
if ((segment.segment[0] !== 20 || segment.segment[1] !== 33
|| segment.category !== "intro" || segment.UUID !== "1-uuid-7") &&
|| segment.category !== "intro" || segment.UUID !== "1-uuid-7" || segment.videoDuration === 500) &&
(segment.segment[0] !== 1 || segment.segment[1] !== 11
|| segment.category !== "intro" || segment.UUID !== "1-uuid-6")) {
|| segment.category !== "intro" || segment.UUID !== "1-uuid-6" || segment.videoDuration === 400)) {
success = false;
break;
}

44
test/cases/getSegmentsByHash.ts → test/cases/getSkipSegmentsByHash.ts
View file

@ -12,11 +12,13 @@ sinonStub.callsFake(YouTubeApiMock.listVideos);
describe('getSegmentsByHash', () => {
before(async () => {
let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "shadowHidden", "hashedVideoID") VALUES';
await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 20, 30, 2, 'getSegmentsByHash-0-1', 'testman', 100, 150, 'intro', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
await db.prepare("run", startOfQuery + "('getSegmentsByHash-noMatchHash', 40, 50, 2, 'getSegmentsByHash-noMatchHash', 'testman', 0, 50, 'sponsor', 0, 'fdaffnoMatchHash')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
await db.prepare("run", startOfQuery + "('getSegmentsByHash-1', 60, 70, 2, 'getSegmentsByHash-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('getSegmentsByHash-1', 1) + "')"); // hash = 3272fa85ee0927f6073ef6f07ad5f3146047c1abba794cfa364d65ab9921692b
let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "service", "hidden", "shadowHidden", "hashedVideoID") VALUES';
await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0', 'testman', 0, 50, 'sponsor', 'YouTube', 0, 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0-1', 'testman', 0, 50, 'sponsor', 'PeerTube', 0, 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 20, 30, 2, 'getSegmentsByHash-0-1', 'testman', 100, 150, 'intro', 'YouTube', 0, 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
await db.prepare("run", startOfQuery + "('getSegmentsByHash-noMatchHash', 40, 50, 2, 'getSegmentsByHash-noMatchHash', 'testman', 0, 50, 'sponsor', 'YouTube', 0, 0, 'fdaffnoMatchHash')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
await db.prepare("run", startOfQuery + "('getSegmentsByHash-1', 60, 70, 2, 'getSegmentsByHash-1', 'testman', 0, 50, 'sponsor', 'YouTube', 0, 0, '" + getHash('getSegmentsByHash-1', 1) + "')"); // hash = 3272fa85ee0927f6073ef6f07ad5f3146047c1abba794cfa364d65ab9921692b
await db.prepare("run", startOfQuery + "('onlyHidden', 60, 70, 2, 'onlyHidden', 'testman', 0, 50, 'sponsor', 'YouTube', 1, 0, '" + getHash('onlyHidden', 1) + "')"); // hash = f3a199e1af001d716cdc6599360e2b062c2d2b3fa2885f6d9d2fd741166cbbd3
});
it('Should be able to get a 200', (done: Done) => {
@ -54,6 +56,19 @@ describe('getSegmentsByHash', () => {
.catch(err => done("Couldn't call endpoint"));
});
it('Should be able to get an empty array if only hidden videos', (done: Done) => {
fetch(getbaseURL() + '/api/skipSegments/f3a1?categories=["sponsor"]')
.then(async res => {
if (res.status !== 404) done("non 404 status code, was " + res.status);
else {
const body = await res.text();
if (JSON.parse(body).length === 0 && body === '[]') done(); // pass
else done("non empty array returned");
}
})
.catch(err => done("Couldn't call endpoint"));
});
it('Should return 400 prefix too short', (done: Done) => {
fetch(getbaseURL() + '/api/skipSegments/11?categories=["shilling"]')
.then(res => {
@ -128,7 +143,24 @@ describe('getSegmentsByHash', () => {
if (body.length !== 2) done("expected 2 videos, got " + body.length);
else if (body[0].segments.length !== 1) done("expected 1 segments for first video, got " + body[0].segments.length);
else if (body[1].segments.length !== 1) done("expected 1 segments for second video, got " + body[1].segments.length);
else if (body[0].segments[0].category !== 'sponsor' || body[1].segments[0].category !== 'sponsor') done("both segments are not sponsor");
else if (body[0].segments[0].category !== 'sponsor'
|| body[0].segments[0].UUID !== 'getSegmentsByHash-0-0'
|| body[1].segments[0].category !== 'sponsor') done("both segments are not sponsor");
else done();
}
})
.catch(err => done("Couldn't call endpoint"));
});
it('Should be able to get 200 for no categories (default sponsor) for a non YouTube service', (done: Done) => {
fetch(getbaseURL() + '/api/skipSegments/fdaf?service=PeerTube')
.then(async res => {
if (res.status !== 200) done("non 200 status code, was " + res.status);
else {
const body = await res.json();
if (body.length !== 1) done("expected 2 videos, got " + body.length);
else if (body[0].segments.length !== 1) done("expected 1 segments for first video, got " + body[0].segments.length);
else if (body[0].segments[0].UUID !== 'getSegmentsByHash-0-0-1') done("both segments are not sponsor");
else done();
}
})

176
test/cases/postSkipSegments.ts
View file

@ -6,6 +6,7 @@ import {db} from '../../src/databases/databases';
import {ImportMock} from 'ts-mock-imports';
import * as YouTubeAPIModule from '../../src/utils/youtubeApi';
import {YouTubeApiMock} from '../youtubeMock';
import e from 'express';
const mockManager = ImportMock.mockStaticClass(YouTubeAPIModule, 'YouTubeAPI');
const sinonStub = mockManager.mock('listVideos');
@ -97,6 +98,177 @@ describe('postSkipSegments', () => {
.catch(err => done(err));
});
it('Should be able to submit a single time with a duration from the YouTube API (JSON method)', (done: Done) => {
fetch(getbaseURL()
+ "/api/postVideoSponsorTimes", {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
userID: "test",
videoID: "dQw4w9WgXZX",
videoDuration: 100,
segments: [{
segment: [0, 10],
category: "sponsor",
}],
}),
})
.then(async res => {
if (res.status === 200) {
const row = await db.prepare('get', `SELECT "startTime", "endTime", "locked", "category", "videoDuration" FROM "sponsorTimes" WHERE "videoID" = ?`, ["dQw4w9WgXZX"]);
if (row.startTime === 0 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.videoDuration === 5010) {
done();
} else {
done("Submitted times were not saved. Actual submission: " + JSON.stringify(row));
}
} else {
done("Status code was " + res.status);
}
})
.catch(err => done(err));
});
it('Should be able to submit a single time with a precise duration close to the one from the YouTube API (JSON method)', (done: Done) => {
fetch(getbaseURL()
+ "/api/postVideoSponsorTimes", {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
userID: "test",
videoID: "dQw4w9WgXZH",
videoDuration: 5010.20,
segments: [{
segment: [1, 10],
category: "sponsor",
}],
}),
})
.then(async res => {
if (res.status === 200) {
const row = await db.prepare('get', `SELECT "startTime", "endTime", "locked", "category", "videoDuration" FROM "sponsorTimes" WHERE "videoID" = ?`, ["dQw4w9WgXZH"]);
if (row.startTime === 1 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.videoDuration === 5010.20) {
done();
} else {
done("Submitted times were not saved. Actual submission: " + JSON.stringify(row));
}
} else {
done("Status code was " + res.status);
}
})
.catch(err => done(err));
});
it('Should be able to submit a single time with a duration in the body (JSON method)', (done: Done) => {
fetch(getbaseURL()
+ "/api/postVideoSponsorTimes", {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
userID: "test",
videoID: "noDuration",
videoDuration: 100,
segments: [{
segment: [0, 10],
category: "sponsor",
}],
}),
})
.then(async res => {
if (res.status === 200) {
const row = await db.prepare('get', `SELECT "startTime", "endTime", "locked", "category", "videoDuration" FROM "sponsorTimes" WHERE "videoID" = ?`, ["noDuration"]);
if (row.startTime === 0 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.videoDuration === 100) {
done();
} else {
done("Submitted times were not saved. Actual submission: " + JSON.stringify(row));
}
} else {
done("Status code was " + res.status);
}
})
.catch(err => done(err));
});
it('Should be able to submit with a new duration, and hide old submissions and remove segment locks', async () => {
await db.prepare("run", `INSERT INTO "noSegments" ("userID", "videoID", "category")
VALUES ('` + getHash("VIPUser-noSegments") + "', 'noDuration', 'sponsor')");
try {
const res = await fetch(getbaseURL()
+ "/api/postVideoSponsorTimes", {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
userID: "test",
videoID: "noDuration",
videoDuration: 100,
segments: [{
segment: [1, 10],
category: "sponsor",
}],
}),
});
if (res.status === 200) {
const noSegmentsRow = await db.prepare('get', `SELECT * from "noSegments" WHERE videoID = ?`, ["noDuration"]);
const videoRows = await db.prepare('all', `SELECT "startTime", "endTime", "locked", "category", "videoDuration"
FROM "sponsorTimes" WHERE "videoID" = ? AND hidden = 0`, ["noDuration"]);
const videoRow = videoRows[0];
const hiddenVideoRows = await db.prepare('all', `SELECT "startTime", "endTime", "locked", "category", "videoDuration"
FROM "sponsorTimes" WHERE "videoID" = ? AND hidden = 1`, ["noDuration"]);
if (noSegmentsRow === undefined && videoRows.length === 1 && hiddenVideoRows.length === 1 && videoRow.startTime === 1 && videoRow.endTime === 10
&& videoRow.locked === 0 && videoRow.category === "sponsor" && videoRow.videoDuration === 100) {
return;
} else {
return "Submitted times were not saved. Actual submission: " + JSON.stringify(videoRow);
}
} else {
return "Status code was " + res.status;
}
} catch (e) {
return e;
}
});
it('Should be able to submit a single time under a different service (JSON method)', (done: Done) => {
fetch(getbaseURL()
+ "/api/postVideoSponsorTimes", {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
userID: "test",
videoID: "dQw4w9WgXcG",
service: "PeerTube",
segments: [{
segment: [0, 10],
category: "sponsor",
}],
}),
})
.then(async res => {
if (res.status === 200) {
const row = await db.prepare('get', `SELECT "startTime", "endTime", "locked", "category", "service" FROM "sponsorTimes" WHERE "videoID" = ?`, ["dQw4w9WgXcG"]);
if (row.startTime === 0 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.service === "PeerTube") {
done();
} else {
done("Submitted times were not saved. Actual submission: " + JSON.stringify(row));
}
} else {
done("Status code was " + res.status);
}
})
.catch(err => done(err));
});
it('VIP submission should start locked', (done: Done) => {
fetch(getbaseURL()
+ "/api/postVideoSponsorTimes", {
@ -244,7 +416,7 @@ describe('postSkipSegments', () => {
}),
})
.then(async res => {
if (res.status === 400) {
if (res.status === 403) {
const rows = await db.prepare('all', `SELECT "startTime", "endTime", "category" FROM "sponsorTimes" WHERE "videoID" = ? and "votes" > -1`, ["n9rIGdXnSJc"]);
let success = true;
if (rows.length === 4) {
@ -292,7 +464,7 @@ describe('postSkipSegments', () => {
}),
})
.then(async res => {
if (res.status === 400) {
if (res.status === 403) {
const rows = await db.prepare('all', `SELECT "startTime", "endTime", "category" FROM "sponsorTimes" WHERE "videoID" = ? and "votes" > -1`, ["80percent_video"]);
let success = rows.length == 2;
for (const row of rows) {