external/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2024-11-10 01:02:30 +01:00
Code Issues Projects Releases Packages Wiki Activity

Make schemas work with postgres

Browse source
This commit is contained in:
Ajay Ramachandran 2021-03-04 19:44:54 -05:00
parent 2c211d4730
commit 1a66be8665
13 changed files with 114 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

58
databases/_sponsorTimes.db.sql
View file

@ -1,39 +1,43 @@
BEGIN TRANSACTION;
CREATE TABLE IF NOT EXISTS "vipUsers" (
"userID" TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS "sponsorTimes" (
"videoID" TEXT NOT NULL,
"startTime" REAL NOT NULL,
"endTime" REAL NOT NULL,
"votes" INTEGER NOT NULL,
"UUID" TEXT NOT NULL UNIQUE,
"userID" TEXT NOT NULL,
"timeSubmitted" INTEGER NOT NULL,
"views" INTEGER NOT NULL,
"category" TEXT NOT NULL,
"shadowHidden" INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS "userNames" (
"userID" TEXT NOT NULL,
"userName" TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS "categoryVotes" (
"UUID" TEXT NOT NULL,
"category" TEXT NOT NULL,
"votes" INTEGER NOT NULL default '0'
CREATE TABLE IF NOT EXISTS vipUsers (
userID TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS "config" (
"key" TEXT NOT NULL UNIQUE,
"value" TEXT NOT NULL
COMMIT;
BEGIN TRANSACTION;
CREATE TABLE IF NOT EXISTS sponsorTimes (
videoID TEXT NOT NULL,
startTime REAL NOT NULL,
endTime REAL NOT NULL,
votes INTEGER NOT NULL,
UUID TEXT NOT NULL UNIQUE,
userID TEXT NOT NULL,
timeSubmitted INTEGER NOT NULL,
views INTEGER NOT NULL,
category TEXT NOT NULL,
shadowHidden INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS userNames (
userID TEXT NOT NULL,
userName TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS categoryVotes (
UUID TEXT NOT NULL,
category TEXT NOT NULL,
votes INTEGER NOT NULL default 0
);
CREATE TABLE IF NOT EXISTS config (
key TEXT NOT NULL UNIQUE,
value TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS sponsorTimes_videoID on sponsorTimes(videoID);
CREATE INDEX IF NOT EXISTS sponsorTimes_UUID on sponsorTimes(UUID);
CREATE EXTENSION IF NOT EXISTS pgcrypto; --!sqlite-ignore
COMMIT;

8
databases/_upgrade_sponsorTimes_1.sql
View file

@ -6,20 +6,20 @@ CREATE TABLE "sqlb_temp_table_1" (
"startTime" REAL NOT NULL,
"endTime" REAL NOT NULL,
"votes" INTEGER NOT NULL,
"incorrectVotes" INTEGER NOT NULL default '1',
"incorrectVotes" INTEGER NOT NULL default 1,
"UUID" TEXT NOT NULL UNIQUE,
"userID" TEXT NOT NULL,
"timeSubmitted" INTEGER NOT NULL,
"views" INTEGER NOT NULL,
"category" TEXT NOT NULL DEFAULT "sponsor",
"category" TEXT NOT NULL DEFAULT 'sponsor',
"shadowHidden" INTEGER NOT NULL
);
INSERT INTO sqlb_temp_table_1 SELECT videoID,startTime,endTime,votes,"1",UUID,userID,timeSubmitted,views,category,shadowHidden FROM sponsorTimes;
INSERT INTO sqlb_temp_table_1 SELECT videoID,startTime,endTime,votes,'1',UUID,userID,timeSubmitted,views,category,shadowHidden FROM sponsorTimes;
DROP TABLE sponsorTimes;
ALTER TABLE sqlb_temp_table_1 RENAME TO "sponsorTimes";
/* Add version to config */
INSERT INTO config (key, value) VALUES("version", 1);
INSERT INTO config (key, value) VALUES('version', 1);
COMMIT;

6
databases/_upgrade_sponsorTimes_3.sql
View file

@ -1,13 +1,13 @@
BEGIN TRANSACTION;
/* hash upgrade test sha256('vid') = '1ff838dc6ca9680d88455341118157d59a055fe6d0e3870f9c002847bebe4663'
/* hash upgrade test sha256('vid') = '1ff838dc6ca9680d88455341118157d59a055fe6d0e3870f9c002847bebe4663' */
/* Add hash field */
ALTER TABLE sponsorTimes ADD hashedVideoID TEXT NOT NULL default "";
ALTER TABLE sponsorTimes ADD hashedVideoID TEXT NOT NULL default '';
UPDATE sponsorTimes SET hashedVideoID = sha256(videoID);
CREATE INDEX IF NOT EXISTS sponsorTimes_hashedVideoID on sponsorTimes(hashedVideoID);
/* Bump version in config */
UPDATE config SET value = 3 WHERE key = "version";
UPDATE config SET value = 3 WHERE key = 'version';
COMMIT;

2
databases/_upgrade_sponsorTimes_4.sql
View file

@ -7,6 +7,6 @@ CREATE TABLE "warnings" (
issuerUserID TEXT NOT NULL
);
UPDATE config SET value = 4 WHERE key = "version";
UPDATE config SET value = 4 WHERE key = 'version';
COMMIT;

4
databases/_upgrade_sponsorTimes_5.sql
View file

@ -10,8 +10,8 @@ CREATE TABLE "sqlb_temp_table_5" (
INSERT INTO sqlb_temp_table_5 SELECT userID,issueTime,issuerUserID,1 FROM warnings;
DROP TABLE warnings;
ALTER TABLE sqlb_temp_table_5 RENAME TO "warnings";
ALTER TABLE sqlb_temp_table_5 RENAME TO "warnings";;
UPDATE config SET value = 5 WHERE key = "version";
UPDATE config SET value = 5 WHERE key = 'version';
COMMIT;

8
databases/_upgrade_sponsorTimes_6.sql
View file

@ -12,16 +12,16 @@ CREATE TABLE "sqlb_temp_table_6" (
"userID" TEXT NOT NULL,
"timeSubmitted" INTEGER NOT NULL,
"views" INTEGER NOT NULL,
"category" TEXT NOT NULL DEFAULT "sponsor",
"category" TEXT NOT NULL DEFAULT 'sponsor',
"shadowHidden" INTEGER NOT NULL,
"hashedVideoID" TEXT NOT NULL default ""
"hashedVideoID" TEXT NOT NULL default ''
);
INSERT INTO sqlb_temp_table_6 SELECT videoID,startTime,endTime,votes,"0",incorrectVotes,UUID,userID,timeSubmitted,views,category,shadowHidden,hashedVideoID FROM sponsorTimes;
INSERT INTO sqlb_temp_table_6 SELECT videoID,startTime,endTime,votes,'0',incorrectVotes,UUID,userID,timeSubmitted,views,category,shadowHidden,hashedVideoID FROM sponsorTimes;
DROP TABLE sponsorTimes;
ALTER TABLE sqlb_temp_table_6 RENAME TO "sponsorTimes";
UPDATE config SET value = 6 WHERE key = "version";
UPDATE config SET value = 6 WHERE key = 'version';
COMMIT;

5
package-lock.json generated
View file

@ -754,6 +754,11 @@
"is-obj": "^1.0.0"
}
},
"dotenv": {
"version": "8.2.0",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.2.0.tgz",
"integrity": "sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw=="
},
"duplexer3": {
"version": "0.1.4",
"resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz",

1
package.json
View file

@ -14,6 +14,7 @@
"license": "MIT",
"dependencies": {
"better-sqlite3": "^5.4.3",
"dotenv": "^8.2.0",
"express": "^4.17.1",
"express-rate-limit": "^5.1.3",
"http": "0.0.0",

1
src/config.ts
View file

@ -45,6 +45,7 @@ addDefaults(config, {
},
userCounterURL: null,
youtubeAPIKey: null,
postgres: null
});
// Add defaults

26
src/databases/Postgres.ts
View file

@ -4,27 +4,25 @@ import { Pool } from 'pg';
import fs from "fs";
export class Mysql implements IDatabase {
export class Postgres implements IDatabase {
private pool: Pool;
constructor(private config: any) {}
init(): void {
this.pool = new Pool();
async init(): Promise<void> {
this.pool = new Pool(this.config.postgres);
if (!this.config.readOnly) {
// Upgrade database if required
this.upgradeDB(this.config.fileNamePrefix, this.config.dbSchemaFolder);
if (this.config.createDbIfNotExists && !this.config.readOnly && fs.existsSync(this.config.dbSchemaFileName)) {
this.pool.query(this.processUpgradeQuery(fs.readFileSync(this.config.dbSchemaFileName).toString()));
await this.pool.query(this.processUpgradeQuery(fs.readFileSync(this.config.dbSchemaFileName).toString()));
}
// Upgrade database if required
await this.upgradeDB(this.config.fileNamePrefix, this.config.dbSchemaFolder);
}
}
async prepare(type: QueryType, query: string, params?: any[]) {
Logger.debug(`prepare (postgres): type: ${type}, query: ${query}, params: ${params}`);
// Convert query to use numbered parameters
let count = 1;
for (let char = 0; char < query.length; char++) {
@ -34,7 +32,9 @@ export class Mysql implements IDatabase {
}
}
const queryResult = await this.pool.query(query, params);
Logger.debug(`prepare (postgres): type: ${type}, query: ${query}, params: ${params}`);
const queryResult = await this.pool.query({text: query, values: params});
switch (type) {
case 'get': {
@ -51,7 +51,7 @@ export class Mysql implements IDatabase {
private async upgradeDB(fileNamePrefix: string, schemaFolder: string) {
const versionCodeInfo = await this.pool.query("SELECT value FROM config WHERE key = 'version'");
let versionCode = versionCodeInfo ? versionCodeInfo.rows[0].value : 0;
let versionCode = versionCodeInfo.rows[0] ? versionCodeInfo.rows[0].value : 0;
let path = schemaFolder + "/_upgrade_" + fileNamePrefix + "_" + (parseInt(versionCode) + 1) + ".sql";
Logger.debug('db update: trying ' + path);
@ -67,7 +67,9 @@ export class Mysql implements IDatabase {
}
private processUpgradeQuery(query: string): string {
let result = query.replace(/sha256\((.*?)\)/gm, "digest($1, 'sha256')");
let result = query.toLocaleLowerCase();
result = result.replace(/sha256\((.*?)\)/gm, "digest($1, 'sha256')");
result = result.replace(/integer/gm, "numeric");
return result;
}

21
src/databases/databases.ts
View file

@ -1,6 +1,7 @@
import {config} from '../config';
import {Sqlite} from './Sqlite';
import {Mysql} from './Mysql';
import {Postgres} from './Postgres';
import {IDatabase} from './IDatabase';
@ -9,6 +10,26 @@ let privateDB: IDatabase;
if (config.mysql) {
db = new Mysql(config.mysql);
privateDB = new Mysql(config.privateMysql);
} else if (config.postgres) {
db = new Postgres({
dbSchemaFileName: config.dbSchema,
dbSchemaFolder: config.schemaFolder,
fileNamePrefix: 'sponsorTimes',
readOnly: config.readOnly,
createDbIfNotExists: config.createDatabaseIfNotExist,
enableWalCheckpointNumber: !config.readOnly && config.mode === "production",
postgres: config.postgres
});
privateDB = new Sqlite({
dbPath: config.privateDB,
dbSchemaFileName: config.privateDBSchema,
dbSchemaFolder: config.schemaFolder,
fileNamePrefix: 'private',
readOnly: config.readOnly,
createDbIfNotExists: config.createDatabaseIfNotExist,
enableWalCheckpointNumber: false
});
} else {
db = new Sqlite({
dbPath: config.db,

19
src/routes/getSkipSegments.ts
View file

@ -52,11 +52,14 @@ async function getSegmentsByVideoID(req: Request, videoID: string, categories: C
const segments: Segment[] = [];
try {
categories.filter((category) => !/[^a-z|_|-]/.test(category));
const segmentsByCategory: SBRecord<Category, DBSegment[]> = (await db
.prepare(
'all',
`SELECT startTime, endTime, votes, locked, UUID, category, shadowHidden FROM sponsorTimes WHERE videoID = ? AND category IN (${Array(categories.length).fill('?').join()}) ORDER BY startTime`,
[videoID, categories]
`SELECT startTime, endTime, votes, locked, UUID, category, shadowHidden FROM sponsorTimes
WHERE videoID = ? AND category IN (${categories.map((c) => "'" + c + "'")}) ORDER BY startTime`,
[videoID]
)).reduce((acc: SBRecord<Category, DBSegment[]>, segment: DBSegment) => {
acc[segment.category] = acc[segment.category] || [];
acc[segment.category].push(segment);
@ -64,10 +67,15 @@ async function getSegmentsByVideoID(req: Request, videoID: string, categories: C
return acc;
}, {});
console.log(segmentsByCategory)
for (const [category, categorySegments] of Object.entries(segmentsByCategory)) {
segments.push(...(await prepareCategorySegments(req, videoID as VideoID, category as Category, categorySegments, cache)));
}
console.log(segments)
return segments;
} catch (err) {
if (err) {
@ -84,11 +92,14 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash,
try {
type SegmentWithHashPerVideoID = SBRecord<VideoID, {hash: VideoIDHash, segmentPerCategory: SBRecord<Category, DBSegment[]>}>;
categories.filter((category) => !/[^a-z|_|-]/.test(category));
const segmentPerVideoID: SegmentWithHashPerVideoID = (await db
.prepare(
'all',
`SELECT videoID, startTime, endTime, votes, locked, UUID, category, shadowHidden, hashedVideoID FROM sponsorTimes WHERE hashedVideoID LIKE ? AND category IN (${Array(categories.length).fill('?').join()}) ORDER BY startTime`,
[hashedVideoIDPrefix + '%', categories]
`SELECT videoID, startTime, endTime, votes, locked, UUID, category, shadowHidden, hashedVideoID FROM sponsorTimes
WHERE hashedVideoID LIKE ? AND category IN (${categories.map((c) => "'" + c + "'")}) ORDER BY startTime`,
[hashedVideoIDPrefix + '%']
)).reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => {
acc[segment.videoID] = acc[segment.videoID] || {
hash: segment.hashedVideoID,

12
src/types/config.model.ts
View file

@ -1,3 +1,4 @@
import { PoolConfig } from 'pg';
import * as redis from 'redis';
export interface SBSConfig {
@ -36,6 +37,7 @@ export interface SBSConfig {
minimumPrefix?: string;
maximumPrefix?: string;
redis?: redis.ClientOpts;
postgres?: PoolConfig;
}
export interface WebhookConfig {
@ -50,3 +52,13 @@ export interface RateLimitConfig {
message: string;
statusCode: number;
}
export interface PostgresConfig {
dbSchemaFileName: string;
dbSchemaFolder: string;
fileNamePrefix: string;
readOnly: boolean;
createDbIfNotExists: boolean;
enableWalCheckpointNumber: boolean;
postgres: PoolConfig;
}