Make schemas work with postgres

databases/_sponsorTimes.db.sql

@@ -1,39 +1,43 @@
 BEGIN TRANSACTION;
 
-CREATE TABLE IF NOT EXISTS "vipUsers" (
+CREATE TABLE IF NOT EXISTS vipUsers (
-	"userID"	TEXT NOT NULL
+	userID	TEXT NOT NULL
-);
-CREATE TABLE IF NOT EXISTS "sponsorTimes" (
-	"videoID"	TEXT NOT NULL,
-	"startTime"	REAL NOT NULL,
-	"endTime"	REAL NOT NULL,
-	"votes"	INTEGER NOT NULL,
-	"UUID"	TEXT NOT NULL UNIQUE,
-	"userID"	TEXT NOT NULL,
-	"timeSubmitted"	INTEGER NOT NULL,
-	"views"	INTEGER NOT NULL,
-    "category" TEXT NOT NULL,
-	"shadowHidden"	INTEGER NOT NULL
-);
-CREATE TABLE IF NOT EXISTS "userNames" (
-	"userID"	TEXT NOT NULL,
-	"userName"	TEXT NOT NULL
-);
-CREATE TABLE IF NOT EXISTS "categoryVotes" (
-	"UUID"	TEXT NOT NULL,
-	"category"	TEXT NOT NULL,
-	"votes"	INTEGER NOT NULL default '0'
 );
 
-CREATE TABLE IF NOT EXISTS "config" (
+COMMIT;
-    "key" TEXT NOT NULL UNIQUE,
+
-	"value" TEXT NOT NULL
+BEGIN TRANSACTION;
+
+CREATE TABLE IF NOT EXISTS sponsorTimes (
+	videoID	TEXT NOT NULL,
+	startTime	REAL NOT NULL,
+	endTime	REAL NOT NULL,
+	votes	INTEGER NOT NULL,
+	UUID	TEXT NOT NULL UNIQUE,
+	userID	TEXT NOT NULL,
+	timeSubmitted	INTEGER NOT NULL,
+	views	INTEGER NOT NULL,
+    category TEXT NOT NULL,
+	shadowHidden	INTEGER NOT NULL
+);
+CREATE TABLE IF NOT EXISTS userNames (
+	userID	TEXT NOT NULL,
+	userName	TEXT NOT NULL
+);
+CREATE TABLE IF NOT EXISTS categoryVotes (
+	UUID	TEXT NOT NULL,
+	category	TEXT NOT NULL,
+	votes	INTEGER NOT NULL default 0
+);
+
+CREATE TABLE IF NOT EXISTS config (
+    key TEXT NOT NULL UNIQUE,
+	value TEXT NOT NULL
 );
 
 CREATE INDEX IF NOT EXISTS sponsorTimes_videoID on sponsorTimes(videoID);
 CREATE INDEX IF NOT EXISTS sponsorTimes_UUID on sponsorTimes(UUID);
 
-
 CREATE EXTENSION IF NOT EXISTS pgcrypto; --!sqlite-ignore
 
 COMMIT;

databases/_upgrade_sponsorTimes_1.sql

@@ -6,20 +6,20 @@ CREATE TABLE "sqlb_temp_table_1" (
 	"startTime"	REAL NOT NULL,
 	"endTime"	REAL NOT NULL,
 	"votes"	INTEGER NOT NULL,
-    "incorrectVotes" INTEGER NOT NULL default '1',
+    "incorrectVotes" INTEGER NOT NULL default 1,
 	"UUID"	TEXT NOT NULL UNIQUE,
 	"userID"	TEXT NOT NULL,
 	"timeSubmitted"	INTEGER NOT NULL,
 	"views"	INTEGER NOT NULL,
-	"category"	TEXT NOT NULL DEFAULT "sponsor",
+	"category"	TEXT NOT NULL DEFAULT 'sponsor',
 	"shadowHidden"	INTEGER NOT NULL
 );
-INSERT INTO sqlb_temp_table_1 SELECT videoID,startTime,endTime,votes,"1",UUID,userID,timeSubmitted,views,category,shadowHidden FROM sponsorTimes;
+INSERT INTO sqlb_temp_table_1 SELECT videoID,startTime,endTime,votes,'1',UUID,userID,timeSubmitted,views,category,shadowHidden FROM sponsorTimes;
 
 DROP TABLE sponsorTimes;
 ALTER TABLE sqlb_temp_table_1 RENAME TO "sponsorTimes";
 
 /* Add version to config */
-INSERT INTO config (key, value) VALUES("version", 1);
+INSERT INTO config (key, value) VALUES('version', 1);
 
 COMMIT;

databases/_upgrade_sponsorTimes_3.sql

@@ -1,13 +1,13 @@
 BEGIN TRANSACTION;
 
-/* hash upgrade test sha256('vid') = '1ff838dc6ca9680d88455341118157d59a055fe6d0e3870f9c002847bebe4663' 
+/* hash upgrade test sha256('vid') = '1ff838dc6ca9680d88455341118157d59a055fe6d0e3870f9c002847bebe4663' */
 /* Add hash field */
-ALTER TABLE sponsorTimes ADD hashedVideoID TEXT NOT NULL default "";
+ALTER TABLE sponsorTimes ADD hashedVideoID TEXT NOT NULL default '';
 UPDATE sponsorTimes SET hashedVideoID = sha256(videoID);
 
 CREATE INDEX IF NOT EXISTS sponsorTimes_hashedVideoID on sponsorTimes(hashedVideoID);
 
 /* Bump version in config */
-UPDATE config SET value = 3 WHERE key = "version";
+UPDATE config SET value = 3 WHERE key = 'version';
 
 COMMIT;

databases/_upgrade_sponsorTimes_4.sql

@@ -7,6 +7,6 @@ CREATE TABLE "warnings" (
     issuerUserID TEXT NOT NULL
 );
 
-UPDATE config SET value = 4 WHERE key = "version";
+UPDATE config SET value = 4 WHERE key = 'version';
 
 COMMIT;

databases/_upgrade_sponsorTimes_5.sql

@@ -10,8 +10,8 @@ CREATE TABLE "sqlb_temp_table_5" (
 INSERT INTO sqlb_temp_table_5 SELECT userID,issueTime,issuerUserID,1 FROM warnings;
 
 DROP TABLE warnings;
-ALTER TABLE sqlb_temp_table_5 RENAME TO "warnings";
+ALTER TABLE sqlb_temp_table_5 RENAME TO "warnings";;
 
-UPDATE config SET value = 5 WHERE key = "version";
+UPDATE config SET value = 5 WHERE key = 'version';
 
 COMMIT;

databases/_upgrade_sponsorTimes_6.sql

@@ -12,16 +12,16 @@ CREATE TABLE "sqlb_temp_table_6" (
 	"userID"	TEXT NOT NULL,
 	"timeSubmitted"	INTEGER NOT NULL,
 	"views"	INTEGER NOT NULL,
-	"category"	TEXT NOT NULL DEFAULT "sponsor",
+	"category"	TEXT NOT NULL DEFAULT 'sponsor',
 	"shadowHidden"	INTEGER NOT NULL,
-    "hashedVideoID" TEXT NOT NULL default ""
+    "hashedVideoID" TEXT NOT NULL default ''
 );
 
-INSERT INTO sqlb_temp_table_6 SELECT videoID,startTime,endTime,votes,"0",incorrectVotes,UUID,userID,timeSubmitted,views,category,shadowHidden,hashedVideoID FROM sponsorTimes;
+INSERT INTO sqlb_temp_table_6 SELECT videoID,startTime,endTime,votes,'0',incorrectVotes,UUID,userID,timeSubmitted,views,category,shadowHidden,hashedVideoID FROM sponsorTimes;
 
 DROP TABLE sponsorTimes;
 ALTER TABLE sqlb_temp_table_6 RENAME TO "sponsorTimes";
 
-UPDATE config SET value = 6 WHERE key = "version";
+UPDATE config SET value = 6 WHERE key = 'version';
 
 COMMIT;

package-lock.json

@@ -754,6 +754,11 @@
         "is-obj": "^1.0.0"
       }
     },
+    "dotenv": {
+      "version": "8.2.0",
+      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.2.0.tgz",
+      "integrity": "sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw=="
+    },
     "duplexer3": {
       "version": "0.1.4",
       "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz",

package.json

@@ -14,6 +14,7 @@
   "license": "MIT",
   "dependencies": {
     "better-sqlite3": "^5.4.3",
+    "dotenv": "^8.2.0",
     "express": "^4.17.1",
     "express-rate-limit": "^5.1.3",
     "http": "0.0.0",

src/config.ts

@@ -45,6 +45,7 @@ addDefaults(config, {
     },
     userCounterURL: null,
     youtubeAPIKey: null,
+    postgres: null
 });
 
 // Add defaults

src/databases/Postgres.ts

@@ -4,27 +4,25 @@ import { Pool } from 'pg';
 
 import fs from "fs";
 
-export class Mysql implements IDatabase {
+export class Postgres implements IDatabase {
     private pool: Pool;
 
     constructor(private config: any) {}
 
-    init(): void {
+    async init(): Promise<void> {
-        this.pool = new Pool();
+        this.pool = new Pool(this.config.postgres);
 
         if (!this.config.readOnly) {
-            // Upgrade database if required
-            this.upgradeDB(this.config.fileNamePrefix, this.config.dbSchemaFolder);
-
             if (this.config.createDbIfNotExists && !this.config.readOnly && fs.existsSync(this.config.dbSchemaFileName)) {
-                this.pool.query(this.processUpgradeQuery(fs.readFileSync(this.config.dbSchemaFileName).toString()));
+                await this.pool.query(this.processUpgradeQuery(fs.readFileSync(this.config.dbSchemaFileName).toString()));
             }
+
+            // Upgrade database if required
+            await this.upgradeDB(this.config.fileNamePrefix, this.config.dbSchemaFolder);
         }
     }
 
     async prepare(type: QueryType, query: string, params?: any[]) {
-        Logger.debug(`prepare (postgres): type: ${type}, query: ${query}, params: ${params}`);
-
         // Convert query to use numbered parameters
         let count = 1;
         for (let char = 0; char < query.length; char++) {
@@ -34,7 +32,9 @@ export class Mysql implements IDatabase {
             }
         }
 
-        const queryResult = await this.pool.query(query, params);
+        Logger.debug(`prepare (postgres): type: ${type}, query: ${query}, params: ${params}`);
+
+        const queryResult = await this.pool.query({text: query, values: params});
 
         switch (type) {
             case 'get': {
@@ -51,7 +51,7 @@ export class Mysql implements IDatabase {
 
     private async upgradeDB(fileNamePrefix: string, schemaFolder: string) {
         const versionCodeInfo = await this.pool.query("SELECT value FROM config WHERE key = 'version'");
-        let versionCode = versionCodeInfo ? versionCodeInfo.rows[0].value : 0;
+        let versionCode = versionCodeInfo.rows[0] ? versionCodeInfo.rows[0].value : 0;
 
         let path = schemaFolder + "/_upgrade_" + fileNamePrefix + "_" + (parseInt(versionCode) + 1) + ".sql";
         Logger.debug('db update: trying ' + path);
@@ -67,7 +67,9 @@ export class Mysql implements IDatabase {
     }
 
     private processUpgradeQuery(query: string): string {
-        let result = query.replace(/sha256\((.*?)\)/gm, "digest($1, 'sha256')");
+        let result = query.toLocaleLowerCase();
+        result = result.replace(/sha256\((.*?)\)/gm, "digest($1, 'sha256')");
+        result = result.replace(/integer/gm, "numeric");
 
         return result;
     }

src/databases/databases.ts

@@ -1,6 +1,7 @@
 import {config} from '../config';
 import {Sqlite} from './Sqlite';
 import {Mysql} from './Mysql';
+import {Postgres} from './Postgres';
 import {IDatabase} from './IDatabase';
 
 
@@ -9,6 +10,26 @@ let privateDB: IDatabase;
 if (config.mysql) {
     db = new Mysql(config.mysql);
     privateDB = new Mysql(config.privateMysql);
+} else if (config.postgres) {
+    db = new Postgres({
+        dbSchemaFileName: config.dbSchema,
+        dbSchemaFolder: config.schemaFolder,
+        fileNamePrefix: 'sponsorTimes',
+        readOnly: config.readOnly,
+        createDbIfNotExists: config.createDatabaseIfNotExist,
+        enableWalCheckpointNumber: !config.readOnly && config.mode === "production",
+        postgres: config.postgres
+    });
+
+    privateDB = new Sqlite({
+        dbPath: config.privateDB,
+        dbSchemaFileName: config.privateDBSchema,
+        dbSchemaFolder: config.schemaFolder,
+        fileNamePrefix: 'private',
+        readOnly: config.readOnly,
+        createDbIfNotExists: config.createDatabaseIfNotExist,
+        enableWalCheckpointNumber: false
+    });
 } else {
     db = new Sqlite({
         dbPath: config.db,

src/routes/getSkipSegments.ts

@@ -52,11 +52,14 @@ async function getSegmentsByVideoID(req: Request, videoID: string, categories: C
     const segments: Segment[] = [];
 
     try {
+        categories.filter((category) => !/[^a-z|_|-]/.test(category));
+
         const segmentsByCategory: SBRecord<Category, DBSegment[]> = (await db
             .prepare(
                 'all',
-                `SELECT startTime, endTime, votes, locked, UUID, category, shadowHidden FROM sponsorTimes WHERE videoID = ? AND category IN (${Array(categories.length).fill('?').join()}) ORDER BY startTime`,
+                `SELECT startTime, endTime, votes, locked, UUID, category, shadowHidden FROM sponsorTimes 
-                [videoID, categories]
+                WHERE videoID = ? AND category IN (${categories.map((c) => "'" + c + "'")}) ORDER BY startTime`,
+                [videoID]
             )).reduce((acc: SBRecord<Category, DBSegment[]>, segment: DBSegment) => {
                 acc[segment.category] = acc[segment.category] || [];
                 acc[segment.category].push(segment);
@@ -64,10 +67,15 @@ async function getSegmentsByVideoID(req: Request, videoID: string, categories: C
                 return acc;
             }, {});
 
+            console.log(segmentsByCategory)
+
+
         for (const [category, categorySegments] of Object.entries(segmentsByCategory)) {
             segments.push(...(await prepareCategorySegments(req, videoID as VideoID, category as Category, categorySegments, cache)));
         }
 
+        console.log(segments)
+
         return segments;
     } catch (err) {
         if (err) {
@@ -84,11 +92,14 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash,
     try {
         type SegmentWithHashPerVideoID = SBRecord<VideoID, {hash: VideoIDHash, segmentPerCategory: SBRecord<Category, DBSegment[]>}>;
 
+        categories.filter((category) => !/[^a-z|_|-]/.test(category));
+
         const segmentPerVideoID: SegmentWithHashPerVideoID = (await db
             .prepare(
                 'all',
-                `SELECT videoID, startTime, endTime, votes, locked, UUID, category, shadowHidden, hashedVideoID FROM sponsorTimes WHERE hashedVideoID LIKE ? AND category IN (${Array(categories.length).fill('?').join()}) ORDER BY startTime`,
+                `SELECT videoID, startTime, endTime, votes, locked, UUID, category, shadowHidden, hashedVideoID FROM sponsorTimes
-                [hashedVideoIDPrefix + '%', categories]
+                WHERE hashedVideoID LIKE ? AND category IN (${categories.map((c) => "'" + c + "'")}) ORDER BY startTime`,
+                [hashedVideoIDPrefix + '%']
             )).reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => {
                 acc[segment.videoID] = acc[segment.videoID] || {
                     hash: segment.hashedVideoID,

src/types/config.model.ts

@@ -1,3 +1,4 @@
+import { PoolConfig } from 'pg';
 import * as redis from 'redis';
 
 export interface SBSConfig {
@@ -36,6 +37,7 @@ export interface SBSConfig {
     minimumPrefix?: string;
     maximumPrefix?: string;
     redis?: redis.ClientOpts;
+    postgres?: PoolConfig;
 }
 
 export interface WebhookConfig {
@@ -50,3 +52,13 @@ export interface RateLimitConfig {
     message: string;
     statusCode: number;
 }
+
+export interface PostgresConfig {
+    dbSchemaFileName: string;
+    dbSchemaFolder: string;
+    fileNamePrefix: string;
+    readOnly: boolean;
+    createDbIfNotExists: boolean;
+    enableWalCheckpointNumber: boolean;
+    postgres: PoolConfig;
+}