From 02e628f5330403bc578d1e92a54d7d4ab08b54f2 Mon Sep 17 00:00:00 2001
From: Ajay Ramachandran <dev@ajay.app>
Date: Sat, 20 Mar 2021 01:08:33 -0400
Subject: [PATCH] Setup csv exports and html status page

---
 .gitignore                 |  1 +
 docker/docker-compose.yml  |  1 +
 src/app.ts                 | 11 +++++--
 src/routes/dumpDatabase.ts | 61 ++++++++++++++++++++++++++++++++++++++
 4 files changed, 71 insertions(+), 3 deletions(-)
 create mode 100644 src/routes/dumpDatabase.ts

diff --git a/.gitignore b/.gitignore
index 3cff924..dd78e49 100644
--- a/.gitignore
+++ b/.gitignore
@@ -99,6 +99,7 @@ test/databases/sponsorTimes.db
 test/databases/sponsorTimes.db-shm
 test/databases/sponsorTimes.db-wal
 test/databases/private.db
+docker/database-export
 
 # Config files
 config.json
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
index a54a3c0..4ee69ee 100644
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -7,6 +7,7 @@ services:
       - database.env
     volumes:
       - database-data:/var/lib/postgresql/data
+      - ./database-export/:/opt/exports
     ports:
       - 127.0.0.1:5432:5432
   redis:
diff --git a/src/app.ts b/src/app.ts
index 098e5f3..026f74c 100644
--- a/src/app.ts
+++ b/src/app.ts
@@ -26,6 +26,7 @@ import {userCounter} from './middleware/userCounter';
 import {loggerMiddleware} from './middleware/logger';
 import {corsMiddleware} from './middleware/cors';
 import {rateLimitMiddleware} from './middleware/requestRateLimit';
+import dumpDatabase from './routes/dumpDatabase';
 
 
 export function createServer(callback: () => void) {
@@ -127,7 +128,11 @@ function setupRoutes(app: Express) {
     //get if user is a vip
     app.post('/api/segmentShift', postSegmentShift);
 
-    app.get('/database.db', function (req: Request, res: Response) {
-        res.sendFile("./databases/sponsorTimes.db", {root: "./"});
-    });
+    if (config.postgres) {
+        app.get('/database', dumpDatabase);
+    } else {
+        app.get('/database.db', function (req: Request, res: Response) {
+            res.sendFile("./databases/sponsorTimes.db", {root: "./"});
+        });
+    }
 }
diff --git a/src/routes/dumpDatabase.ts b/src/routes/dumpDatabase.ts
new file mode 100644
index 0000000..fb8363d
--- /dev/null
+++ b/src/routes/dumpDatabase.ts
@@ -0,0 +1,61 @@
+import {db} from '../databases/databases';
+import {Logger} from '../utils/logger';
+import {Request, Response} from 'express';
+import { config } from '../config';
+
+const ONE_MINUTE = 1000 * 60;
+
+const styleHeader = `<style>body{font-family: sans-serif}</style>`
+
+const licenseHeader = `<p>The API and database follow <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" rel="nofollow">CC BY-NC-SA 4.0</a> unless you have explicit permission.</p>
+<p><a href="https://gist.github.com/ajayyy/4b27dfc66e33941a45aeaadccb51de71">Attribution Template</a></p>
+<p>If you need to use the database or API in a way that violates this license, contact me with your reason and I may grant you access under a different license.</p></a></p>`;
+
+const tables = [{
+    name: "sponsorTimes",
+    order: "timeSubmitted"
+},
+{
+    name: "userNames"
+},
+{
+    name: "categoryVotes"
+},
+{
+    name: "noSegments",
+},
+{
+    name: "warnings",
+    order: "issueTime"
+},
+{
+    name: "vipUsers"
+}];
+
+const links: string = tables.map((table) => `<p><a href="/database/${table.name}.csv">${table.name}.csv</a></p>`)
+                        .reduce((acc, url) => acc + url, "");
+
+let lastUpdate = 0;
+
+export default function dumpDatabase(req: Request, res: Response) {
+    if (!config.postgres) {
+        res.status(404).send("Not supported on this instance");
+        return;
+    }
+
+    const now = Date.now();
+    const updateQueued = now - lastUpdate > ONE_MINUTE;
+
+    res.status(200).send(`${styleHeader}
+                <h1>SponsorBlock database dumps</h1>${licenseHeader}${links}<br/>
+                ${updateQueued ? `Update queued.` : ``} Last updated: ${lastUpdate ? new Date(lastUpdate).toUTCString() : `Unknown`}`);
+
+    if (updateQueued) {
+        lastUpdate = Date.now();
+
+        for (const table of tables) {
+            db.prepare('run', `COPY (SELECT * FROM "${table.name}"${table.order ? ` ORDER BY "${table.order}"` : ``}) 
+                    TO '/opt/exports/${table.name}.csv' WITH (FORMAT CSV, HEADER true);`);
+        }
+    }
+}
\ No newline at end of file