Setup csv exports and html status page

.gitignore

@@ -99,6 +99,7 @@ test/databases/sponsorTimes.db
 test/databases/sponsorTimes.db-shm
 test/databases/sponsorTimes.db-wal
 test/databases/private.db
+docker/database-export
 
 # Config files
 config.json

docker/docker-compose.yml

@@ -7,6 +7,7 @@ services:
       - database.env
     volumes:
       - database-data:/var/lib/postgresql/data
+      - ./database-export/:/opt/exports
     ports:
       - 127.0.0.1:5432:5432
   redis:

src/app.ts

@@ -26,6 +26,7 @@ import {userCounter} from './middleware/userCounter';
 import {loggerMiddleware} from './middleware/logger';
 import {corsMiddleware} from './middleware/cors';
 import {rateLimitMiddleware} from './middleware/requestRateLimit';
+import dumpDatabase from './routes/dumpDatabase';
 
 
 export function createServer(callback: () => void) {
@@ -127,7 +128,11 @@ function setupRoutes(app: Express) {
     //get if user is a vip
     app.post('/api/segmentShift', postSegmentShift);
 
-    app.get('/database.db', function (req: Request, res: Response) {
+    if (config.postgres) {
-        res.sendFile("./databases/sponsorTimes.db", {root: "./"});
+        app.get('/database', dumpDatabase);
-    });
+    } else {
+        app.get('/database.db', function (req: Request, res: Response) {
+            res.sendFile("./databases/sponsorTimes.db", {root: "./"});
+        });
+    }
 }

src/routes/dumpDatabase.ts

@@ -0,0 +1,61 @@
+import {db} from '../databases/databases';
+import {Logger} from '../utils/logger';
+import {Request, Response} from 'express';
+import { config } from '../config';
+
+const ONE_MINUTE = 1000 * 60;
+
+const styleHeader = `<style>body{font-family: sans-serif}</style>`
+
+const licenseHeader = `<p>The API and database follow <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" rel="nofollow">CC BY-NC-SA 4.0</a> unless you have explicit permission.</p>
+<p><a href="https://gist.github.com/ajayyy/4b27dfc66e33941a45aeaadccb51de71">Attribution Template</a></p>
+<p>If you need to use the database or API in a way that violates this license, contact me with your reason and I may grant you access under a different license.</p></a></p>`;
+
+const tables = [{
+    name: "sponsorTimes",
+    order: "timeSubmitted"
+},
+{
+    name: "userNames"
+},
+{
+    name: "categoryVotes"
+},
+{
+    name: "noSegments",
+},
+{
+    name: "warnings",
+    order: "issueTime"
+},
+{
+    name: "vipUsers"
+}];
+
+const links: string = tables.map((table) => `<p><a href="/database/${table.name}.csv">${table.name}.csv</a></p>`)
+                        .reduce((acc, url) => acc + url, "");
+
+let lastUpdate = 0;
+
+export default function dumpDatabase(req: Request, res: Response) {
+    if (!config.postgres) {
+        res.status(404).send("Not supported on this instance");
+        return;
+    }
+
+    const now = Date.now();
+    const updateQueued = now - lastUpdate > ONE_MINUTE;
+
+    res.status(200).send(`${styleHeader}
+                <h1>SponsorBlock database dumps</h1>${licenseHeader}${links}<br/>
+                ${updateQueued ? `Update queued.` : ``} Last updated: ${lastUpdate ? new Date(lastUpdate).toUTCString() : `Unknown`}`);
+
+    if (updateQueued) {
+        lastUpdate = Date.now();
+
+        for (const table of tables) {
+            db.prepare('run', `COPY (SELECT * FROM "${table.name}"${table.order ? ` ORDER BY "${table.order}"` : ``}) 
+                    TO '/opt/exports/${table.name}.csv' WITH (FORMAT CSV, HEADER true);`);
+        }
+    }
+}