Add archive downvote segment cron

DatabaseSchema.md

@@ -10,6 +10,7 @@
 [shadowBannedUsers](#shadowBannedUsers)  
 [unlistedVideos](#unlistedVideos)  
 [config](#config)  
+[archivedSponsorTimes](#archivedSponsorTimes)
 
 ### vipUsers
 | Name | Type | |
@@ -35,6 +36,7 @@
 | timeSubmitted | INTEGER | not null |
 | views | INTEGER | not null |
 | category | TEXT | not null, default 'sponsor' |
+| actionType | TEXT | not null, default 'skip' |
 | service | TEXT | not null, default 'Youtube' |
 | videoDuration | INTEGER | not null, default '0' |
 | hidden | INTEGER | not null, default '0' |
@@ -140,7 +142,28 @@
 | key | TEXT | not null, unique |
 | value | TEXT | not null |
 
+### archivedSponsorTimes
 
+| Name | Type | |
+| -- | :--: | -- |
+| videoID | TEXT | not null |
+| startTime | REAL | not null |
+| endTime | REAL | not null |
+| votes | INTEGER | not null |
+| locked | INTEGER | not null, default '0' |
+| incorrectVotes | INTEGER | not null, default 1 |
+| UUID | TEXT | not null, unique |
+| userID | TEXT | not null |
+| timeSubmitted | INTEGER | not null |
+| views | INTEGER | not null |
+| category | TEXT | not null, default 'sponsor' |
+| actionType | TEXT | not null, default 'skip' |
+| service | TEXT | not null, default 'Youtube' |
+| videoDuration | INTEGER | not null, default '0' |
+| hidden | INTEGER | not null, default '0' |
+| reputation | REAL | not null, default '0' |
+| shadowHidden | INTEGER | not null |
+| hashedVideoID | TEXT | not null, default '', sha256 |
 
 # Private 
 

databases/_upgrade_sponsorTimes_21.sql

@@ -0,0 +1,26 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE IF NOT EXISTS "archivedSponsorTimes" (
+	"videoID" TEXT NOT NULL,
+	"startTime" REAL NOT NULL,
+	"endTime" REAL NOT NULL,
+	"votes" INTEGER NOT NULL,
+	"locked" INTEGER NOT NULL DEFAULT '0',
+	"incorrectVotes" INTEGER NOT NULL DEFAULT 1,
+	"UUID" TEXT NOT NULL UNIQUE,
+	"userID" TEXT NOT NULL,
+	"timeSubmitted" INTEGER NOT NULL,
+	"views" INTEGER NOT NULL,
+	"category" TEXT NOT NULL DEFAULT 'sponsor',
+	"service" TEXT NOT NULL DEFAULT 'Youtube',
+	"actionType" TEXT NOT NULL DEFAULT 'skip',
+	"videoDuration" INTEGER NOT NULL DEFAULT '0',
+	"hidden" INTEGER NOT NULL DEFAULT '0',
+	"reputation" REAL NOT NULL DEFAULT '0',
+	"shadowHidden" INTEGER NOT NULL,
+	"hashedVideoID" TEXT NOT NULL DEFAULT ''
+);
+
+UPDATE "config" SET value = 21 WHERE key = 'version';
+
+COMMIT;

package.json

@@ -18,6 +18,7 @@
     "@ajayyy/lru-diskcache": "^1.1.9",
     "@types/request": "^2.48.6",
     "better-sqlite3": "^7.4.1",
+    "cron": "^1.8.2",
     "express": "^4.17.1",
     "express-promise-router": "^4.1.0",
     "express-rate-limit": "^5.3.0",
@@ -28,6 +29,7 @@
   },
   "devDependencies": {
     "@types/better-sqlite3": "^5.4.3",
+    "@types/cron": "^1.7.3",
     "@types/express": "^4.17.13",
     "@types/express-rate-limit": "^5.1.3",
     "@types/mocha": "^8.2.3",

src/config.ts

@@ -77,7 +77,8 @@ addDefaults(config, {
             name: "vipUsers"
         }]
     },
-    diskCache: null
+    diskCache: null,
+    crons: null
 });
 
 // Add defaults

src/cronjob/downvoteSegmentArchiveJob.ts

@@ -0,0 +1,63 @@
+import { CronJob } from "cron";
+
+import { config as serverConfig } from "../config";
+import { Logger } from "../utils/logger";
+import { db } from "../databases/databases";
+import { DBSegment } from "../types/segments.model";
+
+const jobConfig = serverConfig?.crons?.downvoteSegmentArchive;
+
+export const archiveDownvoteSegment = async (dayLimit: number, voteLimit: number, runTime?: number): Promise<number> => {
+  const timeNow = runTime || new Date().getTime();
+  const threshold = dayLimit * 86400000;
+
+  Logger.info(`DownvoteSegmentArchiveJob starts at ${timeNow}`);
+  try {
+    // insert into archive sponsorTime
+    await db.prepare(
+      'run',
+      `INSERT INTO "archivedSponsorTimes" 
+        SELECT * 
+        FROM "sponsorTimes" 
+        WHERE votes < ? AND (? - timeSubmitted) > ?`,
+      [
+        voteLimit,
+        timeNow,
+        threshold
+      ]
+    ) as DBSegment[];
+
+  } catch (err) {
+    Logger.error('Execption when insert segment in archivedSponsorTimes');
+    Logger.error(err);
+    return 1;
+  }
+
+  // remove from sponsorTime
+  try {
+    await db.prepare(
+      'run', 
+      'DELETE FROM "sponsorTimes" WHERE votes < ? AND (? - timeSubmitted) > ?',
+      [
+        voteLimit,
+        timeNow,
+        threshold
+      ]
+    ) as DBSegment[];
+
+  } catch (err) {
+    Logger.error('Execption when deleting segment in sponsorTimes');
+    Logger.error(err);
+    return 1;
+  }
+
+  Logger.info('DownvoteSegmentArchiveJob finished');
+  return 0;
+};
+
+const DownvoteSegmentArchiveJob = new CronJob(
+  jobConfig?.schedule || new Date(1),
+  () => archiveDownvoteSegment(jobConfig?.timeThresholdInDays, jobConfig?.voteThreshold)
+);
+
+export default DownvoteSegmentArchiveJob;

src/cronjob/index.ts

@@ -0,0 +1,13 @@
+import { Logger } from "../utils/logger";
+import { config } from "../config";
+import DownvoteSegmentArchiveJob from "./downvoteSegmentArchiveJob";
+
+export function startAllCrons (): void {
+  if (config?.crons?.enabled) {
+    Logger.info("Crons started");
+
+    DownvoteSegmentArchiveJob.start();
+  } else {
+    Logger.info("Crons dissabled");
+  }
+}

src/index.ts

@@ -2,13 +2,17 @@ import {config} from "./config";
 import {initDb} from './databases/databases';
 import {createServer} from "./app";
 import {Logger} from "./utils/logger";
+import {startAllCrons} from "./cronjob";
 
 async function init() {
     await initDb();
 
     createServer(() => {
         Logger.info("Server started on port " + config.port + ".");
+
+        // ignite cron job after server created
+        startAllCrons();
     });
 }
- 
+
 init();

src/types/config.model.ts

@@ -44,6 +44,7 @@ export interface SBSConfig {
     postgres?: PoolConfig;
     dumpDatabase?: DumpDatabase;
     diskCache: CacheOptions;
+    crons: CronJobOptions;
 }
 
 export interface WebhookConfig {
@@ -81,3 +82,17 @@ export interface DumpDatabaseTable {
     name: string;
     order?: string;
 }
+
+export interface CronJobDefault {
+    schedule: string;
+}
+
+export interface CronJobOptions {
+    enabled: boolean;
+    downvoteSegmentArchive: CronJobDefault & DownvoteSegmentArchiveCron;
+}
+
+export interface DownvoteSegmentArchiveCron {
+    voteThreshold: number;
+    timeThresholdInDays: number;
+}