From 6b5156468ca1ca60f97396f2e79a5cfbd9ebc9c4 Mon Sep 17 00:00:00 2001 From: Haidang666 Date: Fri, 9 Jul 2021 11:46:04 +0700 Subject: [PATCH] Add archive downvote segment cron --- DatabaseSchema.md | 23 +++++++++ databases/_upgrade_sponsorTimes_21.sql | 26 ++++++++++ package.json | 2 + src/config.ts | 3 +- src/cronjob/downvoteSegmentArchiveJob.ts | 63 ++++++++++++++++++++++++ src/cronjob/index.ts | 13 +++++ src/index.ts | 6 ++- src/types/config.model.ts | 15 ++++++ 8 files changed, 149 insertions(+), 2 deletions(-) create mode 100644 databases/_upgrade_sponsorTimes_21.sql create mode 100644 src/cronjob/downvoteSegmentArchiveJob.ts create mode 100644 src/cronjob/index.ts diff --git a/DatabaseSchema.md b/DatabaseSchema.md index 485c4c3..26f2319 100644 --- a/DatabaseSchema.md +++ b/DatabaseSchema.md @@ -10,6 +10,7 @@ [shadowBannedUsers](#shadowBannedUsers) [unlistedVideos](#unlistedVideos) [config](#config) +[archivedSponsorTimes](#archivedSponsorTimes) ### vipUsers | Name | Type | | @@ -35,6 +36,7 @@ | timeSubmitted | INTEGER | not null | | views | INTEGER | not null | | category | TEXT | not null, default 'sponsor' | +| actionType | TEXT | not null, default 'skip' | | service | TEXT | not null, default 'Youtube' | | videoDuration | INTEGER | not null, default '0' | | hidden | INTEGER | not null, default '0' | @@ -140,7 +142,28 @@ | key | TEXT | not null, unique | | value | TEXT | not null | +### archivedSponsorTimes +| Name | Type | | +| -- | :--: | -- | +| videoID | TEXT | not null | +| startTime | REAL | not null | +| endTime | REAL | not null | +| votes | INTEGER | not null | +| locked | INTEGER | not null, default '0' | +| incorrectVotes | INTEGER | not null, default 1 | +| UUID | TEXT | not null, unique | +| userID | TEXT | not null | +| timeSubmitted | INTEGER | not null | +| views | INTEGER | not null | +| category | TEXT | not null, default 'sponsor' | +| actionType | TEXT | not null, default 'skip' | +| service | TEXT | not null, default 'Youtube' | +| videoDuration | INTEGER | not null, default '0' | +| hidden | INTEGER | not null, default '0' | +| reputation | REAL | not null, default '0' | +| shadowHidden | INTEGER | not null | +| hashedVideoID | TEXT | not null, default '', sha256 | # Private diff --git a/databases/_upgrade_sponsorTimes_21.sql b/databases/_upgrade_sponsorTimes_21.sql new file mode 100644 index 0000000..4e850bd --- /dev/null +++ b/databases/_upgrade_sponsorTimes_21.sql @@ -0,0 +1,26 @@ +BEGIN TRANSACTION; + +CREATE TABLE IF NOT EXISTS "archivedSponsorTimes" ( + "videoID" TEXT NOT NULL, + "startTime" REAL NOT NULL, + "endTime" REAL NOT NULL, + "votes" INTEGER NOT NULL, + "locked" INTEGER NOT NULL DEFAULT '0', + "incorrectVotes" INTEGER NOT NULL DEFAULT 1, + "UUID" TEXT NOT NULL UNIQUE, + "userID" TEXT NOT NULL, + "timeSubmitted" INTEGER NOT NULL, + "views" INTEGER NOT NULL, + "category" TEXT NOT NULL DEFAULT 'sponsor', + "service" TEXT NOT NULL DEFAULT 'Youtube', + "actionType" TEXT NOT NULL DEFAULT 'skip', + "videoDuration" INTEGER NOT NULL DEFAULT '0', + "hidden" INTEGER NOT NULL DEFAULT '0', + "reputation" REAL NOT NULL DEFAULT '0', + "shadowHidden" INTEGER NOT NULL, + "hashedVideoID" TEXT NOT NULL DEFAULT '' +); + +UPDATE "config" SET value = 21 WHERE key = 'version'; + +COMMIT; diff --git a/package.json b/package.json index 56386f6..fcba30c 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "@ajayyy/lru-diskcache": "^1.1.9", "@types/request": "^2.48.6", "better-sqlite3": "^7.4.1", + "cron": "^1.8.2", "express": "^4.17.1", "express-promise-router": "^4.1.0", "express-rate-limit": "^5.3.0", @@ -28,6 +29,7 @@ }, "devDependencies": { "@types/better-sqlite3": "^5.4.3", + "@types/cron": "^1.7.3", "@types/express": "^4.17.13", "@types/express-rate-limit": "^5.1.3", "@types/mocha": "^8.2.3", diff --git a/src/config.ts b/src/config.ts index 1e900b9..9002df7 100644 --- a/src/config.ts +++ b/src/config.ts @@ -77,7 +77,8 @@ addDefaults(config, { name: "vipUsers" }] }, - diskCache: null + diskCache: null, + crons: null }); // Add defaults diff --git a/src/cronjob/downvoteSegmentArchiveJob.ts b/src/cronjob/downvoteSegmentArchiveJob.ts new file mode 100644 index 0000000..c2f8f4e --- /dev/null +++ b/src/cronjob/downvoteSegmentArchiveJob.ts @@ -0,0 +1,63 @@ +import { CronJob } from "cron"; + +import { config as serverConfig } from "../config"; +import { Logger } from "../utils/logger"; +import { db } from "../databases/databases"; +import { DBSegment } from "../types/segments.model"; + +const jobConfig = serverConfig?.crons?.downvoteSegmentArchive; + +export const archiveDownvoteSegment = async (dayLimit: number, voteLimit: number, runTime?: number): Promise => { + const timeNow = runTime || new Date().getTime(); + const threshold = dayLimit * 86400000; + + Logger.info(`DownvoteSegmentArchiveJob starts at ${timeNow}`); + try { + // insert into archive sponsorTime + await db.prepare( + 'run', + `INSERT INTO "archivedSponsorTimes" + SELECT * + FROM "sponsorTimes" + WHERE votes < ? AND (? - timeSubmitted) > ?`, + [ + voteLimit, + timeNow, + threshold + ] + ) as DBSegment[]; + + } catch (err) { + Logger.error('Execption when insert segment in archivedSponsorTimes'); + Logger.error(err); + return 1; + } + + // remove from sponsorTime + try { + await db.prepare( + 'run', + 'DELETE FROM "sponsorTimes" WHERE votes < ? AND (? - timeSubmitted) > ?', + [ + voteLimit, + timeNow, + threshold + ] + ) as DBSegment[]; + + } catch (err) { + Logger.error('Execption when deleting segment in sponsorTimes'); + Logger.error(err); + return 1; + } + + Logger.info('DownvoteSegmentArchiveJob finished'); + return 0; +}; + +const DownvoteSegmentArchiveJob = new CronJob( + jobConfig?.schedule || new Date(1), + () => archiveDownvoteSegment(jobConfig?.timeThresholdInDays, jobConfig?.voteThreshold) +); + +export default DownvoteSegmentArchiveJob; diff --git a/src/cronjob/index.ts b/src/cronjob/index.ts new file mode 100644 index 0000000..46aa80c --- /dev/null +++ b/src/cronjob/index.ts @@ -0,0 +1,13 @@ +import { Logger } from "../utils/logger"; +import { config } from "../config"; +import DownvoteSegmentArchiveJob from "./downvoteSegmentArchiveJob"; + +export function startAllCrons (): void { + if (config?.crons?.enabled) { + Logger.info("Crons started"); + + DownvoteSegmentArchiveJob.start(); + } else { + Logger.info("Crons dissabled"); + } +} diff --git a/src/index.ts b/src/index.ts index 13576a7..babb1a9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,13 +2,17 @@ import {config} from "./config"; import {initDb} from './databases/databases'; import {createServer} from "./app"; import {Logger} from "./utils/logger"; +import {startAllCrons} from "./cronjob"; async function init() { await initDb(); createServer(() => { Logger.info("Server started on port " + config.port + "."); + + // ignite cron job after server created + startAllCrons(); }); } - + init(); \ No newline at end of file diff --git a/src/types/config.model.ts b/src/types/config.model.ts index 1dabdbe..cb5daa2 100644 --- a/src/types/config.model.ts +++ b/src/types/config.model.ts @@ -44,6 +44,7 @@ export interface SBSConfig { postgres?: PoolConfig; dumpDatabase?: DumpDatabase; diskCache: CacheOptions; + crons: CronJobOptions; } export interface WebhookConfig { @@ -81,3 +82,17 @@ export interface DumpDatabaseTable { name: string; order?: string; } + +export interface CronJobDefault { + schedule: string; +} + +export interface CronJobOptions { + enabled: boolean; + downvoteSegmentArchive: CronJobDefault & DownvoteSegmentArchiveCron; +} + +export interface DownvoteSegmentArchiveCron { + voteThreshold: number; + timeThresholdInDays: number; +} \ No newline at end of file