mirror of
https://github.com/ajayyy/SponsorBlockServer.git
synced 2024-09-19 20:51:19 +02:00
Add archive downvote segment cron
This commit is contained in:
parent
ef86fceedd
commit
6b5156468c
8 changed files with 149 additions and 2 deletions
|
@ -10,6 +10,7 @@
|
|||
[shadowBannedUsers](#shadowBannedUsers)
|
||||
[unlistedVideos](#unlistedVideos)
|
||||
[config](#config)
|
||||
[archivedSponsorTimes](#archivedSponsorTimes)
|
||||
|
||||
### vipUsers
|
||||
| Name | Type | |
|
||||
|
@ -35,6 +36,7 @@
|
|||
| timeSubmitted | INTEGER | not null |
|
||||
| views | INTEGER | not null |
|
||||
| category | TEXT | not null, default 'sponsor' |
|
||||
| actionType | TEXT | not null, default 'skip' |
|
||||
| service | TEXT | not null, default 'Youtube' |
|
||||
| videoDuration | INTEGER | not null, default '0' |
|
||||
| hidden | INTEGER | not null, default '0' |
|
||||
|
@ -140,7 +142,28 @@
|
|||
| key | TEXT | not null, unique |
|
||||
| value | TEXT | not null |
|
||||
|
||||
### archivedSponsorTimes
|
||||
|
||||
| Name | Type | |
|
||||
| -- | :--: | -- |
|
||||
| videoID | TEXT | not null |
|
||||
| startTime | REAL | not null |
|
||||
| endTime | REAL | not null |
|
||||
| votes | INTEGER | not null |
|
||||
| locked | INTEGER | not null, default '0' |
|
||||
| incorrectVotes | INTEGER | not null, default 1 |
|
||||
| UUID | TEXT | not null, unique |
|
||||
| userID | TEXT | not null |
|
||||
| timeSubmitted | INTEGER | not null |
|
||||
| views | INTEGER | not null |
|
||||
| category | TEXT | not null, default 'sponsor' |
|
||||
| actionType | TEXT | not null, default 'skip' |
|
||||
| service | TEXT | not null, default 'Youtube' |
|
||||
| videoDuration | INTEGER | not null, default '0' |
|
||||
| hidden | INTEGER | not null, default '0' |
|
||||
| reputation | REAL | not null, default '0' |
|
||||
| shadowHidden | INTEGER | not null |
|
||||
| hashedVideoID | TEXT | not null, default '', sha256 |
|
||||
|
||||
# Private
|
||||
|
||||
|
|
26
databases/_upgrade_sponsorTimes_21.sql
Normal file
26
databases/_upgrade_sponsorTimes_21.sql
Normal file
|
@ -0,0 +1,26 @@
|
|||
BEGIN TRANSACTION;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS "archivedSponsorTimes" (
|
||||
"videoID" TEXT NOT NULL,
|
||||
"startTime" REAL NOT NULL,
|
||||
"endTime" REAL NOT NULL,
|
||||
"votes" INTEGER NOT NULL,
|
||||
"locked" INTEGER NOT NULL DEFAULT '0',
|
||||
"incorrectVotes" INTEGER NOT NULL DEFAULT 1,
|
||||
"UUID" TEXT NOT NULL UNIQUE,
|
||||
"userID" TEXT NOT NULL,
|
||||
"timeSubmitted" INTEGER NOT NULL,
|
||||
"views" INTEGER NOT NULL,
|
||||
"category" TEXT NOT NULL DEFAULT 'sponsor',
|
||||
"service" TEXT NOT NULL DEFAULT 'Youtube',
|
||||
"actionType" TEXT NOT NULL DEFAULT 'skip',
|
||||
"videoDuration" INTEGER NOT NULL DEFAULT '0',
|
||||
"hidden" INTEGER NOT NULL DEFAULT '0',
|
||||
"reputation" REAL NOT NULL DEFAULT '0',
|
||||
"shadowHidden" INTEGER NOT NULL,
|
||||
"hashedVideoID" TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
|
||||
UPDATE "config" SET value = 21 WHERE key = 'version';
|
||||
|
||||
COMMIT;
|
|
@ -18,6 +18,7 @@
|
|||
"@ajayyy/lru-diskcache": "^1.1.9",
|
||||
"@types/request": "^2.48.6",
|
||||
"better-sqlite3": "^7.4.1",
|
||||
"cron": "^1.8.2",
|
||||
"express": "^4.17.1",
|
||||
"express-promise-router": "^4.1.0",
|
||||
"express-rate-limit": "^5.3.0",
|
||||
|
@ -28,6 +29,7 @@
|
|||
},
|
||||
"devDependencies": {
|
||||
"@types/better-sqlite3": "^5.4.3",
|
||||
"@types/cron": "^1.7.3",
|
||||
"@types/express": "^4.17.13",
|
||||
"@types/express-rate-limit": "^5.1.3",
|
||||
"@types/mocha": "^8.2.3",
|
||||
|
|
|
@ -77,7 +77,8 @@ addDefaults(config, {
|
|||
name: "vipUsers"
|
||||
}]
|
||||
},
|
||||
diskCache: null
|
||||
diskCache: null,
|
||||
crons: null
|
||||
});
|
||||
|
||||
// Add defaults
|
||||
|
|
63
src/cronjob/downvoteSegmentArchiveJob.ts
Normal file
63
src/cronjob/downvoteSegmentArchiveJob.ts
Normal file
|
@ -0,0 +1,63 @@
|
|||
import { CronJob } from "cron";
|
||||
|
||||
import { config as serverConfig } from "../config";
|
||||
import { Logger } from "../utils/logger";
|
||||
import { db } from "../databases/databases";
|
||||
import { DBSegment } from "../types/segments.model";
|
||||
|
||||
const jobConfig = serverConfig?.crons?.downvoteSegmentArchive;
|
||||
|
||||
export const archiveDownvoteSegment = async (dayLimit: number, voteLimit: number, runTime?: number): Promise<number> => {
|
||||
const timeNow = runTime || new Date().getTime();
|
||||
const threshold = dayLimit * 86400000;
|
||||
|
||||
Logger.info(`DownvoteSegmentArchiveJob starts at ${timeNow}`);
|
||||
try {
|
||||
// insert into archive sponsorTime
|
||||
await db.prepare(
|
||||
'run',
|
||||
`INSERT INTO "archivedSponsorTimes"
|
||||
SELECT *
|
||||
FROM "sponsorTimes"
|
||||
WHERE votes < ? AND (? - timeSubmitted) > ?`,
|
||||
[
|
||||
voteLimit,
|
||||
timeNow,
|
||||
threshold
|
||||
]
|
||||
) as DBSegment[];
|
||||
|
||||
} catch (err) {
|
||||
Logger.error('Execption when insert segment in archivedSponsorTimes');
|
||||
Logger.error(err);
|
||||
return 1;
|
||||
}
|
||||
|
||||
// remove from sponsorTime
|
||||
try {
|
||||
await db.prepare(
|
||||
'run',
|
||||
'DELETE FROM "sponsorTimes" WHERE votes < ? AND (? - timeSubmitted) > ?',
|
||||
[
|
||||
voteLimit,
|
||||
timeNow,
|
||||
threshold
|
||||
]
|
||||
) as DBSegment[];
|
||||
|
||||
} catch (err) {
|
||||
Logger.error('Execption when deleting segment in sponsorTimes');
|
||||
Logger.error(err);
|
||||
return 1;
|
||||
}
|
||||
|
||||
Logger.info('DownvoteSegmentArchiveJob finished');
|
||||
return 0;
|
||||
};
|
||||
|
||||
const DownvoteSegmentArchiveJob = new CronJob(
|
||||
jobConfig?.schedule || new Date(1),
|
||||
() => archiveDownvoteSegment(jobConfig?.timeThresholdInDays, jobConfig?.voteThreshold)
|
||||
);
|
||||
|
||||
export default DownvoteSegmentArchiveJob;
|
13
src/cronjob/index.ts
Normal file
13
src/cronjob/index.ts
Normal file
|
@ -0,0 +1,13 @@
|
|||
import { Logger } from "../utils/logger";
|
||||
import { config } from "../config";
|
||||
import DownvoteSegmentArchiveJob from "./downvoteSegmentArchiveJob";
|
||||
|
||||
export function startAllCrons (): void {
|
||||
if (config?.crons?.enabled) {
|
||||
Logger.info("Crons started");
|
||||
|
||||
DownvoteSegmentArchiveJob.start();
|
||||
} else {
|
||||
Logger.info("Crons dissabled");
|
||||
}
|
||||
}
|
|
@ -2,13 +2,17 @@ import {config} from "./config";
|
|||
import {initDb} from './databases/databases';
|
||||
import {createServer} from "./app";
|
||||
import {Logger} from "./utils/logger";
|
||||
import {startAllCrons} from "./cronjob";
|
||||
|
||||
async function init() {
|
||||
await initDb();
|
||||
|
||||
createServer(() => {
|
||||
Logger.info("Server started on port " + config.port + ".");
|
||||
|
||||
// ignite cron job after server created
|
||||
startAllCrons();
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
init();
|
|
@ -44,6 +44,7 @@ export interface SBSConfig {
|
|||
postgres?: PoolConfig;
|
||||
dumpDatabase?: DumpDatabase;
|
||||
diskCache: CacheOptions;
|
||||
crons: CronJobOptions;
|
||||
}
|
||||
|
||||
export interface WebhookConfig {
|
||||
|
@ -81,3 +82,17 @@ export interface DumpDatabaseTable {
|
|||
name: string;
|
||||
order?: string;
|
||||
}
|
||||
|
||||
export interface CronJobDefault {
|
||||
schedule: string;
|
||||
}
|
||||
|
||||
export interface CronJobOptions {
|
||||
enabled: boolean;
|
||||
downvoteSegmentArchive: CronJobDefault & DownvoteSegmentArchiveCron;
|
||||
}
|
||||
|
||||
export interface DownvoteSegmentArchiveCron {
|
||||
voteThreshold: number;
|
||||
timeThresholdInDays: number;
|
||||
}
|
Loading…
Reference in a new issue