From 6433f50edff216410663bb1bbbd3c372a2f72e87 Mon Sep 17 00:00:00 2001 From: Michael C Date: Thu, 23 Sep 2021 01:21:10 -0400 Subject: [PATCH] replace node-fetch with axios in src --- package-lock.json | 41 +++++++++++++++ package.json | 1 + src/middleware/userCounter.ts | 4 +- src/routes/getTotalStats.ts | 22 ++++---- src/routes/postSkipSegments.ts | 93 ++++++++++++++------------------- src/routes/voteOnSponsorTime.ts | 56 +++++++++----------- src/utils/webhookUtils.ts | 7 +-- src/utils/youtubeApi.ts | 8 +-- test/cases/getLockReason.ts | 1 - test/cases/userCounter.ts | 24 +++++++++ 10 files changed, 150 insertions(+), 107 deletions(-) create mode 100644 test/cases/userCounter.ts diff --git a/package-lock.json b/package-lock.json index 25dccee..ff8c266 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "dependencies": { "@ajayyy/lru-diskcache": "^1.1.9", "@types/request": "^2.48.7", + "axios": "^0.21.4", "better-sqlite3": "^7.4.3", "cron": "^1.8.2", "express": "^4.17.1", @@ -1111,6 +1112,14 @@ "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" }, + "node_modules/axios": { + "version": "0.21.4", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz", + "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==", + "dependencies": { + "follow-redirects": "^1.14.0" + } + }, "node_modules/babel-runtime": { "version": "6.26.0", "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz", @@ -2570,6 +2579,25 @@ "integrity": "sha512-XprP7lDrVT+kE2c2YlfiV+IfS9zxukiIOvNamPNsImNhXadSsQEbosItdL9bUQlCZXR13SvPk20BjWSWLA7m4A==", "dev": true }, + "node_modules/follow-redirects": { + "version": "1.14.4", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.4.tgz", + "integrity": "sha512-zwGkiSXC1MUJG/qmeIFH2HBJx9u0V46QGUe3YR1fXG8bXQxq7fLj0RjLZQ5nubr9qNJUZrH+xUcwXEoXNpfS+g==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, "node_modules/form-data": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.0.tgz", @@ -6660,6 +6688,14 @@ "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" }, + "axios": { + "version": "0.21.4", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz", + "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==", + "requires": { + "follow-redirects": "^1.14.0" + } + }, "babel-runtime": { "version": "6.26.0", "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz", @@ -7785,6 +7821,11 @@ "integrity": "sha512-XprP7lDrVT+kE2c2YlfiV+IfS9zxukiIOvNamPNsImNhXadSsQEbosItdL9bUQlCZXR13SvPk20BjWSWLA7m4A==", "dev": true }, + "follow-redirects": { + "version": "1.14.4", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.4.tgz", + "integrity": "sha512-zwGkiSXC1MUJG/qmeIFH2HBJx9u0V46QGUe3YR1fXG8bXQxq7fLj0RjLZQ5nubr9qNJUZrH+xUcwXEoXNpfS+g==" + }, "form-data": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.0.tgz", diff --git a/package.json b/package.json index 04103ed..5a2caa6 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "dependencies": { "@ajayyy/lru-diskcache": "^1.1.9", "@types/request": "^2.48.7", + "axios": "^0.21.4", "better-sqlite3": "^7.4.3", "cron": "^1.8.2", "express": "^4.17.1", diff --git a/src/middleware/userCounter.ts b/src/middleware/userCounter.ts index 0818a9e..b4fbbad 100644 --- a/src/middleware/userCounter.ts +++ b/src/middleware/userCounter.ts @@ -1,12 +1,12 @@ -import fetch from "node-fetch"; import {Logger} from "../utils/logger"; import {config} from "../config"; import {getIP} from "../utils/getIP"; import {getHash} from "../utils/getHash"; +import axios from "axios"; import {NextFunction, Request, Response} from "express"; export function userCounter(req: Request, res: Response, next: NextFunction): void { - fetch(`${config.userCounterURL}/api/v1/addIP?hashedIP=${getHash(getIP(req), 1)}`, {method: "POST"}) + axios.post(`${config.userCounterURL}/api/v1/addIP?hashedIP=${getHash(getIP(req), 1)}`) .catch(() => Logger.debug(`Failing to connect to user counter at: ${config.userCounterURL}`)); next(); diff --git a/src/routes/getTotalStats.ts b/src/routes/getTotalStats.ts index 4db4601..2beabc2 100644 --- a/src/routes/getTotalStats.ts +++ b/src/routes/getTotalStats.ts @@ -1,8 +1,8 @@ import {db} from "../databases/databases"; import {config} from "../config"; import {Request, Response} from "express"; -import fetch from "node-fetch"; import {Logger} from "../utils/logger"; +import axios from "axios"; // A cache of the number of chrome web store users let chromeUsersCache = 0; @@ -44,10 +44,9 @@ export async function getTotalStats(req: Request, res: Response): Promise function updateExtensionUsers() { if (config.userCounterURL) { - fetch(`${config.userCounterURL}/api/v1/userCount`) - .then(res => res.json()) - .then(data => { - apiUsersCache = Math.max(apiUsersCache, data.userCount); + axios.get(`${config.userCounterURL}/api/v1/userCount`) + .then(res => { + apiUsersCache = Math.max(apiUsersCache, res.data.userCount); }) .catch(() => Logger.debug(`Failing to connect to user counter at: ${config.userCounterURL}`)); } @@ -55,13 +54,12 @@ function updateExtensionUsers() { const mozillaAddonsUrl = "https://addons.mozilla.org/api/v3/addons/addon/sponsorblock/"; const chromeExtensionUrl = "https://chrome.google.com/webstore/detail/sponsorblock-for-youtube/mnjggcdmjocbbbhaepdhchncahnbgone"; - fetch(mozillaAddonsUrl) - .then(res => res.json()) - .then(data => { - firefoxUsersCache = data.average_daily_users; - fetch(chromeExtensionUrl) - .then(res => res.text()) - .then(body => { + axios.get(mozillaAddonsUrl) + .then(res => { + firefoxUsersCache = res.data.average_daily_users; + axios.get(chromeExtensionUrl) + .then(res => { + const body = res.data; // 2021-01-05 // [...] 1) return; - fetch(config.discordFirstTimeSubmissionsWebhookURL, { - method: "POST", - body: JSON.stringify({ - "embeds": [{ - "title": data?.title, - "url": `https://www.youtube.com/watch?v=${videoID}&t=${(parseInt(startTime.toFixed(0)) - 2)}`, - "description": `Submission ID: ${UUID}\ - \n\nTimestamp: \ - ${getFormattedTime(startTime)} to ${getFormattedTime(endTime)}\ - \n\nCategory: ${segmentInfo.category}`, - "color": 10813440, - "author": { - "name": userID, - }, - "thumbnail": { - "url": getMaxResThumbnail(data) || "", - }, - }], - }), - headers: { - "Content-Type": "application/json" - } + axios.post(config.discordFirstTimeSubmissionsWebhookURL, { + "embeds": [{ + "title": data?.title, + "url": `https://www.youtube.com/watch?v=${videoID}&t=${(parseInt(startTime.toFixed(0)) - 2)}`, + "description": `Submission ID: ${UUID}\ + \n\nTimestamp: \ + ${getFormattedTime(startTime)} to ${getFormattedTime(endTime)}\ + \n\nCategory: ${segmentInfo.category}`, + "color": 10813440, + "author": { + "name": userID, + }, + "thumbnail": { + "url": getMaxResThumbnail(data) || "", + }, + }], }) .then(res => { if (res.status >= 400) { @@ -136,28 +130,22 @@ async function sendWebhooksNB(userID: string, videoID: string, UUID: string, sta // Send discord message if (config.discordNeuralBlockRejectWebhookURL === null) return; - fetch(config.discordNeuralBlockRejectWebhookURL, { - method: "POST", - body: JSON.stringify({ - "embeds": [{ - "title": ytData.items[0].snippet.title, - "url": `https://www.youtube.com/watch?v=${videoID}&t=${(parseFloat(startTime.toFixed(0)) - 2)}`, - "description": `**Submission ID:** ${UUID}\ - \n**Timestamp:** ${getFormattedTime(startTime)} to ${getFormattedTime(endTime)}\ - \n**Predicted Probability:** ${probability}\ - \n**Category:** ${category}\ - \n**Submitted by:** ${submittedBy}\ - \n**Total User Submissions:** ${submissionInfoRow.count}\ - \n**Ignored User Submissions:** ${submissionInfoRow.disregarded}`, - "color": 10813440, - "thumbnail": { - "url": ytData.items[0].snippet.thumbnails.maxres ? ytData.items[0].snippet.thumbnails.maxres.url : "", - }, - }], - }), - headers: { - "Content-Type": "application/json" - } + axios.post(config.discordNeuralBlockRejectWebhookURL, { + "embeds": [{ + "title": ytData.items[0].snippet.title, + "url": `https://www.youtube.com/watch?v=${videoID}&t=${(parseFloat(startTime.toFixed(0)) - 2)}`, + "description": `**Submission ID:** ${UUID}\ + \n**Timestamp:** ${getFormattedTime(startTime)} to ${getFormattedTime(endTime)}\ + \n**Predicted Probability:** ${probability}\ + \n**Category:** ${category}\ + \n**Submitted by:** ${submittedBy}\ + \n**Total User Submissions:** ${submissionInfoRow.count}\ + \n**Ignored User Submissions:** ${submissionInfoRow.disregarded}`, + "color": 10813440, + "thumbnail": { + "url": ytData.items[0].snippet.thumbnails.maxres ? ytData.items[0].snippet.thumbnails.maxres.url : "", + }, + }] }) .then(res => { if (res.status >= 400) { @@ -236,11 +224,11 @@ async function autoModerateSubmission(apiVideoInfo: APIVideoInfo, // Check NeuralBlock const neuralBlockURL = config.neuralBlockURL; if (!neuralBlockURL) return false; - const response = await fetch(`${neuralBlockURL}/api/checkSponsorSegments?vid=${submission.videoID} - &segments=${nbString.substring(0, nbString.length - 1)}`); - if (!response.ok) return false; + const response = await axios.get(`${neuralBlockURL}/api/checkSponsorSegments?vid=${submission.videoID} + &segments=${nbString.substring(0, nbString.length - 1)}`, { validateStatus: () => true}); + if (response.status !== 200) return false; - const nbPredictions = await response.json(); + const nbPredictions = response.data; let nbDecision = false; let predictionIdx = 0; //Keep track because only sponsor categories were submitted for (let i = 0; i < segments.length; i++) { @@ -531,12 +519,9 @@ async function checkRateLimit(userID:string, videoID: VideoID, timeSubmitted: nu } function proxySubmission(req: Request) { - fetch(`${config.proxySubmission}/api/skipSegments?userID=${req.query.userID}&videoID=${req.query.videoID}`, { - method: "POST", - body: req.body, - }) + axios.post(`${config.proxySubmission}/api/skipSegments?userID=${req.query.userID}&videoID=${req.query.videoID}`, req.body) .then(async res => { - Logger.debug(`Proxy Submission: ${res.status} (${(await res.text())})`); + Logger.debug(`Proxy Submission: ${res.status} (${res.data})`); }) .catch(() => { Logger.error("Proxy Submission: Failed to make call"); diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts index 82cc446..e20572b 100644 --- a/src/routes/voteOnSponsorTime.ts +++ b/src/routes/voteOnSponsorTime.ts @@ -1,7 +1,6 @@ import {Request, Response} from "express"; import {Logger} from "../utils/logger"; import {isUserVIP} from "../utils/isUserVIP"; -import fetch from "node-fetch"; import {getMaxResThumbnail, YouTubeAPI} from "../utils/youtubeApi"; import {db, privateDB} from "../databases/databases"; import {dispatchEvent, getVoteAuthor, getVoteAuthorRaw} from "../utils/webhookUtils"; @@ -13,6 +12,7 @@ import { UserID } from "../types/user.model"; import { Category, CategoryActionType, HashedIP, IPAddress, SegmentUUID, Service, VideoID, VideoIDHash, Visibility } from "../types/segments.model"; import { getCategoryActionType } from "../utils/categoryInfo"; import { QueryCacher } from "../utils/queryCacher"; +import axios from "axios"; const voteTypes = { normal: 0, @@ -111,40 +111,34 @@ async function sendWebhooks(voteData: VoteData) { // Send discord message if (webhookURL !== null && !isUpvote) { - fetch(webhookURL, { - method: "POST", - body: JSON.stringify({ - "embeds": [{ - "title": data?.title, - "url": `https://www.youtube.com/watch?v=${submissionInfoRow.videoID}&t=${(submissionInfoRow.startTime.toFixed(0) - 2)}`, - "description": `**${voteData.row.votes} Votes Prior | \ - ${(voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount)} Votes Now | ${voteData.row.views} \ - Views**\n\n**Submission ID:** ${voteData.UUID}\ - \n**Category:** ${submissionInfoRow.category}\ - \n\n**Submitted by:** ${submissionInfoRow.userName}\n${submissionInfoRow.userID}\ - \n\n**Total User Submissions:** ${submissionInfoRow.count}\ - \n**Ignored User Submissions:** ${submissionInfoRow.disregarded}\ - \n\n**Timestamp:** \ - ${getFormattedTime(submissionInfoRow.startTime)} to ${getFormattedTime(submissionInfoRow.endTime)}`, - "color": 10813440, - "author": { - "name": voteData.finalResponse?.webhookMessage ?? - voteData.finalResponse?.finalMessage ?? - getVoteAuthor(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission), - }, - "thumbnail": { - "url": getMaxResThumbnail(data) || "", - }, - }], - }), - headers: { - "Content-Type": "application/json" - } + axios.post(webhookURL, { + "embeds": [{ + "title": data?.title, + "url": `https://www.youtube.com/watch?v=${submissionInfoRow.videoID}&t=${(submissionInfoRow.startTime.toFixed(0) - 2)}`, + "description": `**${voteData.row.votes} Votes Prior | \ + ${(voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount)} Votes Now | ${voteData.row.views} \ + Views**\n\n**Submission ID:** ${voteData.UUID}\ + \n**Category:** ${submissionInfoRow.category}\ + \n\n**Submitted by:** ${submissionInfoRow.userName}\n${submissionInfoRow.userID}\ + \n\n**Total User Submissions:** ${submissionInfoRow.count}\ + \n**Ignored User Submissions:** ${submissionInfoRow.disregarded}\ + \n\n**Timestamp:** \ + ${getFormattedTime(submissionInfoRow.startTime)} to ${getFormattedTime(submissionInfoRow.endTime)}`, + "color": 10813440, + "author": { + "name": voteData.finalResponse?.webhookMessage ?? + voteData.finalResponse?.finalMessage ?? + getVoteAuthor(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission), + }, + "thumbnail": { + "url": getMaxResThumbnail(data) || "", + }, + }], }) .then(async res => { if (res.status >= 400) { Logger.error("Error sending reported submission Discord hook"); - Logger.error(JSON.stringify((await res.text()))); + Logger.error(JSON.stringify((res.data))); Logger.error("\n"); } }) diff --git a/src/utils/webhookUtils.ts b/src/utils/webhookUtils.ts index 2b5153a..11ca36b 100644 --- a/src/utils/webhookUtils.ts +++ b/src/utils/webhookUtils.ts @@ -1,6 +1,6 @@ import {config} from "../config"; import {Logger} from "../utils/logger"; -import fetch from "node-fetch"; +import axios from "axios"; function getVoteAuthorRaw(submissionCount: number, isVIP: boolean, isOwnSubmission: boolean): string { if (isOwnSubmission) { @@ -37,9 +37,10 @@ function dispatchEvent(scope: string, data: Record): void { const scopes = webhook.scopes || []; if (!scopes.includes(scope.toLowerCase())) return; - fetch(webhookURL, { + axios.request({ + url: webhookURL, method: "POST", - body: JSON.stringify(data), + data, headers: { "Authorization": authKey, "Event-Type": scope, // Maybe change this in the future? diff --git a/src/utils/youtubeApi.ts b/src/utils/youtubeApi.ts index 96dd669..6428a9c 100644 --- a/src/utils/youtubeApi.ts +++ b/src/utils/youtubeApi.ts @@ -1,8 +1,8 @@ -import fetch from "node-fetch"; import {config} from "../config"; import {Logger} from "./logger"; import { APIVideoData, APIVideoInfo } from "../types/youtubeApi.model"; import DiskCache from "./diskCache"; +import axios from "axios"; export class YouTubeAPI { static async listVideos(videoID: string, ignoreCache = false): Promise { @@ -27,10 +27,10 @@ export class YouTubeAPI { if (!config.newLeafURLs || config.newLeafURLs.length <= 0) return {err: "NewLeaf URL not found", data: null}; try { - const result = await fetch(`${config.newLeafURLs[Math.floor(Math.random() * config.newLeafURLs.length)]}/api/v1/videos/${videoID}`, { method: "GET" }); + const result = await axios.get(`${config.newLeafURLs[Math.floor(Math.random() * config.newLeafURLs.length)]}/api/v1/videos/${videoID}`); - if (result.ok) { - const data = await result.json(); + if (result.status === 200) { + const data = result.data; if (data.error) { Logger.warn(`NewLeaf API Error for ${videoID}: ${data.error}`); return { err: data.error, data: null }; diff --git a/test/cases/getLockReason.ts b/test/cases/getLockReason.ts index 156a50d..3dbcd5b 100644 --- a/test/cases/getLockReason.ts +++ b/test/cases/getLockReason.ts @@ -91,7 +91,6 @@ describe("getLockReason", () => { .then(async res => { assert.strictEqual(res.status, 200); const data = await res.json(); - console.log(data); const expected = [ { category: "sponsor", locked: 1, reason: "sponsor-reason" }, { category: "interaction", locked: 1, reason: "interaction-reason" }, diff --git a/test/cases/userCounter.ts b/test/cases/userCounter.ts new file mode 100644 index 0000000..26a1adb --- /dev/null +++ b/test/cases/userCounter.ts @@ -0,0 +1,24 @@ +import axios from "axios"; +import assert from "assert"; +import { config } from "../../src/config"; +import { getHash } from "../../src/utils/getHash"; + + +describe("userCounter", () => { + it("Should return 200", (done) => { + if (!config.userCounterURL) return done(); // skip if no userCounterURL is set + axios.request({ + method: "POST", + baseURL: config.userCounterURL, + url: "/api/v1/addIP", + params: { + hashedIP: getHash("127.0.0.1",1) + } + }) + .then(res => { + assert.strictEqual(res.status, 200); + done(); + }) + .catch(err => done(err)); + }); +}); \ No newline at end of file