external/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2024-11-10 01:02:30 +01:00
Code Issues Projects Releases Packages Wiki Activity

replace node-fetch with axios in src

Browse source
This commit is contained in:
Michael C 2021-09-23 01:21:10 -04:00
parent a23387c877
commit 6433f50edf
No known key found for this signature in database
GPG key ID: FFB04FB3B878B7B4
10 changed files with 150 additions and 107 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
package-lock.json generated
View file

@ -11,6 +11,7 @@
"dependencies": {
"@ajayyy/lru-diskcache": "^1.1.9",
"@types/request": "^2.48.7",
"axios": "^0.21.4",
"better-sqlite3": "^7.4.3",
"cron": "^1.8.2",
"express": "^4.17.1",
@ -1111,6 +1112,14 @@
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
"integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k="
},
"node_modules/axios": {
"version": "0.21.4",
"resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz",
"integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==",
"dependencies": {
"follow-redirects": "^1.14.0"
}
},
"node_modules/babel-runtime": {
"version": "6.26.0",
"resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz",
@ -2570,6 +2579,25 @@
"integrity": "sha512-XprP7lDrVT+kE2c2YlfiV+IfS9zxukiIOvNamPNsImNhXadSsQEbosItdL9bUQlCZXR13SvPk20BjWSWLA7m4A==",
"dev": true
},
"node_modules/follow-redirects": {
"version": "1.14.4",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.4.tgz",
"integrity": "sha512-zwGkiSXC1MUJG/qmeIFH2HBJx9u0V46QGUe3YR1fXG8bXQxq7fLj0RjLZQ5nubr9qNJUZrH+xUcwXEoXNpfS+g==",
"funding": [
{
"type": "individual",
"url": "https://github.com/sponsors/RubenVerborgh"
}
],
"engines": {
"node": ">=4.0"
},
"peerDependenciesMeta": {
"debug": {
"optional": true
}
}
},
"node_modules/form-data": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.0.tgz",
@ -6660,6 +6688,14 @@
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
"integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k="
},
"axios": {
"version": "0.21.4",
"resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz",
"integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==",
"requires": {
"follow-redirects": "^1.14.0"
}
},
"babel-runtime": {
"version": "6.26.0",
"resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz",
@ -7785,6 +7821,11 @@
"integrity": "sha512-XprP7lDrVT+kE2c2YlfiV+IfS9zxukiIOvNamPNsImNhXadSsQEbosItdL9bUQlCZXR13SvPk20BjWSWLA7m4A==",
"dev": true
},
"follow-redirects": {
"version": "1.14.4",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.4.tgz",
"integrity": "sha512-zwGkiSXC1MUJG/qmeIFH2HBJx9u0V46QGUe3YR1fXG8bXQxq7fLj0RjLZQ5nubr9qNJUZrH+xUcwXEoXNpfS+g=="
},
"form-data": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.0.tgz",

1
package.json
View file

@ -18,6 +18,7 @@
"dependencies": {
"@ajayyy/lru-diskcache": "^1.1.9",
"@types/request": "^2.48.7",
"axios": "^0.21.4",
"better-sqlite3": "^7.4.3",
"cron": "^1.8.2",
"express": "^4.17.1",

4
src/middleware/userCounter.ts
View file

@ -1,12 +1,12 @@
import fetch from "node-fetch";
import {Logger} from "../utils/logger";
import {config} from "../config";
import {getIP} from "../utils/getIP";
import {getHash} from "../utils/getHash";
import axios from "axios";
import {NextFunction, Request, Response} from "express";
export function userCounter(req: Request, res: Response, next: NextFunction): void {
fetch(`${config.userCounterURL}/api/v1/addIP?hashedIP=${getHash(getIP(req), 1)}`, {method: "POST"})
axios.post(`${config.userCounterURL}/api/v1/addIP?hashedIP=${getHash(getIP(req), 1)}`)
.catch(() => Logger.debug(`Failing to connect to user counter at: ${config.userCounterURL}`));
next();

22
src/routes/getTotalStats.ts
View file

@ -1,8 +1,8 @@
import {db} from "../databases/databases";
import {config} from "../config";
import {Request, Response} from "express";
import fetch from "node-fetch";
import {Logger} from "../utils/logger";
import axios from "axios";
// A cache of the number of chrome web store users
let chromeUsersCache = 0;
@ -44,10 +44,9 @@ export async function getTotalStats(req: Request, res: Response): Promise<void>
function updateExtensionUsers() {
if (config.userCounterURL) {
fetch(`${config.userCounterURL}/api/v1/userCount`)
.then(res => res.json())
.then(data => {
apiUsersCache = Math.max(apiUsersCache, data.userCount);
axios.get(`${config.userCounterURL}/api/v1/userCount`)
.then(res => {
apiUsersCache = Math.max(apiUsersCache, res.data.userCount);
})
.catch(() => Logger.debug(`Failing to connect to user counter at: ${config.userCounterURL}`));
}
@ -55,13 +54,12 @@ function updateExtensionUsers() {
const mozillaAddonsUrl = "https://addons.mozilla.org/api/v3/addons/addon/sponsorblock/";
const chromeExtensionUrl = "https://chrome.google.com/webstore/detail/sponsorblock-for-youtube/mnjggcdmjocbbbhaepdhchncahnbgone";
fetch(mozillaAddonsUrl)
.then(res => res.json())
.then(data => {
firefoxUsersCache = data.average_daily_users;
fetch(chromeExtensionUrl)
.then(res => res.text())
.then(body => {
axios.get(mozillaAddonsUrl)
.then(res => {
firefoxUsersCache = res.data.average_daily_users;
axios.get(chromeExtensionUrl)
.then(res => {
const body = res.data;
// 2021-01-05
// [...]<span><meta itemprop="interactionCount" content="UserDownloads:100.000+"/><meta itemprop="opera[...]
const matchingString = '"UserDownloads:';

93
src/routes/postSkipSegments.ts
View file

@ -3,7 +3,6 @@ import {Logger} from "../utils/logger";
import {db, privateDB} from "../databases/databases";
import {getMaxResThumbnail, YouTubeAPI} from "../utils/youtubeApi";
import {getSubmissionUUID} from "../utils/getSubmissionUUID";
import fetch from "node-fetch";
import {getHash} from "../utils/getHash";
import {getIP} from "../utils/getIP";
import {getFormattedTime} from "../utils/getFormattedTime";
@ -20,6 +19,7 @@ import { UserID } from "../types/user.model";
import { isUserVIP } from "../utils/isUserVIP";
import { parseUserAgent } from "../utils/userAgent";
import { getService } from "../utils/getService";
import axios from "axios";
type CheckResult = {
pass: boolean,
@ -80,28 +80,22 @@ async function sendWebhooks(apiVideoInfo: APIVideoInfo, userID: string, videoID:
// Then send a notification to discord
if (config.discordFirstTimeSubmissionsWebhookURL === null || userSubmissionCountRow.submissionCount > 1) return;
fetch(config.discordFirstTimeSubmissionsWebhookURL, {
method: "POST",
body: JSON.stringify({
"embeds": [{
"title": data?.title,
"url": `https://www.youtube.com/watch?v=${videoID}&t=${(parseInt(startTime.toFixed(0)) - 2)}`,
"description": `Submission ID: ${UUID}\
\n\nTimestamp: \
${getFormattedTime(startTime)} to ${getFormattedTime(endTime)}\
\n\nCategory: ${segmentInfo.category}`,
"color": 10813440,
"author": {
"name": userID,
},
"thumbnail": {
"url": getMaxResThumbnail(data) || "",
},
}],
}),
headers: {
"Content-Type": "application/json"
}
axios.post(config.discordFirstTimeSubmissionsWebhookURL, {
"embeds": [{
"title": data?.title,
"url": `https://www.youtube.com/watch?v=${videoID}&t=${(parseInt(startTime.toFixed(0)) - 2)}`,
"description": `Submission ID: ${UUID}\
\n\nTimestamp: \
${getFormattedTime(startTime)} to ${getFormattedTime(endTime)}\
\n\nCategory: ${segmentInfo.category}`,
"color": 10813440,
"author": {
"name": userID,
},
"thumbnail": {
"url": getMaxResThumbnail(data) || "",
},
}],
})
.then(res => {
if (res.status >= 400) {
@ -136,28 +130,22 @@ async function sendWebhooksNB(userID: string, videoID: string, UUID: string, sta
// Send discord message
if (config.discordNeuralBlockRejectWebhookURL === null) return;
fetch(config.discordNeuralBlockRejectWebhookURL, {
method: "POST",
body: JSON.stringify({
"embeds": [{
"title": ytData.items[0].snippet.title,
"url": `https://www.youtube.com/watch?v=${videoID}&t=${(parseFloat(startTime.toFixed(0)) - 2)}`,
"description": `**Submission ID:** ${UUID}\
\n**Timestamp:** ${getFormattedTime(startTime)} to ${getFormattedTime(endTime)}\
\n**Predicted Probability:** ${probability}\
\n**Category:** ${category}\
\n**Submitted by:** ${submittedBy}\
\n**Total User Submissions:** ${submissionInfoRow.count}\
\n**Ignored User Submissions:** ${submissionInfoRow.disregarded}`,
"color": 10813440,
"thumbnail": {
"url": ytData.items[0].snippet.thumbnails.maxres ? ytData.items[0].snippet.thumbnails.maxres.url : "",
},
}],
}),
headers: {
"Content-Type": "application/json"
}
axios.post(config.discordNeuralBlockRejectWebhookURL, {
"embeds": [{
"title": ytData.items[0].snippet.title,
"url": `https://www.youtube.com/watch?v=${videoID}&t=${(parseFloat(startTime.toFixed(0)) - 2)}`,
"description": `**Submission ID:** ${UUID}\
\n**Timestamp:** ${getFormattedTime(startTime)} to ${getFormattedTime(endTime)}\
\n**Predicted Probability:** ${probability}\
\n**Category:** ${category}\
\n**Submitted by:** ${submittedBy}\
\n**Total User Submissions:** ${submissionInfoRow.count}\
\n**Ignored User Submissions:** ${submissionInfoRow.disregarded}`,
"color": 10813440,
"thumbnail": {
"url": ytData.items[0].snippet.thumbnails.maxres ? ytData.items[0].snippet.thumbnails.maxres.url : "",
},
}]
})
.then(res => {
if (res.status >= 400) {
@ -236,11 +224,11 @@ async function autoModerateSubmission(apiVideoInfo: APIVideoInfo,
// Check NeuralBlock
const neuralBlockURL = config.neuralBlockURL;
if (!neuralBlockURL) return false;
const response = await fetch(`${neuralBlockURL}/api/checkSponsorSegments?vid=${submission.videoID}
&segments=${nbString.substring(0, nbString.length - 1)}`);
if (!response.ok) return false;
const response = await axios.get(`${neuralBlockURL}/api/checkSponsorSegments?vid=${submission.videoID}
&segments=${nbString.substring(0, nbString.length - 1)}`, { validateStatus: () => true});
if (response.status !== 200) return false;
const nbPredictions = await response.json();
const nbPredictions = response.data;
let nbDecision = false;
let predictionIdx = 0; //Keep track because only sponsor categories were submitted
for (let i = 0; i < segments.length; i++) {
@ -531,12 +519,9 @@ async function checkRateLimit(userID:string, videoID: VideoID, timeSubmitted: nu
}
function proxySubmission(req: Request) {
fetch(`${config.proxySubmission}/api/skipSegments?userID=${req.query.userID}&videoID=${req.query.videoID}`, {
method: "POST",
body: req.body,
})
axios.post(`${config.proxySubmission}/api/skipSegments?userID=${req.query.userID}&videoID=${req.query.videoID}`, req.body)
.then(async res => {
Logger.debug(`Proxy Submission: ${res.status} (${(await res.text())})`);
Logger.debug(`Proxy Submission: ${res.status} (${res.data})`);
})
.catch(() => {
Logger.error("Proxy Submission: Failed to make call");

56
src/routes/voteOnSponsorTime.ts
View file

@ -1,7 +1,6 @@
import {Request, Response} from "express";
import {Logger} from "../utils/logger";
import {isUserVIP} from "../utils/isUserVIP";
import fetch from "node-fetch";
import {getMaxResThumbnail, YouTubeAPI} from "../utils/youtubeApi";
import {db, privateDB} from "../databases/databases";
import {dispatchEvent, getVoteAuthor, getVoteAuthorRaw} from "../utils/webhookUtils";
@ -13,6 +12,7 @@ import { UserID } from "../types/user.model";
import { Category, CategoryActionType, HashedIP, IPAddress, SegmentUUID, Service, VideoID, VideoIDHash, Visibility } from "../types/segments.model";
import { getCategoryActionType } from "../utils/categoryInfo";
import { QueryCacher } from "../utils/queryCacher";
import axios from "axios";
const voteTypes = {
normal: 0,
@ -111,40 +111,34 @@ async function sendWebhooks(voteData: VoteData) {
// Send discord message
if (webhookURL !== null && !isUpvote) {
fetch(webhookURL, {
method: "POST",
body: JSON.stringify({
"embeds": [{
"title": data?.title,
"url": `https://www.youtube.com/watch?v=${submissionInfoRow.videoID}&t=${(submissionInfoRow.startTime.toFixed(0) - 2)}`,
"description": `**${voteData.row.votes} Votes Prior | \
${(voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount)} Votes Now | ${voteData.row.views} \
Views**\n\n**Submission ID:** ${voteData.UUID}\
\n**Category:** ${submissionInfoRow.category}\
\n\n**Submitted by:** ${submissionInfoRow.userName}\n${submissionInfoRow.userID}\
\n\n**Total User Submissions:** ${submissionInfoRow.count}\
\n**Ignored User Submissions:** ${submissionInfoRow.disregarded}\
\n\n**Timestamp:** \
${getFormattedTime(submissionInfoRow.startTime)} to ${getFormattedTime(submissionInfoRow.endTime)}`,
"color": 10813440,
"author": {
"name": voteData.finalResponse?.webhookMessage ??
voteData.finalResponse?.finalMessage ??
getVoteAuthor(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission),
},
"thumbnail": {
"url": getMaxResThumbnail(data) || "",
},
}],
}),
headers: {
"Content-Type": "application/json"
}
axios.post(webhookURL, {
"embeds": [{
"title": data?.title,
"url": `https://www.youtube.com/watch?v=${submissionInfoRow.videoID}&t=${(submissionInfoRow.startTime.toFixed(0) - 2)}`,
"description": `**${voteData.row.votes} Votes Prior | \
${(voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount)} Votes Now | ${voteData.row.views} \
Views**\n\n**Submission ID:** ${voteData.UUID}\
\n**Category:** ${submissionInfoRow.category}\
\n\n**Submitted by:** ${submissionInfoRow.userName}\n${submissionInfoRow.userID}\
\n\n**Total User Submissions:** ${submissionInfoRow.count}\
\n**Ignored User Submissions:** ${submissionInfoRow.disregarded}\
\n\n**Timestamp:** \
${getFormattedTime(submissionInfoRow.startTime)} to ${getFormattedTime(submissionInfoRow.endTime)}`,
"color": 10813440,
"author": {
"name": voteData.finalResponse?.webhookMessage ??
voteData.finalResponse?.finalMessage ??
getVoteAuthor(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission),
},
"thumbnail": {
"url": getMaxResThumbnail(data) || "",
},
}],
})
.then(async res => {
if (res.status >= 400) {
Logger.error("Error sending reported submission Discord hook");
Logger.error(JSON.stringify((await res.text())));
Logger.error(JSON.stringify((res.data)));
Logger.error("\n");
}
})

7
src/utils/webhookUtils.ts
View file

@ -1,6 +1,6 @@
import {config} from "../config";
import {Logger} from "../utils/logger";
import fetch from "node-fetch";
import axios from "axios";
function getVoteAuthorRaw(submissionCount: number, isVIP: boolean, isOwnSubmission: boolean): string {
if (isOwnSubmission) {
@ -37,9 +37,10 @@ function dispatchEvent(scope: string, data: Record<string, unknown>): void {
const scopes = webhook.scopes || [];
if (!scopes.includes(scope.toLowerCase())) return;
fetch(webhookURL, {
axios.request({
url: webhookURL,
method: "POST",
body: JSON.stringify(data),
data,
headers: {
"Authorization": authKey,
"Event-Type": scope, // Maybe change this in the future?

8
src/utils/youtubeApi.ts
View file

@ -1,8 +1,8 @@
import fetch from "node-fetch";
import {config} from "../config";
import {Logger} from "./logger";
import { APIVideoData, APIVideoInfo } from "../types/youtubeApi.model";
import DiskCache from "./diskCache";
import axios from "axios";
export class YouTubeAPI {
static async listVideos(videoID: string, ignoreCache = false): Promise<APIVideoInfo> {
@ -27,10 +27,10 @@ export class YouTubeAPI {
if (!config.newLeafURLs || config.newLeafURLs.length <= 0) return {err: "NewLeaf URL not found", data: null};
try {
const result = await fetch(`${config.newLeafURLs[Math.floor(Math.random() * config.newLeafURLs.length)]}/api/v1/videos/${videoID}`, { method: "GET" });
const result = await axios.get(`${config.newLeafURLs[Math.floor(Math.random() * config.newLeafURLs.length)]}/api/v1/videos/${videoID}`);
if (result.ok) {
const data = await result.json();
if (result.status === 200) {
const data = result.data;
if (data.error) {
Logger.warn(`NewLeaf API Error for ${videoID}: ${data.error}`);
return { err: data.error, data: null };

1
test/cases/getLockReason.ts
View file

@ -91,7 +91,6 @@ describe("getLockReason", () => {
.then(async res => {
assert.strictEqual(res.status, 200);
const data = await res.json();
console.log(data);
const expected = [
{ category: "sponsor", locked: 1, reason: "sponsor-reason" },
{ category: "interaction", locked: 1, reason: "interaction-reason" },

24
test/cases/userCounter.ts Normal file
View file

@ -0,0 +1,24 @@
import axios from "axios";
import assert from "assert";
import { config } from "../../src/config";
import { getHash } from "../../src/utils/getHash";
describe("userCounter", () => {
it("Should return 200", (done) => {
if (!config.userCounterURL) return done(); // skip if no userCounterURL is set
axios.request({
method: "POST",
baseURL: config.userCounterURL,
url: "/api/v1/addIP",
params: {
hashedIP: getHash("127.0.0.1",1)
}
})
.then(res => {
assert.strictEqual(res.status, 200);
done();
})
.catch(err => done(err));
});
});