diff --git a/.gitignore b/.gitignore index dd78e49..fa2dd8b 100644 --- a/.gitignore +++ b/.gitignore @@ -99,6 +99,7 @@ test/databases/sponsorTimes.db test/databases/sponsorTimes.db-shm test/databases/sponsorTimes.db-wal test/databases/private.db +databases/cache docker/database-export # Config files diff --git a/package-lock.json b/package-lock.json index 4d3d560..a90825d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "0.1.0", "license": "MIT", "dependencies": { + "@ajayyy/lru-diskcache": "^1.1.9", "abort-controller": "^3.0.0", "better-sqlite3": "^7.1.5", "dotenv": "^8.2.0", @@ -40,6 +41,20 @@ "typescript": "^4.0.3" } }, + "node_modules/@ajayyy/lru-diskcache": { + "version": "1.1.9", + "resolved": "https://registry.npmjs.org/@ajayyy/lru-diskcache/-/lru-diskcache-1.1.9.tgz", + "integrity": "sha512-RT04axsNvACDXYW2vSdBnrIYFTTk4jpHg39P29BQv/vZmckLyo2LTH7wZhfRui7Wlllrff2Z12T418pdGhohRw==", + "dependencies": { + "crc": "^3.4.0", + "fs-extra": "^0.26.5", + "fswrite-stream": "^1.0.0", + "lodash": "^4.17.10", + "lru-cache": "^4.1.5", + "q": "^1.4.1", + "tmp": "^0.0.28" + } + }, "node_modules/@sinonjs/commons": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.1.tgz", @@ -452,8 +467,7 @@ "node_modules/balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", - "dev": true + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" }, "node_modules/barrage": { "version": "1.1.0", @@ -616,7 +630,6 @@ "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dev": true, "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -858,8 +871,7 @@ "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", - "dev": true + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" }, "node_modules/configstore": { "version": "3.1.2", @@ -930,6 +942,14 @@ "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, + "node_modules/crc": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/crc/-/crc-3.8.0.tgz", + "integrity": "sha512-iX3mfgcTMIq3ZKLIsVFAbv7+Mc10kxabAGQb8HvjA1o3T1PIYprbakQ65d3I+2HGHt6nSKkM9PYjgoJO2KcFBQ==", + "dependencies": { + "buffer": "^5.1.0" + } + }, "node_modules/create-error-class": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/create-error-class/-/create-error-class-3.0.2.tgz", @@ -1318,6 +1338,18 @@ "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" }, + "node_modules/fs-extra": { + "version": "0.26.7", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-0.26.7.tgz", + "integrity": "sha1-muH92UiXeY7at20JGM9C0MMYT6k=", + "dependencies": { + "graceful-fs": "^4.1.2", + "jsonfile": "^2.1.0", + "klaw": "^1.0.0", + "path-is-absolute": "^1.0.0", + "rimraf": "^2.2.8" + } + }, "node_modules/fs-minipass": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", @@ -1332,8 +1364,7 @@ "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", - "dev": true + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" }, "node_modules/fsevents": { "version": "2.3.2", @@ -1349,6 +1380,14 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/fswrite-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fswrite-stream/-/fswrite-stream-1.0.0.tgz", + "integrity": "sha1-esPP5gkt3/mYQ1Z1UCOJ5lKjb64=", + "dependencies": { + "length-stream": "^0.1.1" + } + }, "node_modules/gauge": { "version": "2.7.4", "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", @@ -1423,7 +1462,6 @@ "version": "7.1.6", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", - "dev": true, "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -1488,8 +1526,7 @@ "node_modules/graceful-fs": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.3.tgz", - "integrity": "sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ==", - "dev": true + "integrity": "sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ==" }, "node_modules/growl": { "version": "1.10.5", @@ -1601,7 +1638,6 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", - "dev": true, "dependencies": { "once": "^1.3.0", "wrappy": "1" @@ -1795,12 +1831,28 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/jsonfile": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz", + "integrity": "sha1-NzaitCi4e72gzIO1P6PWM6NcKug=", + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, "node_modules/just-extend": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.1.1.tgz", "integrity": "sha512-aWgeGFW67BP3e5181Ep1Fv2v8z//iBJfrvyTnq8wG86vEESwmonn1zPBJ0VfmT9CJq2FIT0VsETtrNFm2a+SHA==", "dev": true }, + "node_modules/klaw": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/klaw/-/klaw-1.3.1.tgz", + "integrity": "sha1-QIhDO0azsbolnXh4XY6W9zugJDk=", + "optionalDependencies": { + "graceful-fs": "^4.1.9" + } + }, "node_modules/latest-version": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/latest-version/-/latest-version-3.1.0.tgz", @@ -1813,6 +1865,17 @@ "node": ">=4" } }, + "node_modules/length-stream": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/length-stream/-/length-stream-0.1.1.tgz", + "integrity": "sha1-5ySxvi46lh1MQxNDfkllaRD65tY=", + "dependencies": { + "pass-stream": "~0.1.0" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/locate-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", @@ -1828,6 +1891,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + }, "node_modules/lodash.get": { "version": "4.4.2", "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", @@ -1859,7 +1927,6 @@ "version": "4.1.5", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", - "dev": true, "dependencies": { "pseudomap": "^1.0.2", "yallist": "^2.1.2" @@ -1868,8 +1935,7 @@ "node_modules/lru-cache/node_modules/yallist": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", - "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=", - "dev": true + "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=" }, "node_modules/make-dir": { "version": "1.3.0", @@ -1955,7 +2021,6 @@ "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", - "dev": true, "dependencies": { "brace-expansion": "^1.1.7" }, @@ -2393,6 +2458,14 @@ "wrappy": "1" } }, + "node_modules/os-tmpdir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/p-finally": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", @@ -2460,6 +2533,23 @@ "node": ">= 0.8" } }, + "node_modules/pass-stream": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/pass-stream/-/pass-stream-0.1.5.tgz", + "integrity": "sha1-njr6TVglzdE3YHW9tW3jbfszZJ8=", + "dependencies": { + "readable-stream": "https://github.com/jeffbski/readable-stream/archive/v1.0.2-object-transform2-ret-self.tar.gz" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/pass-stream/node_modules/readable-stream": { + "version": "1.0.2", + "resolved": "https://github.com/jeffbski/readable-stream/archive/v1.0.2-object-transform2-ret-self.tar.gz", + "integrity": "sha512-4DfqsAX+1OToUZ/uHLsC3NUHN7wCn2BNfkG/Eyr2ii4c4suSipjLFbVymS7pj49+aRcSutnK9dIuHS3X7fwApQ==", + "license": "BSD" + }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -2473,7 +2563,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -2683,8 +2772,7 @@ "node_modules/pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", - "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=", - "dev": true + "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=" }, "node_modules/pstree.remy": { "version": "1.1.7", @@ -2701,6 +2789,15 @@ "once": "^1.3.1" } }, + "node_modules/q": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz", + "integrity": "sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc=", + "engines": { + "node": ">=0.6.0", + "teleport": ">=0.2.0" + } + }, "node_modules/qs": { "version": "6.7.0", "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", @@ -2857,6 +2954,17 @@ "node": ">=0.10.0" } }, + "node_modules/rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + } + }, "node_modules/safe-buffer": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.0.tgz", @@ -3301,6 +3409,17 @@ "node": ">=0.10.0" } }, + "node_modules/tmp": { + "version": "0.0.28", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.28.tgz", + "integrity": "sha1-Fyc1t/YU6nrzlmT6hM8N5OUV0SA=", + "dependencies": { + "os-tmpdir": "~1.0.1" + }, + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -3813,6 +3932,20 @@ } }, "dependencies": { + "@ajayyy/lru-diskcache": { + "version": "1.1.9", + "resolved": "https://registry.npmjs.org/@ajayyy/lru-diskcache/-/lru-diskcache-1.1.9.tgz", + "integrity": "sha512-RT04axsNvACDXYW2vSdBnrIYFTTk4jpHg39P29BQv/vZmckLyo2LTH7wZhfRui7Wlllrff2Z12T418pdGhohRw==", + "requires": { + "crc": "^3.4.0", + "fs-extra": "^0.26.5", + "fswrite-stream": "^1.0.0", + "lodash": "^4.17.10", + "lru-cache": "^4.1.5", + "q": "^1.4.1", + "tmp": "^0.0.28" + } + }, "@sinonjs/commons": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.1.tgz", @@ -4204,8 +4337,7 @@ "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", - "dev": true + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" }, "barrage": { "version": "1.1.0", @@ -4349,7 +4481,6 @@ "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dev": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -4526,8 +4657,7 @@ "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", - "dev": true + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" }, "configstore": { "version": "3.1.2", @@ -4588,6 +4718,14 @@ "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, + "crc": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/crc/-/crc-3.8.0.tgz", + "integrity": "sha512-iX3mfgcTMIq3ZKLIsVFAbv7+Mc10kxabAGQb8HvjA1o3T1PIYprbakQ65d3I+2HGHt6nSKkM9PYjgoJO2KcFBQ==", + "requires": { + "buffer": "^5.1.0" + } + }, "create-error-class": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/create-error-class/-/create-error-class-3.0.2.tgz", @@ -4888,6 +5026,18 @@ "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" }, + "fs-extra": { + "version": "0.26.7", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-0.26.7.tgz", + "integrity": "sha1-muH92UiXeY7at20JGM9C0MMYT6k=", + "requires": { + "graceful-fs": "^4.1.2", + "jsonfile": "^2.1.0", + "klaw": "^1.0.0", + "path-is-absolute": "^1.0.0", + "rimraf": "^2.2.8" + } + }, "fs-minipass": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", @@ -4899,8 +5049,7 @@ "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", - "dev": true + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" }, "fsevents": { "version": "2.3.2", @@ -4909,6 +5058,14 @@ "dev": true, "optional": true }, + "fswrite-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fswrite-stream/-/fswrite-stream-1.0.0.tgz", + "integrity": "sha1-esPP5gkt3/mYQ1Z1UCOJ5lKjb64=", + "requires": { + "length-stream": "^0.1.1" + } + }, "gauge": { "version": "2.7.4", "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", @@ -4970,7 +5127,6 @@ "version": "7.1.6", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", - "dev": true, "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -5020,8 +5176,7 @@ "graceful-fs": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.3.tgz", - "integrity": "sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ==", - "dev": true + "integrity": "sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ==" }, "growl": { "version": "1.10.5", @@ -5098,7 +5253,6 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", - "dev": true, "requires": { "once": "^1.3.0", "wrappy": "1" @@ -5244,12 +5398,28 @@ "argparse": "^2.0.1" } }, + "jsonfile": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz", + "integrity": "sha1-NzaitCi4e72gzIO1P6PWM6NcKug=", + "requires": { + "graceful-fs": "^4.1.6" + } + }, "just-extend": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.1.1.tgz", "integrity": "sha512-aWgeGFW67BP3e5181Ep1Fv2v8z//iBJfrvyTnq8wG86vEESwmonn1zPBJ0VfmT9CJq2FIT0VsETtrNFm2a+SHA==", "dev": true }, + "klaw": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/klaw/-/klaw-1.3.1.tgz", + "integrity": "sha1-QIhDO0azsbolnXh4XY6W9zugJDk=", + "requires": { + "graceful-fs": "^4.1.9" + } + }, "latest-version": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/latest-version/-/latest-version-3.1.0.tgz", @@ -5259,6 +5429,14 @@ "package-json": "^4.0.0" } }, + "length-stream": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/length-stream/-/length-stream-0.1.1.tgz", + "integrity": "sha1-5ySxvi46lh1MQxNDfkllaRD65tY=", + "requires": { + "pass-stream": "~0.1.0" + } + }, "locate-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", @@ -5268,6 +5446,11 @@ "p-locate": "^5.0.0" } }, + "lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + }, "lodash.get": { "version": "4.4.2", "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", @@ -5293,7 +5476,6 @@ "version": "4.1.5", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", - "dev": true, "requires": { "pseudomap": "^1.0.2", "yallist": "^2.1.2" @@ -5302,8 +5484,7 @@ "yallist": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", - "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=", - "dev": true + "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=" } } }, @@ -5364,7 +5545,6 @@ "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", - "dev": true, "requires": { "brace-expansion": "^1.1.7" } @@ -5708,6 +5888,11 @@ "wrappy": "1" } }, + "os-tmpdir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=" + }, "p-finally": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", @@ -5754,6 +5939,20 @@ "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==" }, + "pass-stream": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/pass-stream/-/pass-stream-0.1.5.tgz", + "integrity": "sha1-njr6TVglzdE3YHW9tW3jbfszZJ8=", + "requires": { + "readable-stream": "https://github.com/jeffbski/readable-stream/archive/v1.0.2-object-transform2-ret-self.tar.gz" + }, + "dependencies": { + "readable-stream": { + "version": "https://github.com/jeffbski/readable-stream/archive/v1.0.2-object-transform2-ret-self.tar.gz", + "integrity": "sha512-4DfqsAX+1OToUZ/uHLsC3NUHN7wCn2BNfkG/Eyr2ii4c4suSipjLFbVymS7pj49+aRcSutnK9dIuHS3X7fwApQ==" + } + } + }, "path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -5763,8 +5962,7 @@ "path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", - "dev": true + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" }, "path-is-inside": { "version": "1.0.2", @@ -5931,8 +6129,7 @@ "pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", - "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=", - "dev": true + "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=" }, "pstree.remy": { "version": "1.1.7", @@ -5949,6 +6146,11 @@ "once": "^1.3.1" } }, + "q": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz", + "integrity": "sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc=" + }, "qs": { "version": "6.7.0", "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", @@ -6068,6 +6270,14 @@ "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=", "dev": true }, + "rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "requires": { + "glob": "^7.1.3" + } + }, "safe-buffer": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.0.tgz", @@ -6441,6 +6651,14 @@ "integrity": "sha1-8y6srFoXW+ol1/q1Zas+2HQe9W8=", "dev": true }, + "tmp": { + "version": "0.0.28", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.28.tgz", + "integrity": "sha1-Fyc1t/YU6nrzlmT6hM8N5OUV0SA=", + "requires": { + "os-tmpdir": "~1.0.1" + } + }, "to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", diff --git a/package.json b/package.json index 214f242..f8b69b9 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "author": "Ajay Ramachandran", "license": "MIT", "dependencies": { + "@ajayyy/lru-diskcache": "^1.1.9", "abort-controller": "^3.0.0", "better-sqlite3": "^7.1.5", "dotenv": "^8.2.0", diff --git a/src/config.ts b/src/config.ts index afebd4f..fca30c6 100644 --- a/src/config.ts +++ b/src/config.ts @@ -75,7 +75,8 @@ addDefaults(config, { { name: "vipUsers" }] - } + }, + diskCache: null }); // Add defaults diff --git a/src/types/config.model.ts b/src/types/config.model.ts index dc16d0d..3685919 100644 --- a/src/types/config.model.ts +++ b/src/types/config.model.ts @@ -1,5 +1,6 @@ import { PoolConfig } from 'pg'; import * as redis from 'redis'; +import { CacheOptions } from "@ajayyy/lru-diskcache"; export interface SBSConfig { port: number; @@ -41,6 +42,7 @@ export interface SBSConfig { maxRewardTimePerSegmentInSeconds?: number; postgres?: PoolConfig; dumpDatabase?: DumpDatabase; + diskCache: CacheOptions; } export interface WebhookConfig { diff --git a/src/types/lib.model.ts b/src/types/lib.model.ts index b58087f..e148eae 100644 --- a/src/types/lib.model.ts +++ b/src/types/lib.model.ts @@ -1,5 +1,5 @@ /** - * Better ecord that will work with branded types + * Better record that will work with branded types * Keys still don't work properly though and are always string */ export type SBRecord = { diff --git a/src/utils/diskCache.ts b/src/utils/diskCache.ts new file mode 100644 index 0000000..f98d961 --- /dev/null +++ b/src/utils/diskCache.ts @@ -0,0 +1,32 @@ +import LRU from "@ajayyy/lru-diskcache"; +import { config } from "../config"; + +let DiskCache: LRU; + +if (config.diskCache) { + DiskCache = new LRU('./databases/cache', config.diskCache); + DiskCache.init(); +} else { + DiskCache = { + // constructor(rootPath, options): {}; + + init() {}, + + reset() {}, + + has(key) { return false; }, + + get(key, opts) { return null; }, + + // Returns size + set(key, dataOrSteam) { return new Promise((resolve) => 0); }, + + del(key) {}, + + size() { return 0; }, + + prune() {}, + }; +} + +export default DiskCache; \ No newline at end of file diff --git a/src/utils/youtubeApi.ts b/src/utils/youtubeApi.ts index ded8769..a523351 100644 --- a/src/utils/youtubeApi.ts +++ b/src/utils/youtubeApi.ts @@ -1,8 +1,8 @@ import fetch from 'node-fetch'; import {config} from '../config'; import {Logger} from './logger'; -import redis from './redis'; import { APIVideoData, APIVideoInfo } from '../types/youtubeApi.model'; +import DiskCache from './diskCache'; export class YouTubeAPI { static async listVideos(videoID: string, ignoreCache = false): Promise { @@ -10,14 +10,17 @@ export class YouTubeAPI { return { err: "Invalid video ID" }; } - const redisKey = "yt.newleaf.video." + videoID; + const cacheKey = "yt.newleaf.video." + videoID; if (!ignoreCache) { - const {err, reply} = await redis.getAsync(redisKey); + try { + const data = await DiskCache.get(cacheKey); - if (!err && reply) { - Logger.debug("redis: no cache for video information: " + videoID); - - return { err: err?.message, data: JSON.parse(reply) } + if (data) { + Logger.debug("YouTube API: cache used for video information: " + videoID); + return { err: null, data: JSON.parse(data) } + } + } catch (err) { + return { err } } } @@ -33,13 +36,9 @@ export class YouTubeAPI { return { err: data.error, data: null }; } - redis.setAsync(redisKey, JSON.stringify(data)).then((result) => { - if (result?.err) { - Logger.warn(result?.err.message); - } else { - Logger.debug("redis: video information cache set for: " + videoID); - } - }); + DiskCache.set(cacheKey, JSON.stringify(data)) + .catch((err) => Logger.warn(err)) + .then(() => Logger.debug("YouTube API: video information cache set for: " + videoID)); return { err: false, data }; } else {