diff --git a/package-lock.json b/package-lock.json index 718d4ee..5e45422 100644 --- a/package-lock.json +++ b/package-lock.json @@ -42,6 +42,7 @@ "pino": "^10.1.0", "pino-http": "^11.0.0", "prisma": "^5.22.0", + "puppeteer": "^24.40.0", "reflect-metadata": "^0.2.2", "rxjs": "^7.8.1", "zod": "^4.3.5" @@ -1115,7 +1116,6 @@ "version": "7.27.1", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.27.1.tgz", "integrity": "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==", - "dev": true, "dependencies": { "@babel/helper-validator-identifier": "^7.27.1", "js-tokens": "^4.0.0", @@ -1275,7 +1275,6 @@ "version": "7.28.5", "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", - "dev": true, "engines": { "node": ">=6.9.0" } @@ -3772,6 +3771,39 @@ "@prisma/debug": "5.22.0" } }, + "node_modules/@puppeteer/browsers": { + "version": "2.13.0", + "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.13.0.tgz", + "integrity": "sha512-46BZJYJjc/WwmKjsvDFykHtXrtomsCIrwYQPOP7VfMJoZY2bsDF9oROBABR3paDjDcmkUye1Pb1BqdcdiipaWA==", + "license": "Apache-2.0", + "dependencies": { + "debug": "^4.4.3", + "extract-zip": "^2.0.1", + "progress": "^2.0.3", + "proxy-agent": "^6.5.0", + "semver": "^7.7.4", + "tar-fs": "^3.1.1", + "yargs": "^17.7.2" + }, + "bin": { + "browsers": "lib/cjs/main-cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@puppeteer/browsers/node_modules/semver": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/@redis/bloom": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz", @@ -4567,6 +4599,12 @@ "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz", "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==" }, + "node_modules/@tootallnate/quickjs-emscripten": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", + "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==", + "license": "MIT" + }, "node_modules/@tsconfig/node10": { "version": "1.0.12", "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.12.tgz", @@ -4925,6 +4963,16 @@ "integrity": "sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==", "dev": true }, + "node_modules/@types/yauzl": { + "version": "2.10.3", + "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", + "integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==", + "license": "MIT", + "optional": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "8.52.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.52.0.tgz", @@ -5847,6 +5895,18 @@ "node": ">=0.8" } }, + "node_modules/ast-types": { + "version": "0.13.4", + "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", + "integrity": "sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==", + "license": "MIT", + "dependencies": { + "tslib": "^2.0.1" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -5875,6 +5935,20 @@ "integrity": "sha512-lHe62zvbTB5eEABUVi/AwVh0ZKY9rMMDhmm+eeyuuUQbQ3+J+fONVQOZyj+DdrvD4BY33uYniyRJ4UJIaSKAfw==", "license": "MIT" }, + "node_modules/b4a": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.8.0.tgz", + "integrity": "sha512-qRuSmNSkGQaHwNbM7J78Wwy+ghLEYF1zNrSeMxj4Kgw6y33O3mXcQ6Ie9fRvfU/YnxWkOchPXbaLb73TkIsfdg==", + "license": "Apache-2.0", + "peerDependencies": { + "react-native-b4a": "*" + }, + "peerDependenciesMeta": { + "react-native-b4a": { + "optional": true + } + } + }, "node_modules/babel-jest": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-30.2.0.tgz", @@ -5971,6 +6045,93 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, + "node_modules/bare-events": { + "version": "2.8.2", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.8.2.tgz", + "integrity": "sha512-riJjyv1/mHLIPX4RwiK+oW9/4c3TEUeORHKefKAKnZ5kyslbN+HXowtbaVEqt4IMUB7OXlfixcs6gsFeo/jhiQ==", + "license": "Apache-2.0", + "peerDependencies": { + "bare-abort-controller": "*" + }, + "peerDependenciesMeta": { + "bare-abort-controller": { + "optional": true + } + } + }, + "node_modules/bare-fs": { + "version": "4.5.6", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.5.6.tgz", + "integrity": "sha512-1QovqDrR80Pmt5HPAsMsXTCFcDYr+NSUKW6nd6WO5v0JBmnItc/irNRzm2KOQ5oZ69P37y+AMujNyNtG+1Rggw==", + "license": "Apache-2.0", + "dependencies": { + "bare-events": "^2.5.4", + "bare-path": "^3.0.0", + "bare-stream": "^2.6.4", + "bare-url": "^2.2.2", + "fast-fifo": "^1.3.2" + }, + "engines": { + "bare": ">=1.16.0" + }, + "peerDependencies": { + "bare-buffer": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + } + } + }, + "node_modules/bare-os": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-3.8.0.tgz", + "integrity": "sha512-Dc9/SlwfxkXIGYhvMQNUtKaXCaGkZYGcd1vuNUUADVqzu4/vQfvnMkYYOUnt2VwQ2AqKr/8qAVFRtwETljgeFg==", + "license": "Apache-2.0", + "engines": { + "bare": ">=1.14.0" + } + }, + "node_modules/bare-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-3.0.0.tgz", + "integrity": "sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw==", + "license": "Apache-2.0", + "dependencies": { + "bare-os": "^3.0.1" + } + }, + "node_modules/bare-stream": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.10.0.tgz", + "integrity": "sha512-DOPZF/DDcDruKDA43cOw6e9Quq5daua7ygcAwJE/pKJsRWhgSSemi7qVNGE5kyDIxIeN1533G/zfbvWX7Wcb9w==", + "license": "Apache-2.0", + "dependencies": { + "streamx": "^2.25.0", + "teex": "^1.0.1" + }, + "peerDependencies": { + "bare-buffer": "*", + "bare-events": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + }, + "bare-events": { + "optional": true + } + } + }, + "node_modules/bare-url": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/bare-url/-/bare-url-2.4.0.tgz", + "integrity": "sha512-NSTU5WN+fy/L0DDenfE8SXQna4voXuW0FHM7wH8i3/q9khUSchfPbPezO4zSFMnDGIf9YE+mt/RWhZgNRKRIXA==", + "license": "Apache-2.0", + "dependencies": { + "bare-path": "^3.0.0" + } + }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -6007,6 +6168,15 @@ "baseline-browser-mapping": "dist/cli.js" } }, + "node_modules/basic-ftp": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.2.0.tgz", + "integrity": "sha512-VoMINM2rqJwJgfdHq6RiUudKt2BV+FY5ZFezP/ypmwayk68+NzzAQy4XXLlqsGD4MCzq3DrmNFD/uUmBJuGoXw==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/bcrypt": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/bcrypt/-/bcrypt-6.0.0.tgz", @@ -6251,6 +6421,15 @@ "ieee754": "^1.1.13" } }, + "node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, "node_modules/buffer-equal-constant-time": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", @@ -6412,7 +6591,6 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", - "dev": true, "engines": { "node": ">=6" } @@ -6556,6 +6734,28 @@ "node": ">=6.0" } }, + "node_modules/chromium-bidi": { + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-14.0.0.tgz", + "integrity": "sha512-9gYlLtS6tStdRWzrtXaTMnqcM4dudNegMXJxkR0I/CXObHalYeYcAMPrL19eroNZHtJ8DQmu1E+ZNOYu/IXMXw==", + "license": "Apache-2.0", + "dependencies": { + "mitt": "^3.0.1", + "zod": "^3.24.1" + }, + "peerDependencies": { + "devtools-protocol": "*" + } + }, + "node_modules/chromium-bidi/node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, "node_modules/ci-info": { "version": "4.3.1", "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.3.1.tgz", @@ -6655,7 +6855,6 @@ "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", - "dev": true, "dependencies": { "string-width": "^4.2.0", "strip-ansi": "^6.0.1", @@ -6669,7 +6868,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", @@ -7044,6 +7242,20 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/degenerator": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", + "integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==", + "license": "MIT", + "dependencies": { + "ast-types": "^0.13.4", + "escodegen": "^2.1.0", + "esprima": "^4.0.1" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -7086,6 +7298,12 @@ "node": ">=8" } }, + "node_modules/devtools-protocol": { + "version": "0.0.1581282", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1581282.tgz", + "integrity": "sha512-nv7iKtNZQshSW2hKzYNr46nM/Cfh5SEvE2oV0/SEGgc9XupIY5ggf84Cz8eJIkBce7S3bmTAauFD6aysMpnqsQ==", + "license": "BSD-3-Clause" + }, "node_modules/dezalgo": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz", @@ -7286,7 +7504,6 @@ "version": "1.4.5", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", - "dev": true, "dependencies": { "once": "^1.4.0" } @@ -7382,11 +7599,19 @@ "url": "https://github.com/fb55/entities?sponsor=1" } }, + "node_modules/env-paths": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", + "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/error-ex": { "version": "1.3.4", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.4.tgz", "integrity": "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==", - "dev": true, "dependencies": { "is-arrayish": "^0.2.1" } @@ -7444,7 +7669,6 @@ "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", - "dev": true, "engines": { "node": ">=6" } @@ -7466,6 +7690,37 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/escodegen": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", + "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", + "license": "BSD-2-Clause", + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^5.2.0", + "esutils": "^2.0.2" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=6.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, + "node_modules/escodegen/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "license": "BSD-3-Clause", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/eslint": { "version": "9.39.2", "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.2.tgz", @@ -7619,7 +7874,6 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", - "dev": true, "bin": { "esparse": "bin/esparse.js", "esvalidate": "bin/esvalidate.js" @@ -7656,7 +7910,6 @@ "version": "5.3.0", "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", - "dev": true, "engines": { "node": ">=4.0" } @@ -7665,7 +7918,6 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -7692,6 +7944,15 @@ "node": ">=0.8.x" } }, + "node_modules/events-universal": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/events-universal/-/events-universal-1.0.1.tgz", + "integrity": "sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==", + "license": "Apache-2.0", + "dependencies": { + "bare-events": "^2.7.0" + } + }, "node_modules/execa": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", @@ -7794,6 +8055,41 @@ "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" }, + "node_modules/extract-zip": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", + "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", + "license": "BSD-2-Clause", + "dependencies": { + "debug": "^4.1.1", + "get-stream": "^5.1.0", + "yauzl": "^2.10.0" + }, + "bin": { + "extract-zip": "cli.js" + }, + "engines": { + "node": ">= 10.17.0" + }, + "optionalDependencies": { + "@types/yauzl": "^2.9.1" + } + }, + "node_modules/extract-zip/node_modules/get-stream": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", + "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", + "license": "MIT", + "dependencies": { + "pump": "^3.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/extsprintf": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", @@ -7820,6 +8116,12 @@ "integrity": "sha512-VxPP4NqbUjj6MaAOafWeUn2cXWLcCtljklUtZf0Ind4XQ+QPtmA0b18zZy0jIQx+ExRVCR/ZQpBmik5lXshNsw==", "dev": true }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", + "license": "MIT" + }, "node_modules/fast-json-stable-stringify": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", @@ -7878,6 +8180,15 @@ "bser": "2.1.1" } }, + "node_modules/fd-slicer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", + "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", + "license": "MIT", + "dependencies": { + "pend": "~1.2.0" + } + }, "node_modules/fdir": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", @@ -8319,6 +8630,29 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/get-uri": { + "version": "6.0.5", + "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.5.tgz", + "integrity": "sha512-b1O07XYq8eRuVzBNgJLstU6FYc1tS6wnMtF1I1D9lE8LxZSOGZ7LhxN54yPP6mGw5f2CkXY2BQUL9Fx41qvcIg==", + "license": "MIT", + "dependencies": { + "basic-ftp": "^5.0.2", + "data-uri-to-buffer": "^6.0.2", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/get-uri/node_modules/data-uri-to-buffer": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz", + "integrity": "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, "node_modules/getpass": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", @@ -8634,6 +8968,19 @@ "url": "https://opencollective.com/express" } }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/http-signature": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", @@ -8717,7 +9064,6 @@ "version": "3.3.1", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", - "dev": true, "dependencies": { "parent-module": "^1.0.0", "resolve-from": "^4.0.0" @@ -8796,6 +9142,15 @@ "url": "https://opencollective.com/ioredis" } }, + "node_modules/ip-address": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.1.0.tgz", + "integrity": "sha512-XXADHxXmvT9+CRxhXg56LJovE+bmWnEWB78LB83VZTprKTmaC5QfruXocxzTZ2Kl0DNwKuBdlIhjL8LeY8Sf8Q==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -8807,8 +9162,7 @@ "node_modules/is-arrayish": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", - "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", - "dev": true + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==" }, "node_modules/is-binary-path": { "version": "2.1.0", @@ -9724,8 +10078,7 @@ "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", - "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", - "dev": true + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==" }, "node_modules/js-yaml": { "version": "4.1.1", @@ -9773,8 +10126,7 @@ "node_modules/json-parse-even-better-errors": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", - "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", - "dev": true + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==" }, "node_modules/json-schema": { "version": "0.4.0", @@ -9922,8 +10274,7 @@ "node_modules/lines-and-columns": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", - "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", - "dev": true + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==" }, "node_modules/load-esm": { "version": "1.0.3", @@ -10267,6 +10618,12 @@ "node": ">=16 || 14 >=14.17" } }, + "node_modules/mitt": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz", + "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==", + "license": "MIT" + }, "node_modules/mkdirp": { "version": "0.5.6", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", @@ -10504,6 +10861,15 @@ "rxjs": "^7.1.0" } }, + "node_modules/netmask": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", + "integrity": "sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, "node_modules/node-abort-controller": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.1.1.tgz", @@ -10796,6 +11162,38 @@ "node": ">=6" } }, + "node_modules/pac-proxy-agent": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.2.0.tgz", + "integrity": "sha512-TEB8ESquiLMc0lV8vcd5Ql/JAKAoyzHFXaStwjkzpOpC5Yv+pIzLfHvjTSdf3vpa2bMiUQrg9i6276yn8666aA==", + "license": "MIT", + "dependencies": { + "@tootallnate/quickjs-emscripten": "^0.23.0", + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "get-uri": "^6.0.1", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.6", + "pac-resolver": "^7.0.1", + "socks-proxy-agent": "^8.0.5" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-resolver": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.1.tgz", + "integrity": "sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==", + "license": "MIT", + "dependencies": { + "degenerator": "^5.0.0", + "netmask": "^2.0.2" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/package-json-from-dist": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", @@ -10805,7 +11203,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", - "dev": true, "dependencies": { "callsites": "^3.0.0" }, @@ -10817,7 +11214,6 @@ "version": "5.2.0", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", - "dev": true, "dependencies": { "@babel/code-frame": "^7.0.0", "error-ex": "^1.3.1", @@ -10996,6 +11392,12 @@ "resolved": "https://registry.npmjs.org/pause/-/pause-0.0.1.tgz", "integrity": "sha512-KG8UEiEVkR3wGEb4m5yZkVCzigAD+cVEJck2CzYZO37ZGJfctvVptVO192MwrtPhzONn6go8ylnOdMhKqi4nfg==" }, + "node_modules/pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", + "license": "MIT" + }, "node_modules/performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", @@ -11005,8 +11407,7 @@ "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", - "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", - "dev": true + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==" }, "node_modules/picomatch": { "version": "4.0.2", @@ -11287,6 +11688,15 @@ } ] }, + "node_modules/progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/promise-coalesce": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/promise-coalesce/-/promise-coalesce-1.5.0.tgz", @@ -11307,6 +11717,40 @@ "node": ">= 0.10" } }, + "node_modules/proxy-agent": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.5.0.tgz", + "integrity": "sha512-TmatMXdr2KlRiA2CyDu8GqR8EjahTG3aY3nXjdzFyoZbmB8hrBsTyMezhULIXKnC0jpfjlmiZ3+EaCzoInSu/A==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "http-proxy-agent": "^7.0.1", + "https-proxy-agent": "^7.0.6", + "lru-cache": "^7.14.1", + "pac-proxy-agent": "^7.1.0", + "proxy-from-env": "^1.1.0", + "socks-proxy-agent": "^8.0.5" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/lru-cache": { + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "license": "MIT" + }, "node_modules/psl": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/psl/-/psl-1.15.0.tgz", @@ -11323,7 +11767,6 @@ "version": "3.0.3", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.3.tgz", "integrity": "sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA==", - "dev": true, "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -11337,6 +11780,92 @@ "node": ">=6" } }, + "node_modules/puppeteer": { + "version": "24.40.0", + "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-24.40.0.tgz", + "integrity": "sha512-IxQbDq93XHVVLWHrAkFP7F7iHvb9o0mgfsSIMlhHb+JM+JjM1V4v4MNSQfcRWJopx9dsNOr9adYv0U5fm9BJBQ==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "@puppeteer/browsers": "2.13.0", + "chromium-bidi": "14.0.0", + "cosmiconfig": "^9.0.0", + "devtools-protocol": "0.0.1581282", + "puppeteer-core": "24.40.0", + "typed-query-selector": "^2.12.1" + }, + "bin": { + "puppeteer": "lib/cjs/puppeteer/node/cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/puppeteer-core": { + "version": "24.40.0", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-24.40.0.tgz", + "integrity": "sha512-MWL3XbUCfVgGR0gRsidzT6oKJT2QydPLhMITU6HoVWiiv4gkb6gJi3pcdAa8q4HwjBTbqISOWVP4aJiiyUJvag==", + "license": "Apache-2.0", + "dependencies": { + "@puppeteer/browsers": "2.13.0", + "chromium-bidi": "14.0.0", + "debug": "^4.4.3", + "devtools-protocol": "0.0.1581282", + "typed-query-selector": "^2.12.1", + "webdriver-bidi-protocol": "0.4.1", + "ws": "^8.19.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/puppeteer-core/node_modules/ws": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.20.0.tgz", + "integrity": "sha512-sAt8BhgNbzCtgGbt2OxmpuryO63ZoDk/sqaB/znQm94T4fCEsy/yV+7CdC1kJhOU9lboAEU7R3kquuycDoibVA==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/puppeteer/node_modules/cosmiconfig": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-9.0.1.tgz", + "integrity": "sha512-hr4ihw+DBqcvrsEDioRO31Z17x71pUYoNe/4h6Z0wB72p7MU7/9gH8Q3s12NFhHPfYBBOV3qyfUxmr/Yn3shnQ==", + "license": "MIT", + "dependencies": { + "env-paths": "^2.2.1", + "import-fresh": "^3.3.0", + "js-yaml": "^4.1.0", + "parse-json": "^5.2.0" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/d-fischer" + }, + "peerDependencies": { + "typescript": ">=4.9.5" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, "node_modules/pure-rand": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-7.0.1.tgz", @@ -11560,7 +12089,6 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -11599,7 +12127,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", - "dev": true, "engines": { "node": ">=4" } @@ -11962,6 +12489,16 @@ "node": ">=8" } }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "license": "MIT", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, "node_modules/socket.io": { "version": "4.8.3", "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-4.8.3.tgz", @@ -12039,6 +12576,34 @@ "node": ">= 0.6" } }, + "node_modules/socks": { + "version": "2.8.7", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.7.tgz", + "integrity": "sha512-HLpt+uLy/pxB+bum/9DzAgiKS8CX1EvbWxI4zlmgGCExImLdiad2iCwXT5Z4c9c3Eq8rP2318mPW2c+QbtjK8A==", + "license": "MIT", + "dependencies": { + "ip-address": "^10.0.1", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks-proxy-agent": { + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz", + "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "socks": "^2.8.3" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/sonic-boom": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-4.2.0.tgz", @@ -12156,6 +12721,17 @@ "node": ">=10.0.0" } }, + "node_modules/streamx": { + "version": "2.25.0", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.25.0.tgz", + "integrity": "sha512-0nQuG6jf1w+wddNEEXCF4nTg3LtufWINB5eFEN+5TNZW7KWJp6x87+JFL43vaAUPyCfH1wID+mNVyW6OHtFamg==", + "license": "MIT", + "dependencies": { + "events-universal": "^1.0.0", + "fast-fifo": "^1.3.2", + "text-decoder": "^1.1.0" + } + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -12378,6 +12954,41 @@ "url": "https://opencollective.com/webpack" } }, + "node_modules/tar-fs": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.1.2.tgz", + "integrity": "sha512-QGxxTxxyleAdyM3kpFs14ymbYmNFrfY+pHj7Z8FgtbZ7w2//VAgLMac7sT6nRpIHjppXO2AwwEOg0bPFVRcmXw==", + "license": "MIT", + "dependencies": { + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + }, + "optionalDependencies": { + "bare-fs": "^4.0.1", + "bare-path": "^3.0.0" + } + }, + "node_modules/tar-stream": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.8.tgz", + "integrity": "sha512-U6QpVRyCGHva435KoNWy9PRoi2IFYCgtEhq9nmrPPpbRacPs9IH4aJ3gbrFC8dPcXvdSZ4XXfXT5Fshbp2MtlQ==", + "license": "MIT", + "dependencies": { + "b4a": "^1.6.4", + "bare-fs": "^4.5.5", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, + "node_modules/teex": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/teex/-/teex-1.0.1.tgz", + "integrity": "sha512-eYE6iEI62Ni1H8oIa7KlDU6uQBtqr4Eajni3wX7rpfXD8ysFx8z0+dri+KWEPWpBsxXfxu58x/0jvTVT1ekOSg==", + "license": "MIT", + "dependencies": { + "streamx": "^2.12.5" + } + }, "node_modules/terser": { "version": "5.44.1", "resolved": "https://registry.npmjs.org/terser/-/terser-5.44.1.tgz", @@ -12570,6 +13181,15 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/text-decoder": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.7.tgz", + "integrity": "sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ==", + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.6.4" + } + }, "node_modules/thread-stream": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-3.1.0.tgz", @@ -12907,6 +13527,12 @@ "node": ">= 0.6" } }, + "node_modules/typed-query-selector": { + "version": "2.12.1", + "resolved": "https://registry.npmjs.org/typed-query-selector/-/typed-query-selector-2.12.1.tgz", + "integrity": "sha512-uzR+FzI8qrUEIu96oaeBJmd9E7CFEiQ3goA5qCVgc4s5llSubcfGHq9yUstZx/k4s9dXHVKsE35YWoFyvEqEHA==", + "license": "MIT" + }, "node_modules/typedarray": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", @@ -12916,7 +13542,7 @@ "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "dev": true, + "devOptional": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -13204,6 +13830,12 @@ "node": ">= 8" } }, + "node_modules/webdriver-bidi-protocol": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.4.1.tgz", + "integrity": "sha512-ARrjNjtWRRs2w4Tk7nqrf2gBI0QXWuOmMCx2hU+1jUt6d00MjMxURrhxhGbrsoiZKJrhTSTzbIrc554iKI10qw==", + "license": "Apache-2.0" + }, "node_modules/webpack": { "version": "5.104.1", "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.104.1.tgz", @@ -13548,7 +14180,6 @@ "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "dev": true, "engines": { "node": ">=10" } @@ -13563,7 +14194,6 @@ "version": "17.7.2", "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", - "dev": true, "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", @@ -13581,11 +14211,20 @@ "version": "21.1.1", "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", - "dev": true, "engines": { "node": ">=12" } }, + "node_modules/yauzl": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", + "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", + "license": "MIT", + "dependencies": { + "buffer-crc32": "~0.2.3", + "fd-slicer": "~1.1.0" + } + }, "node_modules/yn": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", diff --git a/package.json b/package.json index 6a4d7b7..305e433 100644 --- a/package.json +++ b/package.json @@ -52,6 +52,7 @@ "pino": "^10.1.0", "pino-http": "^11.0.0", "prisma": "^5.22.0", + "puppeteer": "^24.40.0", "reflect-metadata": "^0.2.2", "rxjs": "^7.8.1", "zod": "^4.3.5" diff --git a/src/modules/content-generation/content-generation.module.ts b/src/modules/content-generation/content-generation.module.ts index 48a7e00..a3865d7 100644 --- a/src/modules/content-generation/content-generation.module.ts +++ b/src/modules/content-generation/content-generation.module.ts @@ -15,6 +15,7 @@ import { SeoModule } from '../seo/seo.module'; import { NeuroMarketingModule } from '../neuro-marketing/neuro-marketing.module'; import { GeminiModule } from '../gemini/gemini.module'; import { VisualGenerationModule } from '../visual-generation/visual-generation.module'; +import { WebScraperService } from '../trends/services/web-scraper.service'; @Module({ @@ -28,6 +29,7 @@ import { VisualGenerationModule } from '../visual-generation/visual-generation.m HashtagService, BrandVoiceService, VariationService, + WebScraperService, ], controllers: [ContentGenerationController], exports: [ContentGenerationService], diff --git a/src/modules/content-generation/content-generation.service.ts b/src/modules/content-generation/content-generation.service.ts index 9106e9d..b857968 100644 --- a/src/modules/content-generation/content-generation.service.ts +++ b/src/modules/content-generation/content-generation.service.ts @@ -14,11 +14,13 @@ import { SeoService, FullSeoAnalysis as SeoDTO } from '../seo/seo.service'; import { NeuroMarketingService } from '../neuro-marketing/neuro-marketing.service'; import { StorageService } from '../visual-generation/services/storage.service'; import { VisualGenerationService } from '../visual-generation/visual-generation.service'; +import { WebScraperService, ScrapedContent } from '../trends/services/web-scraper.service'; import { ContentType as PrismaContentType, ContentStatus as PrismaContentStatus, MasterContentType as PrismaMasterContentType } from '@prisma/client'; export interface ContentGenerationRequest { topic: string; + sourceUrl?: string; niche?: string; platforms: Platform[]; includeResearch?: boolean; @@ -76,6 +78,7 @@ export class ContentGenerationService { private readonly neuroService: NeuroMarketingService, private readonly storageService: StorageService, private readonly visualService: VisualGenerationService, + private readonly webScraperService: WebScraperService, ) { } @@ -87,6 +90,7 @@ export class ContentGenerationService { async generateContent(request: ContentGenerationRequest): Promise { const { topic, + sourceUrl, niche, platforms, includeResearch = true, @@ -99,6 +103,26 @@ export class ContentGenerationService { console.log(`[ContentGenerationService] Starting generation for topic: ${topic}, platforms: ${platforms.join(', ')}`); + // ========== STEP 1: Scrape source article if URL provided ========== + let scrapedSource: ScrapedContent | null = null; + if (sourceUrl) { + this.logger.log(`Scraping source article: ${sourceUrl}`); + try { + scrapedSource = await this.webScraperService.scrapeUrl(sourceUrl, { + extractImages: true, + extractLinks: true, + timeout: 15000, + }, topic); + if (scrapedSource) { + this.logger.log(`Scraped source: ${scrapedSource.wordCount} words, ${scrapedSource.images.length} images, ${scrapedSource.videoLinks.length} videos`); + } else { + this.logger.warn(`Failed to scrape source URL: ${sourceUrl}`); + } + } catch (err) { + this.logger.warn(`Source scraping error: ${err.message}`); + } + } + // Analyze niche if provided let nicheAnalysis: NicheAnalysis | undefined; if (niche) { @@ -116,6 +140,23 @@ export class ContentGenerationService { }); } + // ========== Build enriched context from scraped source ========== + let sourceContext = ''; + if (scrapedSource) { + const articleText = scrapedSource.content.substring(0, 3000); + const videoInfo = scrapedSource.videoLinks.length > 0 + ? `\nVİDEO LİNKLERİ: ${scrapedSource.videoLinks.join(', ')}` + : ''; + const importantLinks = scrapedSource.links + .filter(l => l.isExternal && !l.href.includes('facebook') && !l.href.includes('twitter')) + .slice(0, 5) + .map(l => `${l.text}: ${l.href}`) + .join('\n'); + const linkInfo = importantLinks ? `\nÖNEMLİ LİNKLER:\n${importantLinks}` : ''; + + sourceContext = `\n\n📰 KAYNAK MAKALE İÇERİĞİ (ZORUNLU REFERANS):\n${articleText}${videoInfo}${linkInfo}\n\n⚠️ ÖNEMLİ: Yukarıdaki kaynak makaledeki TÜM özneleri (kişi, ürün, oyun adları, tarihler, fiyatlar, markalar) habere dahil et. Hiçbir önemli bilgiyi atlama. Video linkleri ve önemli dış linkler varsa bunları da içerikte paylaş.`; + } + // Generate content for each platform using AI const platformContent: GeneratedContent[] = []; for (const platform of platforms) { @@ -127,11 +168,13 @@ export class ContentGenerationService { const sanitizedSummary = this.sanitizeResearchSummary( research?.summary || `Everything you need to know about ${topic}` ); + // Append scraped source context to give AI the full article details + const enrichedSummary = sanitizedSummary + sourceContext; // Normalize platform to lowercase for consistency const normalizedPlatform = platform.toLowerCase(); const aiContent = await this.platformService.generateAIContent( topic, - sanitizedSummary, + enrichedSummary, normalizedPlatform as any, // Cast to any/Platform to resolve type mismatch if Platform is strict union 'standard', 'tr', @@ -145,6 +188,9 @@ export class ContentGenerationService { this.logger.warn(`AI Content is empty for ${platform}`); } + // Use scraped image from source if available + const sourceImageUrl = scrapedSource?.images?.[0]?.src || undefined; + const config = this.platformService.getPlatformConfig(platform); let content: GeneratedContent = { platform, @@ -163,10 +209,19 @@ export class ContentGenerationService { content.content = voiceApplied.branded; } - // Add hashtags if requested + // Add hashtags using AI (based on actual generated content) if (includeHashtags) { - const hashtagSet = this.hashtagService.generateHashtags(topic, platform); - content.hashtags = hashtagSet.hashtags.map((h) => h.hashtag); + try { + content.hashtags = await this.platformService.generateAIHashtags( + content.content, + topic, + platform as any, + 'tr', + ); + } catch (hashErr) { + this.logger.warn(`AI hashtag generation failed, skipping: ${hashErr.message}`); + content.hashtags = []; + } } // Generate image for visual platforms @@ -180,11 +235,31 @@ export class ContentGenerationService { platform: platformKey, enhancePrompt: true, }); - content.imageUrl = image.url; - this.logger.log(`Image generated for ${platform}: ${image.url}`); + + // Check if image is a real image or just a placeholder + const isPlaceholder = image.url?.includes('placehold.co') || image.url?.includes('placeholder'); + if (!isPlaceholder) { + content.imageUrl = image.url; + this.logger.log(`Image generated for ${platform}: ${image.url}`); + } else if (sourceImageUrl) { + // Use scraped source image instead of placeholder + content.imageUrl = sourceImageUrl; + this.logger.log(`Using scraped source image instead of placeholder: ${sourceImageUrl}`); + } else { + content.imageUrl = image.url; + this.logger.log(`Image generated for ${platform}: ${image.url} (placeholder, no source image available)`); + } } catch (imgError) { this.logger.warn(`Image generation failed for ${platform}, continuing without image`, imgError); + // Fallback to scraped source image + if (sourceImageUrl) { + content.imageUrl = sourceImageUrl; + this.logger.log(`Using scraped source image as fallback: ${sourceImageUrl}`); + } } + } else if (sourceImageUrl && !content.imageUrl) { + // For non-visual platforms, still attach source image if available + content.imageUrl = sourceImageUrl; } platformContent.push(content); @@ -358,7 +433,7 @@ export class ContentGenerationService { userId: effectiveUserId!, masterContentId: masterContent.id, type: contentType, - title: `${bundle.topic} - ${platformContent.platform}`, + title: this.sanitizeResearchSummary(`${bundle.topic}`) + ` - ${platformContent.platform}`, body: platformContent.content, hashtags: platformContent.hashtags, status: PrismaContentStatus.DRAFT, @@ -548,6 +623,8 @@ KURALLAR: 6. Karakter limitini koru 7. Platformun tonuna uygun yaz 8. SADECE yayınlanacak metni yaz +9. Hiçbir haber sitesi, kaynak, ajans veya web sitesi adı kullanma +10. "...göre", "...haberlere göre", "...kaynağına göre" gibi atıf ifadeleri ASLA kullanma SADECE yeniden yazılmış metni döndür, açıklama ekleme.`; @@ -589,25 +666,43 @@ SADECE yeniden yazılmış metni döndür, açıklama ekleme.`; sanitized = sanitized.replace(/https?:\/\/[^\s]+/gi, ''); sanitized = sanitized.replace(/www\.[^\s]+/gi, ''); - // Remove common Turkish attribution phrases + // Remove common attribution phrases (Turkish and English) const attributionPatterns = [ /\b\w+\.com(\.tr)?\b/gi, /\b\w+\.org(\.tr)?\b/gi, /\b\w+\.net(\.tr)?\b/gi, /\bkaynağına göre\b/gi, /\b'e göre\b/gi, + /\b'(i|a|e|u|ü|\u0131)n(da|de) (yayınlanan|yer alan|çıkan)\b/gi, + /\b(da|de) (çıkan|yayınlanan|yer alan) (haberlere|habere|bilgilere) göre\b/gi, + /\bhaberlere göre\b/gi, + /\braporuna göre\b/gi, + /\bsitesinde yer alan\b/gi, + /\baçıklamasına göre\b/gi, + /\byazısına göre\b/gi, + /\bhaberine göre\b/gi, + /\btarafından yapılan\b/gi, /\baccording to [^,.]+/gi, + /\breported by [^,.]+/gi, + /\bas reported in [^,.]+/gi, /\bsource:\s*[^,.]+/gi, /\breferans:\s*[^,.]+/gi, /\bkaynak:\s*[^,.]+/gi, ]; - // Common Turkish tech/news source brands to strip + // Comprehensive list of Turkish tech/news source brands to strip const sourceNames = [ - 'donanımhaber', 'technopat', 'webtekno', 'shiftdelete', + 'tamindir', 'donanımhaber', 'technopat', 'webtekno', 'shiftdelete', 'chip online', 'log.com', 'mediatrend', 'bbc', 'cnn', 'reuters', 'anadolu ajansı', 'hürriyet', 'milliyet', 'sabah', 'forbes', 'bloomberg', 'techcrunch', + 'the verge', 'engadget', 'ars technica', 'wired', + 'mashable', 'gizmodo', 'tom\'s hardware', 'tom\'s guide', + 'ntv', 'habertürk', 'sozcu', 'sözcü', 'cumhuriyet', 'star', + 'posta', 'aksam', 'yeni safak', 'yeni şafak', 'takvim', + 'mynet', 'ensonhaber', 'haber7', 'internethaber', + 'ad hoc news', 'finanzen.net', 'der aktionär', 'aktionar', + 'business insider', 'cnbc', 'financial times', 'wall street journal', ]; for (const pattern of attributionPatterns) { @@ -615,12 +710,15 @@ SADECE yeniden yazılmış metni döndür, açıklama ekleme.`; } for (const source of sourceNames) { - const regex = new RegExp(`\\b${source}\\b`, 'gi'); + const regex = new RegExp(`\\b${source.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`, 'gi'); sanitized = sanitized.replace(regex, ''); } - // Clean up multiple spaces and trailing commas - sanitized = sanitized.replace(/\s{2,}/g, ' ').replace(/,\s*,/g, ',').trim(); + // Also remove "- site_name" patterns from titles (e.g. "Great News - Tamindir") + sanitized = sanitized.replace(/\s*-\s*$/gm, ''); + + // Clean up multiple spaces, trailing commas, and orphaned punctuation + sanitized = sanitized.replace(/\s{2,}/g, ' ').replace(/,\s*,/g, ',').replace(/\s+([.,;:!?])/g, '$1').trim(); return sanitized; } diff --git a/src/modules/content-generation/services/platform-generator.service.ts b/src/modules/content-generation/services/platform-generator.service.ts index 0e942c9..87c0dc3 100644 --- a/src/modules/content-generation/services/platform-generator.service.ts +++ b/src/modules/content-generation/services/platform-generator.service.ts @@ -502,21 +502,39 @@ TON: ${config.tone}${styleInstruction}${ctaInstruction} Bu platform için özgün, ilgi çekici ve viral potansiyeli yüksek bir içerik oluştur. -KURALLAR: +📈 SEO OPTİMİZASYONU (ZORUNLU): +- Bu konuyu Google'da, YouTube'da veya sosyal medyada arayan biri hangi kelimeleri kullanır? O kelimeleri belirle ve içeriğe yerleştir. +- İLK 2 CÜMLEDE arama hacmi en yüksek anahtar kelimeleri MUTLAKA kullan. +- Hook/giriş cümlesi birincil anahtar kelimeyi içersin. +- Anahtar kelimeleri doğal bir akış içinde kullan, zoraki tekrar yapma. +- Konu ile ilgili en çok aranan terimleri, teknik terimleri ve marka/ürün adlarını (haberin konusu olan markaları — kaynak değil) ön plana çıkar. + +KRİTİK KURALLAR: 1. Karakter limitine uy 2. Platformun tonuna uygun yaz 3. Hook (dikkat çeken giriş) ile başla 4. CTA ile bitir (yukarıdaki CTA talimatına göre) 5. Emoji kullan ama aşırıya kaçma 6. ${language === 'tr' ? 'Türkçe' : 'İngilizce'} yaz -7. ASLA resim URL'i, medya linki veya [görsel] gibi yer tutucular ekleme -8. Görsel betimlemeleri metnin içine YAZMA -9. İçerik %100 özgün olmalı - asla kaynak kopyası yapma -10. Kaynak linklerini, URL'leri veya atıfları ASLA ekleme -11. Mevcut içeriklerden alıntı yapma, tamamen yeni ve orijinal yaz -12. Bilgiyi kendi cümlelerinle ifade et, paraphrase bile yapma -13. Araştırma kaynaklarının isimlerini (web siteleri, haber siteleri, markalar, gazeteler) ASLA metinde kullanma veya referans verme -14. "...göre", "...kaynağına göre", "according to" gibi atıf ifadeleri ASLA kullanma +7. İçerik %100 özgün olmalı - asla kaynak kopyası yapma +8. Bilgiyi kendi cümlelerinle ifade et, paraphrase bile yapma + +⚠️ YAYIN HAZIR İÇERİK (ÇOK ÖNEMLİ): +- İçerik doğrudan kopyala-yapıştır ile yayınlanabilir olmalı +- "[Buraya Link]", "[Link Ekle]", "[URL]", "[Görsel]", "[Video]" gibi YER TUTUCU İFADELER ASLA kullanma +- Resim URL'i, medya linki veya placeholder ASLA ekleme +- Görsel betimlemeleri metnin içine YAZMA +- "Linke tıklayın", "Bio'daki linke gidin" gibi CTA'lar kullanabilirsin ama asla köşeli parantez içinde placeholder koyma +- Eğer bir link veya URL bilmiyorsan, o kısmı tamamen atla — placeholder bırakma +- İçerikte doldurulması gereken boşluk OLMAMALI + +⛔ KAYNAK YASAĞI (EN ÖNEMLİ KURAL): +- Hiçbir haber sitesi, web sitesi, gazete, ajans, blog veya medya kuruluşu adını ASLA yazma +- "Tamindir", "Webtekno", "DonanımHaber", "ShiftDelete", "TechCrunch", "BBC", "CNN", "Reuters", "Forbes", "Bloomberg" gibi site/kaynak adlarını ASLA kullanma +- "...haberlere göre", "...raporuna göre", "...kaynağına göre", "...sitesinde yer alan", "...çıkan haberlere göre", "according to", "...tarafından yapılan" gibi ATıF İFADELERİ ASLA kullanma +- Haberin nereden alındığını BELİRTME, doğrudan bilgiyi kendi cümlelerinle anlat +- İçerikte kaynak gösterme, referans verme veya atıf yapma YOK +- Bilgi/veri paylaşırken kaynağı belirtmeden doğrudan bilgiyi ver SADECE yayınlanacak metni yaz, açıklama veya başlık ekleme.`; @@ -532,6 +550,107 @@ SADECE yayınlanacak metni yaz, açıklama veya başlık ekleme.`; } } + /** + * Generate relevant, SEO-optimized hashtags using AI + * Replaces the old mock-based hashtag generation + */ + async generateAIHashtags( + content: string, + topic: string, + platform: Platform, + language: string = 'tr', + ): Promise { + const config = this.platforms[platform]; + if (!config || config.maxHashtags === 0) return []; + + if (!this.gemini.isAvailable()) { + this.logger.warn('Gemini not available for hashtag generation, using fallback'); + return this.generateFallbackHashtags(topic, config.maxHashtags); + } + + const maxCount = Math.min(config.maxHashtags, platform === 'instagram' ? 15 : config.maxHashtags); + + const prompt = `Sen bir sosyal medya SEO uzmanısın. Aşağıdaki içerik ve konu için ${platform.toUpperCase()} platformunda kullanılacak EN UYGUN ${maxCount} hashtag üret. + +KONU: ${topic} +İÇERİK: +${content.substring(0, 500)} + +HASHTAG KURALLARI: +1. Her hashtag DOĞRUDAN içerikle ilgili olmalı — genel veya ilişkisiz hashtag OLMASIN +2. Arama hacmi yüksek, gerçek kullanıcıların arayacağı kelimelerden oluştur +3. Konunun ana terimleri, teknik terimleri, marka/ürün adları (haberin konusu olanlar) ve sektör terimleri olsun +4. "tips", "howto", "life", "community", "motivation", "goals" gibi genel son ekler KULLANMA +5. ${language === 'tr' ? 'Türkçe ve İngilizce karışık olabilir, hangisi daha çok aranıyorsa onu seç' : 'Use English hashtags'} +6. Hashtag'ı # ile başlat +7. Tek kelime veya kısa bileşik kelimeler kullan (boşluk yok) +8. Haber kaynağı olan sitelerin adlarını (Webtekno, Tamindir, DonanımHaber, ShiftDelete, TechCrunch, BBC, CNN vb.) ASLA hashtag olarak kullanma — bunlar bizim kaynağımız, içeriğimiz değil + +SADECE hashtag listesini döndür, her satırda bir hashtag. Başka açıklama ekleme. +ÖRNEK FORMAT: +#hashtag1 +#hashtag2 +#hashtag3`; + + try { + const response = await this.gemini.generateText(prompt, { + temperature: 0.4, + maxTokens: 200, + }); + + // Banned source names that should never appear as hashtags + const bannedSources = [ + 'tamindir', 'webtekno', 'donanımhaber', 'donanımhaber', 'shiftdelete', + 'technopat', 'chipsonline', 'chiponline', 'mediatrend', 'hürriyet', + 'milliyet', 'sabah', 'ntv', 'habertürk', 'sözcü', 'sozcu', + 'cumhuriyet', 'posta', 'aksam', 'takvim', 'mynet', 'ensonhaber', + 'haber7', 'internethaber', 'bbc', 'cnn', 'reuters', 'forbes', + 'bloomberg', 'techcrunch', 'theverge', 'engadget', 'wired', + 'gizmodo', 'mashable', 'businessinsider', 'cnbc', 'adhoçnews', + 'finanzennet', 'deraktionär', 'aktionar', + ]; + + const hashtags = response.text + .split('\n') + .map(line => line.trim()) + .filter(line => line.startsWith('#') && line.length > 1) + .map(tag => tag.replace(/\s+/g, '')) + .filter(tag => { + const cleanTag = tag.replace('#', '').toLowerCase(); + return !bannedSources.some(source => + cleanTag === source || cleanTag === source.replace(/\s/g, '') + ); + }) + .slice(0, maxCount); + + if (hashtags.length === 0) { + this.logger.warn('AI returned no valid hashtags, using fallback'); + return this.generateFallbackHashtags(topic, maxCount); + } + + this.logger.log(`AI generated ${hashtags.length} hashtags for ${platform}: ${hashtags.join(', ')}`); + return hashtags; + } catch (error) { + this.logger.error(`AI hashtag generation failed: ${error.message}`); + return this.generateFallbackHashtags(topic, maxCount); + } + } + + /** + * Fallback hashtag generation when AI is unavailable + * Extracts meaningful words from the topic instead of appending generic suffixes + */ + private generateFallbackHashtags(topic: string, maxCount: number): string[] { + const stopWords = new Set(['ve', 'ile', 'bir', 'bu', 'için', 'da', 'de', 'the', 'a', 'an', 'and', 'or', 'for', 'in', 'on', 'is', 'are', 'was', 'of', 'to']); + return topic + .toLowerCase() + .replace(/[^a-zçğıöşü\w\s]/gi, '') + .split(/\s+/) + .filter(w => w.length > 3 && !stopWords.has(w)) + .slice(0, maxCount) + .map(w => `#${w}`); + } + private generateTemplateContent( topic: string, mainMessage: string, diff --git a/src/modules/seo/services/content-optimization.service.ts b/src/modules/seo/services/content-optimization.service.ts index 085f26a..19eae53 100644 --- a/src/modules/seo/services/content-optimization.service.ts +++ b/src/modules/seo/services/content-optimization.service.ts @@ -270,7 +270,7 @@ export class ContentOptimizationService { if (!keyword) return 50; const words = content.split(/\s+/).length; - const kwCount = (content.toLowerCase().match(new RegExp(keyword.toLowerCase(), 'g')) || []).length; + const kwCount = content.toLowerCase().split(keyword.toLowerCase()).length - 1; const density = (kwCount / words) * 100; if (density >= this.optimalParams.keywordDensity.min && diff --git a/src/modules/trends/services/web-scraper.service.ts b/src/modules/trends/services/web-scraper.service.ts index 3570d81..d0de25d 100644 --- a/src/modules/trends/services/web-scraper.service.ts +++ b/src/modules/trends/services/web-scraper.service.ts @@ -2,6 +2,7 @@ // Path: src/modules/trends/services/web-scraper.service.ts import { Injectable, Logger } from '@nestjs/common'; +import * as puppeteer from 'puppeteer'; export interface ScrapedContent { url: string; @@ -12,6 +13,7 @@ export interface ScrapedContent { headings: { level: number; text: string }[]; links: { text: string; href: string; isExternal: boolean }[]; images: { src: string; alt: string }[]; + videoLinks: string[]; metadata: { author?: string; publishDate?: string; @@ -63,37 +65,362 @@ export interface ScraperOptions { export class WebScraperService { private readonly logger = new Logger(WebScraperService.name); private readonly contentCache = new Map(); - private readonly defaultUserAgent = 'ContentHunter/1.0 (Research Bot)'; + private readonly defaultUserAgent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'; /** - * Scrape content from a web page + * Scrape content from a web page. + * Automatically resolves Google News redirect URLs. + * @param url The URL to scrape + * @param options Scraper options + * @param articleTitle The article title (used for Google News URL resolution via search) */ - async scrapeUrl(url: string, options?: ScraperOptions): Promise { + async scrapeUrl(url: string, options?: ScraperOptions, articleTitle?: string): Promise { // Validate URL if (!this.isValidUrl(url)) { this.logger.warn(`Invalid URL: ${url}`); return null; } + // Resolve Google News redirect URLs to actual article URLs + let resolvedUrl = url; + if (url.includes('news.google.com') || url.includes('google.com/rss')) { + this.logger.log(`Detected Google News URL: ${url}`); + + // Strategy 1: Use Puppeteer headless browser to follow JS redirects (most reliable) + const puppeteerResult = await this.resolveGoogleNewsWithPuppeteer(url); + if (puppeteerResult) { + resolvedUrl = puppeteerResult; + this.logger.log(`Puppeteer resolved Google News URL to: ${resolvedUrl}`); + } else { + // Strategy 2: Fall back to DuckDuckGo title search + this.logger.warn('Puppeteer resolution failed, trying DuckDuckGo title search...'); + if (articleTitle) { + const searchResult = await this.findArticleByTitle(articleTitle); + if (searchResult) { + resolvedUrl = searchResult; + this.logger.log(`DuckDuckGo found article URL: ${resolvedUrl}`); + } else { + this.logger.warn('Both Puppeteer and DuckDuckGo failed. Cannot resolve Google News URL.'); + return null; + } + } else { + this.logger.warn('No article title provided for fallback search. Cannot resolve Google News URL.'); + return null; + } + } + } + // Check cache - const cached = this.contentCache.get(url); + const cached = this.contentCache.get(resolvedUrl); if (cached && this.isCacheValid(cached)) { return cached; } try { - const response = await this.fetchPage(url, options); - if (!response) return null; + this.logger.log(`Scraping URL: ${resolvedUrl}`); + const response = await this.fetchPage(resolvedUrl, options); + if (!response) { + this.logger.warn(`fetchPage returned null for ${resolvedUrl}`); + return null; + } - const content = this.parseHtml(response.html, url, options); + this.logger.log(`Fetched HTML: ${response.html.length} chars`); + const content = this.parseHtml(response.html, resolvedUrl, options); content.html = options?.includeHtml ? response.html : ''; // Cache the result - this.contentCache.set(url, content); + this.contentCache.set(resolvedUrl, content); + this.logger.log(`Scraped successfully: ${content.title}, ${content.images.length} images, ${content.videoLinks.length} videos, ${content.wordCount} words`); return content; } catch (error) { - this.logger.error(`Failed to scrape ${url}:`, error); + this.logger.error(`Failed to scrape ${resolvedUrl}:`, error); + return null; + } + } + + /** + * Resolve a Google News redirect URL to the actual article URL using Puppeteer. + * Google News uses JavaScript-only redirects that cannot be followed via HTTP. + * Puppeteer launches a headless browser to follow the redirect. + */ + private async resolveGoogleNewsWithPuppeteer(googleNewsUrl: string): Promise { + let browser: puppeteer.Browser | null = null; + try { + this.logger.log(`Launching Puppeteer to resolve Google News URL...`); + + browser = await puppeteer.launch({ + headless: true, + args: [ + '--no-sandbox', + '--disable-setuid-sandbox', + '--disable-dev-shm-usage', + '--disable-gpu', + '--disable-extensions', + '--disable-background-networking', + '--window-size=1280,720', + ], + timeout: 20000, + }); + + const page = await browser.newPage(); + + // Set a realistic user agent + await page.setUserAgent(this.defaultUserAgent); + + // Block unnecessary resources to speed up loading + await page.setRequestInterception(true); + page.on('request', (request) => { + const resourceType = request.resourceType(); + if (['image', 'stylesheet', 'font', 'media'].includes(resourceType)) { + request.abort(); + } else { + request.continue(); + } + }); + + // Navigate to the Google News URL + await page.goto(googleNewsUrl, { + waitUntil: 'networkidle2', + timeout: 15000, + }); + + // Wait for the redirect to complete — check periodically if URL changed + let finalUrl = page.url(); + const startTime = Date.now(); + const maxWait = 10000; // 10 seconds max + + while (finalUrl.includes('news.google.com') && (Date.now() - startTime) < maxWait) { + await new Promise(resolve => setTimeout(resolve, 500)); + finalUrl = page.url(); + } + + await browser.close(); + browser = null; + + // Check if we successfully left Google News + if (!finalUrl.includes('news.google.com') && !finalUrl.includes('consent.google.com')) { + this.logger.log(`Puppeteer resolved to: ${finalUrl}`); + return finalUrl; + } else { + this.logger.warn(`Puppeteer could not resolve - still on Google domain: ${finalUrl}`); + return null; + } + } catch (error) { + this.logger.warn(`Puppeteer resolution failed: ${error.message}`); + return null; + } finally { + if (browser) { + try { await browser.close(); } catch (e) { /* ignore */ } + } + } + } + + /** + * Find actual article URL by searching for the article title. + * Extracts the source name from the title (e.g., "Title - Webtekno" → searches "site:webtekno.com Title") + * Uses DuckDuckGo HTML search (more bot-friendly than Google) + */ + private async findArticleByTitle(title: string): Promise { + try { + // Extract source name from title (usually at the end after " - ") + const parts = title.split(/\s+-\s+/); + const sourceName = parts.length > 1 ? parts[parts.length - 1].trim() : ''; + const cleanTitle = parts.length > 1 ? parts.slice(0, -1).join(' - ').trim() : title; + + // Remove brackets and special chars from title for better search + const searchableTitle = cleanTitle + .replace(/\[.*?\]/g, '') + .replace(/[^\w\s\u00C0-\u024F\u0100-\u017F\u011E-\u011F\u0130-\u0131\u015E-\u015F\u00D6\u00F6\u00DC\u00FC\u00C7\u00E7]/g, ' ') + .replace(/\s+/g, ' ') + .trim(); + + // Build search query — prefer site: filter if we know the source + const sourceDomain = sourceName.toLowerCase().replace(/\s+/g, ''); + const siteFilter = sourceName ? `${sourceDomain}.com ` : ''; + const searchQuery = `${siteFilter}${searchableTitle}`; + + this.logger.log(`Searching DuckDuckGo for article: ${searchQuery}`); + + // Use DuckDuckGo HTML search (more bot-friendly) + const searchUrl = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(searchQuery)}`; + const response = await fetch(searchUrl, { + headers: { + 'User-Agent': this.defaultUserAgent, + 'Accept': 'text/html,application/xhtml+xml', + 'Accept-Language': 'tr-TR,tr;q=0.9,en-US;q=0.8,en;q=0.7', + }, + }); + + if (!response.ok) { + this.logger.warn(`DuckDuckGo search returned ${response.status}`); + return null; + } + + const html = await response.text(); + + // DuckDuckGo HTML results contain article URLs in uddg= parameters + const ddgPattern = /uddg=(https?[^&"]+)/g; + const foundUrls: string[] = []; + const seen = new Set(); + let match; + while ((match = ddgPattern.exec(html)) !== null) { + const foundUrl = decodeURIComponent(match[1]); + if (!seen.has(foundUrl) && !foundUrl.includes('duckduckgo.com') && !foundUrl.includes('google.com')) { + seen.add(foundUrl); + foundUrls.push(foundUrl); + } + } + + if (foundUrls.length > 0) { + // Prefer URLs matching the source name + if (sourceName) { + const sourceUrl = foundUrls.find(u => u.toLowerCase().includes(sourceDomain)); + if (sourceUrl) { + this.logger.log(`Found matching source URL via DuckDuckGo: ${sourceUrl}`); + return sourceUrl; + } + } + this.logger.log(`Using first DuckDuckGo result: ${foundUrls[0]}`); + return foundUrls[0]; + } + + this.logger.warn('No search results found for article title'); + return null; + } catch (error) { + this.logger.warn(`Article search failed: ${error.message}`); + return null; + } + } + + /** + * Resolve redirect URLs (especially Google News) to the final destination URL. + * Google News RSS URLs encode the actual article URL in a base64 segment in the path. + */ + private async resolveRedirectUrl(url: string): Promise { + // Strategy 1: Decode Google News base64-encoded URL from the path + try { + const decoded = this.decodeGoogleNewsUrl(url); + if (decoded) { + this.logger.log(`Decoded Google News URL: ${decoded}`); + return decoded; + } + } catch (e) { + this.logger.warn(`Base64 decode failed: ${e.message}`); + } + + // Strategy 2: Follow HTTP redirects + try { + // First try with redirect: 'manual' to get Location header + const headResponse = await fetch(url, { + method: 'HEAD', + headers: { + 'User-Agent': this.defaultUserAgent, + 'Accept': 'text/html', + }, + redirect: 'manual', + }); + + const locationHeader = headResponse.headers.get('location'); + if (locationHeader && !locationHeader.includes('news.google.com')) { + this.logger.log(`Redirect Location header: ${locationHeader}`); + return locationHeader; + } + + // Try full GET with redirect follow + const getResponse = await fetch(url, { + method: 'GET', + headers: { + 'User-Agent': this.defaultUserAgent, + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + }, + redirect: 'follow', + }); + + if (getResponse.url && !getResponse.url.includes('news.google.com')) { + return getResponse.url; + } + + // Strategy 3: Parse HTML for article link / meta refresh / canonical + const html = await getResponse.text(); + + // Check data-redirect attribute + const dataRedirect = html.match(/data-(?:redirect|href|url)=["'](https?:\/\/(?!news\.google\.com)[^"']+)["']/i); + if (dataRedirect) return dataRedirect[1]; + + // Meta refresh + const metaRefresh = html.match(/]*http-equiv=["']refresh["'][^>]*content=["']\d+;\s*url=([^"']+)["']/i); + if (metaRefresh) return metaRefresh[1]; + + // Canonical + const canonical = html.match(/]*rel=["']canonical["'][^>]*href=["']([^"']+)["']/i); + if (canonical && !canonical[1].includes('news.google.com')) return canonical[1]; + + // Any external link that looks like an article + const externalLink = html.match(/href=["'](https?:\/\/(?!(?:news|www)\.google\.com)[^"']+)["']/i); + if (externalLink) return externalLink[1]; + + this.logger.warn(`Could not resolve Google News URL, returning original`); + return getResponse.url; + } catch (error) { + this.logger.warn(`Failed to resolve redirect for ${url}: ${error.message}`); + return null; + } + } + + /** + * Decode actual article URL from Google News RSS URL. + * Google News encodes URLs in the path segment as base64. + * Format: https://news.google.com/rss/articles/CBMi{base64payload} + */ + private decodeGoogleNewsUrl(googleUrl: string): string | null { + try { + // Extract the base64 part from the URL path + const urlObj = new URL(googleUrl); + const pathParts = urlObj.pathname.split('/'); + // Find the article ID part (after /articles/) + const articlesIndex = pathParts.indexOf('articles'); + if (articlesIndex === -1 || articlesIndex + 1 >= pathParts.length) { + return null; + } + + let articleId = pathParts[articlesIndex + 1]; + // Remove query params if they got attached + if (articleId.includes('?')) { + articleId = articleId.split('?')[0]; + } + + // The article ID starts with "CBMi" prefix, try to decode the base64 + // Make base64 URL-safe: replace - with + and _ with / + let base64 = articleId + .replace(/-/g, '+') + .replace(/_/g, '/'); + + // Add padding if needed + while (base64.length % 4 !== 0) { + base64 += '='; + } + + // Decode base64 + const decoded = Buffer.from(base64, 'base64').toString('utf-8'); + + // Extract URLs from the decoded string using regex + const urlMatches = decoded.match(/https?:\/\/[^\s"'<>\x00-\x1F]+/g); + if (urlMatches && urlMatches.length > 0) { + // Filter out Google URLs + const nonGoogleUrl = urlMatches.find(u => !u.includes('google.com')); + if (nonGoogleUrl) { + // Clean up any trailing garbage characters + const cleanUrl = nonGoogleUrl.replace(/[\x00-\x1F\x7F-\x9F]+.*$/, ''); + this.logger.log(`Decoded article URL from base64: ${cleanUrl}`); + return cleanUrl; + } + return urlMatches[0]; + } + + return null; + } catch (error) { + this.logger.warn(`Failed to decode Google News URL: ${error.message}`); return null; } } @@ -129,64 +456,57 @@ export class WebScraperService { } /** - * Fetch page content (simulated) + * Fetch page content using real HTTP fetch */ private async fetchPage(url: string, options?: ScraperOptions): Promise<{ html: string } | null> { - // In production, use: - // 1. node-fetch or axios for simple pages - // 2. Puppeteer/Playwright for JavaScript-rendered pages - // 3. Cheerio for HTML parsing + const timeout = options?.timeout || 15000; + const userAgent = options?.userAgent || this.defaultUserAgent; - // Simulated HTML for demonstration - const mockHtml = ` - - - - Sample Article: Content Creation Strategies - - - - - - - - -
-

10 Content Creation Strategies for 2024

-

By John Doe | Published: January 15, 2024

- -

Introduction

-

Content creation has evolved significantly over the past year. In this comprehensive guide, we'll explore the most effective strategies for creating engaging content.

- -

1. Focus on Value First

-

The most successful content creators prioritize providing value to their audience. According to a recent study, 78% of consumers prefer brands that create custom content.

- -

2. Embrace Short-Form Video

-

Short-form video continues to dominate. TikTok and Instagram Reels have shown that 15-60 second videos can generate massive engagement.

- -
"Content is king, but distribution is queen." - Gary Vaynerchuk
- -

3. Use AI Wisely

-

AI tools like ChatGPT and Claude can help with ideation and drafting, but human creativity remains essential for authentic content.

- -

Key Statistics

-
    -
  • 85% of marketers use content marketing
  • -
  • Video content generates 1200% more shares
  • -
  • Long-form content gets 77% more backlinks
  • -
- -

Conclusion

-

Success in content creation requires a balance of strategy, creativity, and consistency. Start implementing these strategies today!

- - Read more articles - External resource -
- - - `; + try { + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), timeout); - return { html: mockHtml }; + const response = await fetch(url, { + headers: { + 'User-Agent': userAgent, + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'Accept-Language': 'tr-TR,tr;q=0.9,en-US;q=0.8,en;q=0.7', + 'Accept-Encoding': 'identity', + }, + signal: controller.signal, + redirect: 'follow', + }); + + clearTimeout(timeoutId); + + if (!response.ok) { + this.logger.warn(`HTTP ${response.status} for ${url}`); + return null; + } + + const contentType = response.headers.get('content-type') || ''; + if (!contentType.includes('text/html') && !contentType.includes('application/xhtml')) { + this.logger.warn(`Non-HTML content type: ${contentType} for ${url}`); + return null; + } + + const html = await response.text(); + + if (!html || html.length < 100) { + this.logger.warn(`Empty or very short response from ${url}`); + return null; + } + + this.logger.log(`Successfully fetched ${url} (${html.length} chars)`); + return { html }; + } catch (error) { + if (error.name === 'AbortError') { + this.logger.warn(`Request timed out for ${url}`); + } else { + this.logger.error(`Failed to fetch ${url}: ${error.message}`); + } + return null; + } } /** @@ -194,6 +514,7 @@ export class WebScraperService { */ private parseHtml(html: string, url: string, options?: ScraperOptions): ScrapedContent { const domain = new URL(url).hostname; + const baseUrl = new URL(url).origin; // Extract title const titleMatch = html.match(/]*>([^<]+)<\/title>/i); @@ -212,8 +533,11 @@ export class WebScraperService { // Extract links const links = options?.extractLinks !== false ? this.extractLinks(html, domain) : []; - // Extract images - const images = options?.extractImages !== false ? this.extractImages(html) : []; + // Extract images with absolute URL resolution + const images = options?.extractImages !== false ? this.extractImages(html, baseUrl) : []; + + // Extract video links (YouTube, etc.) + const videoLinks = this.extractVideoLinks(html); // Extract metadata const metadata = this.extractMetadata(html); @@ -232,6 +556,7 @@ export class WebScraperService { headings, links, images, + videoLinks, metadata, wordCount, readingTime, @@ -306,21 +631,82 @@ export class WebScraperService { } /** - * Extract images from HTML + * Extract images from HTML with absolute URL resolution */ - private extractImages(html: string): { src: string; alt: string }[] { + private extractImages(html: string, baseUrl?: string): { src: string; alt: string }[] { const images: { src: string; alt: string }[] = []; + // Match src before alt, or alt before src const regex = /]*src=["']([^"']+)["'][^>]*(?:alt=["']([^"']*)["'])?/gi; + const regex2 = /]*alt=["']([^"']*)["'][^>]*src=["']([^"']+)["']/gi; let match; + const addImage = (src: string, alt: string) => { + // Skip tiny tracking pixels, icons, and data URIs + if (src.includes('1x1') || src.includes('pixel') || src.includes('data:image/gif')) return; + if (src.endsWith('.svg') || src.endsWith('.ico')) return; + + // Resolve relative URLs + let resolvedSrc = src; + if (baseUrl && !src.startsWith('http') && !src.startsWith('//')) { + resolvedSrc = src.startsWith('/') ? `${baseUrl}${src}` : `${baseUrl}/${src}`; + } else if (src.startsWith('//')) { + resolvedSrc = `https:${src}`; + } + + // Avoid duplicates + if (!images.some(img => img.src === resolvedSrc)) { + images.push({ src: resolvedSrc, alt: alt || '' }); + } + }; + while ((match = regex.exec(html)) !== null) { - images.push({ - src: match[1], - alt: match[2] || '', - }); + addImage(match[1], match[2] || ''); + } + while ((match = regex2.exec(html)) !== null) { + addImage(match[2], match[1] || ''); } - return images.slice(0, 20); // Limit to 20 images + // Also check og:image + const ogImageMatch = html.match(/]*property=["']og:image["'][^>]*content=["']([^"']+)["']/i); + if (ogImageMatch) { + addImage(ogImageMatch[1], 'og-image'); + } + + return images.slice(0, 20); + } + + /** + * Extract video links (YouTube, Vimeo, etc.) from HTML + */ + private extractVideoLinks(html: string): string[] { + const videos: Set = new Set(); + + // YouTube iframe embeds + const iframeRegex = /]*src=["']([^"']*(?:youtube\.com|youtu\.be|vimeo\.com)[^"']*)["']/gi; + let match; + while ((match = iframeRegex.exec(html)) !== null) { + let url = match[1]; + if (url.startsWith('//')) url = `https:${url}`; + // Convert embed URL to watch URL + url = url.replace('/embed/', '/watch?v=').replace('?feature=oembed', ''); + videos.add(url); + } + + // YouTube links in anchors + const anchorRegex = /]*href=["']([^"']*(?:youtube\.com\/watch|youtu\.be\/)[^"']*)["']/gi; + while ((match = anchorRegex.exec(html)) !== null) { + videos.add(match[1]); + } + + // data-video-url or data-src attributes + const dataRegex = /data-(?:video-url|src)=["']([^"']*(?:youtube\.com|youtu\.be|vimeo\.com)[^"']*)["']/gi; + while ((match = dataRegex.exec(html)) !== null) { + let url = match[1]; + if (url.startsWith('//')) url = `https:${url}`; + videos.add(url); + } + + return [...videos].slice(0, 10); } /**