diff --git a/package-lock.json b/package-lock.json
index 8a1d37939f..817da94dc5 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -24,6 +24,7 @@
"bcrypt": "^5.1.1",
"body-parser": "^1.20.3",
"bootstrap": "^4.6.2",
+ "bullmq": "^5.71.1",
"compression": "^1.8.1",
"cookie-session": "^2.1.1",
"coupon-code": "^0.4.5",
@@ -49,6 +50,7 @@
"heapdump": "^0.3.15",
"helmet": "^4.6.0",
"in-app-purchase": "^1.11.3",
+ "ioredis": "^5.10.1",
"js2xmlparser": "^5.0.0",
"jsonwebtoken": "^9.0.2",
"jwks-rsa": "^2.1.5",
@@ -72,7 +74,6 @@
"pp-ipn": "^1.1.0",
"ps-tree": "^1.0.0",
"rate-limiter-flexible": "^2.4.2",
- "redis": "^3.1.2",
"remove-markdown": "^0.5.0",
"rimraf": "^3.0.2",
"short-uuid": "^4.2.2",
@@ -2837,6 +2838,12 @@
"resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.2.tgz",
"integrity": "sha512-6EwiSjwWYP7pTckG6I5eyFANjPhmPjUX9JRLUSfNPC7FX7zK9gyZAfUEaECL6ALTpGX5AjnBq3C9XmVWPitNpw=="
},
+ "node_modules/@ioredis/commands": {
+ "version": "1.5.1",
+ "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.5.1.tgz",
+ "integrity": "sha512-JH8ZL/ywcJyR9MmJ5BNqZllXNZQqQbnVZOqpPQqE1vHiFgAw4NHbvE0FOduNU8IX9babitBT46571OnPTT0Zcw==",
+ "license": "MIT"
+ },
"node_modules/@istanbuljs/load-nyc-config": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz",
@@ -3073,6 +3080,84 @@
"sparse-bitfield": "^3.0.3"
}
},
+ "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": {
+ "version": "3.0.3",
+ "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz",
+ "integrity": "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==",
+ "cpu": [
+ "arm64"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "darwin"
+ ]
+ },
+ "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": {
+ "version": "3.0.3",
+ "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz",
+ "integrity": "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==",
+ "cpu": [
+ "x64"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "darwin"
+ ]
+ },
+ "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": {
+ "version": "3.0.3",
+ "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz",
+ "integrity": "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==",
+ "cpu": [
+ "arm"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ]
+ },
+ "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": {
+ "version": "3.0.3",
+ "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz",
+ "integrity": "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==",
+ "cpu": [
+ "arm64"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ]
+ },
+ "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": {
+ "version": "3.0.3",
+ "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.3.tgz",
+ "integrity": "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==",
+ "cpu": [
+ "x64"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ]
+ },
+ "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": {
+ "version": "3.0.3",
+ "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz",
+ "integrity": "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==",
+ "cpu": [
+ "x64"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "win32"
+ ]
+ },
"node_modules/@nodelib/fs.scandir": {
"version": "2.1.5",
"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
@@ -6604,6 +6689,52 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
+ "node_modules/bullmq": {
+ "version": "5.71.1",
+ "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.71.1.tgz",
+ "integrity": "sha512-kOBfdcsHmO6wwmIjpersoVdYQ7jkjTgky4Yop0loc7QwSdgxliSzD69U9ijZuRrkyCJwz5p5eqxeGeQkJ0YGZQ==",
+ "license": "MIT",
+ "dependencies": {
+ "cron-parser": "4.9.0",
+ "ioredis": "5.10.1",
+ "msgpackr": "1.11.5",
+ "node-abort-controller": "3.1.1",
+ "semver": "7.7.4",
+ "tslib": "2.8.1",
+ "uuid": "11.1.0"
+ }
+ },
+ "node_modules/bullmq/node_modules/semver": {
+ "version": "7.7.4",
+ "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz",
+ "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==",
+ "license": "ISC",
+ "bin": {
+ "semver": "bin/semver.js"
+ },
+ "engines": {
+ "node": ">=10"
+ }
+ },
+ "node_modules/bullmq/node_modules/tslib": {
+ "version": "2.8.1",
+ "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
+ "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==",
+ "license": "0BSD"
+ },
+ "node_modules/bullmq/node_modules/uuid": {
+ "version": "11.1.0",
+ "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.0.tgz",
+ "integrity": "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==",
+ "funding": [
+ "https://github.com/sponsors/broofa",
+ "https://github.com/sponsors/ctavan"
+ ],
+ "license": "MIT",
+ "bin": {
+ "uuid": "dist/esm/bin/uuid"
+ }
+ },
"node_modules/bytes": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
@@ -7084,6 +7215,15 @@
"safe-buffer": "~5.1.0"
}
},
+ "node_modules/cluster-key-slot": {
+ "version": "1.1.2",
+ "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz",
+ "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==",
+ "license": "Apache-2.0",
+ "engines": {
+ "node": ">=0.10.0"
+ }
+ },
"node_modules/coa": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/coa/-/coa-2.0.2.tgz",
@@ -7607,6 +7747,18 @@
"node": "*"
}
},
+ "node_modules/cron-parser": {
+ "version": "4.9.0",
+ "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz",
+ "integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==",
+ "license": "MIT",
+ "dependencies": {
+ "luxon": "^3.2.1"
+ },
+ "engines": {
+ "node": ">=12.0.0"
+ }
+ },
"node_modules/cross-spawn": {
"version": "7.0.5",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.5.tgz",
@@ -8140,6 +8292,7 @@
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/denque/-/denque-1.5.1.tgz",
"integrity": "sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw==",
+ "dev": true,
"engines": {
"node": ">=0.10"
}
@@ -13462,6 +13615,39 @@
"node": ">=0.10.0"
}
},
+ "node_modules/ioredis": {
+ "version": "5.10.1",
+ "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.10.1.tgz",
+ "integrity": "sha512-HuEDBTI70aYdx1v6U97SbNx9F1+svQKBDo30o0b9fw055LMepzpOOd0Ccg9Q6tbqmBSJaMuY0fB7yw9/vjBYCA==",
+ "license": "MIT",
+ "dependencies": {
+ "@ioredis/commands": "1.5.1",
+ "cluster-key-slot": "^1.1.0",
+ "debug": "^4.3.4",
+ "denque": "^2.1.0",
+ "lodash.defaults": "^4.2.0",
+ "lodash.isarguments": "^3.1.0",
+ "redis-errors": "^1.2.0",
+ "redis-parser": "^3.0.0",
+ "standard-as-callback": "^2.1.0"
+ },
+ "engines": {
+ "node": ">=12.22.0"
+ },
+ "funding": {
+ "type": "opencollective",
+ "url": "https://opencollective.com/ioredis"
+ }
+ },
+ "node_modules/ioredis/node_modules/denque": {
+ "version": "2.1.0",
+ "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
+ "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==",
+ "license": "Apache-2.0",
+ "engines": {
+ "node": ">=0.10"
+ }
+ },
"node_modules/iota-array": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/iota-array/-/iota-array-1.0.0.tgz",
@@ -14651,6 +14837,12 @@
"resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz",
"integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow=="
},
+ "node_modules/lodash.defaults": {
+ "version": "4.2.0",
+ "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz",
+ "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==",
+ "license": "MIT"
+ },
"node_modules/lodash.flattendeep": {
"version": "4.4.0",
"resolved": "https://registry.npmjs.org/lodash.flattendeep/-/lodash.flattendeep-4.4.0.tgz",
@@ -14667,6 +14859,12 @@
"resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz",
"integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w=="
},
+ "node_modules/lodash.isarguments": {
+ "version": "3.1.0",
+ "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz",
+ "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==",
+ "license": "MIT"
+ },
"node_modules/lodash.isboolean": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz",
@@ -14856,6 +15054,15 @@
"resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz",
"integrity": "sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A=="
},
+ "node_modules/luxon": {
+ "version": "3.7.2",
+ "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.7.2.tgz",
+ "integrity": "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=12"
+ }
+ },
"node_modules/make-dir": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz",
@@ -15880,6 +16087,37 @@
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
+ "node_modules/msgpackr": {
+ "version": "1.11.5",
+ "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.5.tgz",
+ "integrity": "sha512-UjkUHN0yqp9RWKy0Lplhh+wlpdt9oQBYgULZOiFhV3VclSF1JnSQWZ5r9gORQlNYaUKQoR8itv7g7z1xDDuACA==",
+ "license": "MIT",
+ "optionalDependencies": {
+ "msgpackr-extract": "^3.0.2"
+ }
+ },
+ "node_modules/msgpackr-extract": {
+ "version": "3.0.3",
+ "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz",
+ "integrity": "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==",
+ "hasInstallScript": true,
+ "license": "MIT",
+ "optional": true,
+ "dependencies": {
+ "node-gyp-build-optional-packages": "5.2.2"
+ },
+ "bin": {
+ "download-msgpackr-prebuilds": "bin/download-prebuilds.js"
+ },
+ "optionalDependencies": {
+ "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3",
+ "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3",
+ "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3",
+ "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3",
+ "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3",
+ "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3"
+ }
+ },
"node_modules/multimatch": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/multimatch/-/multimatch-5.0.0.tgz",
@@ -16146,6 +16384,12 @@
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.2.1.tgz",
"integrity": "sha512-JLyh7xT1kizaEvcaXOQwOc2/Yhw6KZOvPf1S8401UyLk86CU79LN3vl7ztXGm/pZ+YjoyAJ4rxmHwbkBXJX+yw=="
},
+ "node_modules/node-abort-controller": {
+ "version": "3.1.1",
+ "resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.1.1.tgz",
+ "integrity": "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==",
+ "license": "MIT"
+ },
"node_modules/node-addon-api": {
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-5.1.0.tgz",
@@ -16217,6 +16461,21 @@
"ms": "^2.1.1"
}
},
+ "node_modules/node-gyp-build-optional-packages": {
+ "version": "5.2.2",
+ "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz",
+ "integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==",
+ "license": "MIT",
+ "optional": true,
+ "dependencies": {
+ "detect-libc": "^2.0.1"
+ },
+ "bin": {
+ "node-gyp-build-optional-packages": "bin.js",
+ "node-gyp-build-optional-packages-optional": "optional.js",
+ "node-gyp-build-optional-packages-test": "build-test.js"
+ }
+ },
"node_modules/node-loggly-bulk": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/node-loggly-bulk/-/node-loggly-bulk-4.0.1.tgz",
@@ -18383,29 +18642,6 @@
"node": ">=0.10.0"
}
},
- "node_modules/redis": {
- "version": "3.1.2",
- "resolved": "https://registry.npmjs.org/redis/-/redis-3.1.2.tgz",
- "integrity": "sha512-grn5KoZLr/qrRQVwoSkmzdbw6pwF+/rwODtrOr6vuBRiR/f3rjSTGupbF90Zpqm2oenix8Do6RV7pYEkGwlKkw==",
- "dependencies": {
- "denque": "^1.5.0",
- "redis-commands": "^1.7.0",
- "redis-errors": "^1.2.0",
- "redis-parser": "^3.0.0"
- },
- "engines": {
- "node": ">=10"
- },
- "funding": {
- "type": "opencollective",
- "url": "https://opencollective.com/node-redis"
- }
- },
- "node_modules/redis-commands": {
- "version": "1.7.0",
- "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz",
- "integrity": "sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ=="
- },
"node_modules/redis-errors": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
@@ -19910,6 +20146,12 @@
"node": "*"
}
},
+ "node_modules/standard-as-callback": {
+ "version": "2.1.0",
+ "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz",
+ "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==",
+ "license": "MIT"
+ },
"node_modules/static-extend": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/static-extend/-/static-extend-0.1.2.tgz",
diff --git a/package.json b/package.json
index c024c2ee71..6a360e1564 100644
--- a/package.json
+++ b/package.json
@@ -19,6 +19,7 @@
"bcrypt": "^5.1.1",
"body-parser": "^1.20.3",
"bootstrap": "^4.6.2",
+ "bullmq": "^5.71.1",
"compression": "^1.8.1",
"cookie-session": "^2.1.1",
"coupon-code": "^0.4.5",
@@ -44,6 +45,7 @@
"heapdump": "^0.3.15",
"helmet": "^4.6.0",
"in-app-purchase": "^1.11.3",
+ "ioredis": "^5.10.1",
"js2xmlparser": "^5.0.0",
"jsonwebtoken": "^9.0.2",
"jwks-rsa": "^2.1.5",
@@ -67,7 +69,6 @@
"pp-ipn": "^1.1.0",
"ps-tree": "^1.0.0",
"rate-limiter-flexible": "^2.4.2",
- "redis": "^3.1.2",
"remove-markdown": "^0.5.0",
"rimraf": "^3.0.2",
"short-uuid": "^4.2.2",
diff --git a/test/api/unit/libs/email.test.js b/test/api/unit/libs/email.test.js
index 49b2e89538..6ceb5d38f2 100644
--- a/test/api/unit/libs/email.test.js
+++ b/test/api/unit/libs/email.test.js
@@ -1,9 +1,9 @@
/* eslint-disable global-require */
-import got from 'got';
import nconf from 'nconf';
import requireAgain from 'require-again';
import { TAVERN_ID } from '../../../../website/server/models/group';
import { defer } from '../../../helpers/api-unit.helper';
+import worker from '../../../../website/server/libs/worker';
function getUser () {
return {
@@ -127,7 +127,7 @@ describe('emails', () => {
let sendTxn = null;
beforeEach(() => {
- sandbox.stub(got, 'post').returns(defer().promise);
+ sandbox.stub(worker, 'sendJob').returns(defer().promise);
const nconfGetStub = sandbox.stub(nconf, 'get');
nconfGetStub.withArgs('IS_PROD').returns(true);
@@ -149,13 +149,12 @@ describe('emails', () => {
};
sendTxn(mailingInfo, emailType);
- expect(got.post).to.be.called;
- expect(got.post).to.be.calledWith('http://example.com/job', sinon.match({
- json: {
- data: {
- emailType: sinon.match.same(emailType),
- to: sinon.match(value => Array.isArray(value) && value[0].name === mailingInfo.name, 'matches mailing info array'),
- },
+ expect(worker.sendJob).to.be.called;
+ expect(worker.sendJob).to.be.calledWith('email', sinon.match({
+ identifier: emailType,
+ data: {
+ emailType: sinon.match.same(emailType),
+ to: sinon.match(value => Array.isArray(value) && value[0].name === mailingInfo.name, 'matches mailing info array'),
},
}));
});
@@ -168,7 +167,7 @@ describe('emails', () => {
};
sendTxn(mailingInfo, emailType);
- expect(got.post).not.to.be.called;
+ expect(worker.sendJob).not.to.be.called;
});
it('throws error when mail target is only a string', async () => {
@@ -233,13 +232,12 @@ describe('emails', () => {
const mailingInfo = getUser();
sendTxn(mailingInfo, emailType);
- expect(got.post).to.be.called;
- expect(got.post).to.be.calledWith('http://example.com/job', sinon.match({
- json: {
- data: {
- emailType: sinon.match.same(emailType),
- to: sinon.match(val => val[0]._id === mailingInfo._id),
- },
+ expect(worker.sendJob).to.be.called;
+ expect(worker.sendJob).to.be.calledWith('email', sinon.match({
+ identifier: emailType,
+ data: {
+ emailType: sinon.match.same(emailType),
+ to: sinon.match(val => val[0]._id === mailingInfo._id),
},
}));
});
@@ -253,15 +251,14 @@ describe('emails', () => {
const variables = [];
sendTxn(mailingInfo, emailType, variables);
- expect(got.post).to.be.called;
- expect(got.post).to.be.calledWith('http://example.com/job', sinon.match({
- json: {
- data: {
- variables: sinon.match(value => value[0].name === 'BASE_URL', 'matches variables'),
- personalVariables: sinon.match(value => value[0].rcpt === mailingInfo.email
- && value[0].vars[0].name === 'RECIPIENT_NAME'
+ expect(worker.sendJob).to.be.called;
+ expect(worker.sendJob).to.be.calledWith('email', sinon.match({
+ identifier: emailType,
+ data: {
+ variables: sinon.match(value => value[0].name === 'BASE_URL', 'matches variables'),
+ personalVariables: sinon.match(value => value[0].rcpt === mailingInfo.email
+ && value[0].vars[0].name === 'RECIPIENT_NAME'
&& value[0].vars[1].name === 'RECIPIENT_UNSUB_URL', 'matches personal variables'),
- },
},
}));
});
diff --git a/test/api/v3/integration/quests/POST-groups_groupId_quests_force-start.test.js b/test/api/v3/integration/quests/POST-groups_groupId_quests_force-start.test.js
index bc084c2f44..353cb1269d 100644
--- a/test/api/v3/integration/quests/POST-groups_groupId_quests_force-start.test.js
+++ b/test/api/v3/integration/quests/POST-groups_groupId_quests_force-start.test.js
@@ -193,23 +193,6 @@ describe('POST /groups/:groupId/quests/force-start', () => {
expect(questingGroup.quest.members[notInPartyUser._id]).to.not.exist;
});
- it('removes users who have been deleted from quest.members', async () => {
- await leader.post(`/groups/${questingGroup._id}/quests/invite/${PET_QUEST}`);
- await partyMembers[0].post(`/groups/${questingGroup._id}/quests/accept`);
-
- await partyMembers[0].del('/user', {
- password: 'password',
- });
-
- await leader.post(`/groups/${questingGroup._id}/quests/force-start`);
-
- await sleep(0.5);
-
- await questingGroup.sync();
-
- expect(questingGroup.quest.members[partyMembers[0]._id]).to.not.exist;
- });
-
it('removes users who don\'t have true value in quest.members from quest.members', async () => {
const partyMemberThatRejects = partyMembers[1];
const partyMemberThatIgnores = partyMembers[2];
diff --git a/test/api/v3/integration/user/DELETE-user.test.js b/test/api/v3/integration/user/DELETE-user.test.js
index cd90f87e23..4552c536d1 100644
--- a/test/api/v3/integration/user/DELETE-user.test.js
+++ b/test/api/v3/integration/user/DELETE-user.test.js
@@ -1,13 +1,7 @@
-import {
- each,
- map,
-} from 'lodash';
import {
checkExistence,
- createAndPopulateGroup,
generateGroup,
generateUser,
- generateChallenge,
translate as t,
} from '../../../../helpers/api-integration/v3';
import {
@@ -15,6 +9,7 @@ import {
sha1Encrypt as sha1EncryptPassword,
} from '../../../../../website/server/libs/password';
import * as email from '../../../../../website/server/libs/email';
+import sendJob from '../../../../../website/server/libs/worker';
const DELETE_CONFIRMATION = 'DELETE';
@@ -47,12 +42,13 @@ describe('DELETE /user', () => {
});
});
- it('deletes the user', async () => {
- await expect(checkExistence('users', user._id)).to.eventually.eql(true);
+ it('sends deletion job to worker', async () => {
+ const workerStub = sandbox.stub(sendJob, 'sendJob');
await user.del('/user', {
password,
});
- await expect(checkExistence('users', user._id)).to.eventually.eql(false);
+ expect(workerStub).to.be.calledOnce;
+ workerStub.restore();
});
it('returns an error if excessive feedback is supplied', async () => {
@@ -84,53 +80,6 @@ describe('DELETE /user', () => {
});
});
- it('deletes the user\'s tasks', async () => {
- await user.post('/tasks/user', {
- text: 'test habit',
- type: 'habit',
- });
- await user.sync();
-
- // gets the user's tasks ids
- const ids = [];
- each(user.tasksOrder, idsForOrder => {
- ids.push(...idsForOrder);
- });
-
- expect(ids.length).to.be.above(0); // make sure the user has some task to delete
-
- await user.del('/user', {
- password,
- });
-
- await Promise.all(map(ids, id => expect(checkExistence('tasks', id)).to.eventually.eql(false)));
- });
-
- it('reduces memberCount in challenges user is linked to', async () => {
- const populatedGroup = await createAndPopulateGroup({
- members: 2,
- });
-
- const { group } = populatedGroup;
- const authorizedUser = populatedGroup.members[1];
-
- const challenge = await generateChallenge(populatedGroup.groupLeader, group);
- await populatedGroup.groupLeader.post(`/challenges/${challenge._id}/join`);
- await authorizedUser.post(`/challenges/${challenge._id}/join`);
-
- await challenge.sync();
-
- expect(challenge.memberCount).to.eql(2);
-
- await authorizedUser.del('/user', {
- password,
- });
-
- await challenge.sync();
-
- expect(challenge.memberCount).to.eql(1);
- });
-
it('sends feedback to the admin email', async () => {
sandbox.spy(email, 'sendTxn');
@@ -158,10 +107,10 @@ describe('DELETE /user', () => {
});
it('deletes the user with a legacy sha1 password', async () => {
- await expect(checkExistence('users', user._id)).to.eventually.eql(true);
const textPassword = 'mySecretPassword';
const salt = sha1MakeSalt();
const sha1HashedPassword = sha1EncryptPassword(textPassword, salt);
+ const workerStub = sandbox.stub(sendJob, 'sendJob');
await user.updateOne({
'auth.local.hashed_password': sha1HashedPassword,
@@ -179,7 +128,8 @@ describe('DELETE /user', () => {
await user.del('/user', {
password: textPassword,
});
- await expect(checkExistence('users', user._id)).to.eventually.eql(false);
+ expect(workerStub).to.be.calledOnce;
+ workerStub.restore();
});
context('last member of a party', () => {
@@ -213,11 +163,12 @@ describe('DELETE /user', () => {
});
it('deletes a Google user', async () => {
- await expect(checkExistence('users', user._id)).to.eventually.eql(true);
+ const workerStub = sandbox.stub(sendJob, 'sendJob');
await user.del('/user', {
password: DELETE_CONFIRMATION,
});
- await expect(checkExistence('users', user._id)).to.eventually.eql(false);
+ expect(workerStub).to.be.calledOnce;
+ workerStub.restore();
});
});
@@ -232,12 +183,13 @@ describe('DELETE /user', () => {
});
});
- it('deletes a Apple user', async () => {
- await expect(checkExistence('users', user._id)).to.eventually.eql(true);
+ it('deletes an Apple user', async () => {
+ const workerStub = sandbox.stub(sendJob, 'sendJob');
await user.del('/user', {
password: DELETE_CONFIRMATION,
});
- await expect(checkExistence('users', user._id)).to.eventually.eql(false);
+ expect(workerStub).to.be.calledOnce;
+ workerStub.restore();
});
});
});
diff --git a/website/client/src/pages/settings/settingRows/deleteAccount.vue b/website/client/src/pages/settings/settingRows/deleteAccount.vue
index f9d62517ff..4bacb56e59 100644
--- a/website/client/src/pages/settings/settingRows/deleteAccount.vue
+++ b/website/client/src/pages/settings/settingRows/deleteAccount.vue
@@ -67,7 +67,7 @@
diff --git a/website/common/locales/en/settings.json b/website/common/locales/en/settings.json
index 6105305720..eb8abbf0d8 100644
--- a/website/common/locales/en/settings.json
+++ b/website/common/locales/en/settings.json
@@ -33,7 +33,7 @@
"resetAccPop": "Start over, removing all levels, gold, gear, history, and tasks.",
"deleteAccount": "Delete Account",
"deleteAccPop": "Cancel and remove your Habitica account.",
- "feedback": "If you'd like to give us feedback, please enter it below - we'd love to hear your feedback! It will be anonymous unless you choose to enter your contact details. Don't speak English well? No problem! Use the language you prefer.",
+ "feedback": "We'd love to hear your feedback! If you'd like to share any, enter it below. It will be anonymous unless you choose to include your contact details.",
"feedbackPlaceholder": "Add your feedback",
"dataExport": "Data Export",
"saveData": "Here are a few options for saving your data.",
@@ -82,8 +82,8 @@
"resetText2": "Another option is using an Orb of Rebirth, which will reset everything else while preserving your Tasks and Equipment.",
"resetTextLocal": "If you're absolutely certain, type your password into the text box below.",
"resetTextSocial": "If you're absolutely certain, type \"<%= magicWord %>\" into the text box below.",
- "deleteLocalAccountText": "Are you sure? This will delete your account forever, and it can never be restored! You will need to register a new account to use Habitica again. Banked or spent Gems will not be refunded. If you're absolutely certain, type your password into the text box below.",
- "deleteSocialAccountText": "Are you sure? This will delete your account forever, and it can never be restored! You will need to register a new account to use Habitica again. Banked or spent Gems will not be refunded. If you're absolutely certain, type \"<%= magicWord %>\" into the text box below.",
+ "deleteLocalAccountText": "Are you sure? This action is permanent. Deleting your account will remove all of your data, and it cannot be recovered. Gems will not be refunded.
Please allow up to 24 hours for account deletion to complete, and up to 30 days for analytics data to be removed if you opted in. Once complete, you'll be able to register for a new Habitica account using your previous login information.
To continue, type your password below.",
+ "deleteSocialAccountText": "Are you sure? This action is permanent. Deleting your account will remove all of your data, and it cannot be recovered. Gems will not be refunded.
Please allow up to 24 hours for account deletion to complete, and up to 30 days for analytics data to be removed if you opted in. Once complete, you'll be able to register for a new Habitica account using your previous login information.
To continue, type <%= magicWord %> below.",
"API": "API",
"APICopied": "API token copied to clipboard.",
"APITokenTitle": "API Token",
diff --git a/website/server/controllers/api-v3/user.js b/website/server/controllers/api-v3/user.js
index 216070dfb8..755181e194 100644
--- a/website/server/controllers/api-v3/user.js
+++ b/website/server/controllers/api-v3/user.js
@@ -9,10 +9,6 @@ import {
BadRequest,
NotAuthorized,
} from '../../libs/errors';
-import {
- basicFields as basicGroupFields,
- model as Group,
-} from '../../models/group';
import * as Tasks from '../../models/task';
import * as passwordUtils from '../../libs/password';
import {
@@ -22,6 +18,7 @@ import {
getUserInfo,
sendTxn,
} from '../../libs/email';
+import worker from '../../libs/worker';
import * as inboxLib from '../../libs/inbox';
import * as userLib from '../../libs/user';
import { model as UserHistory } from '../../models/userHistory';
@@ -297,21 +294,6 @@ api.deleteUser = {
throw new NotAuthorized(res.t('cannotDeleteActiveAccount'));
}
- const types = ['party', 'guilds'];
- const groupFields = basicGroupFields.concat(' leader memberCount purchased');
-
- const groupsUserIsMemberOf = await Group.getGroups({ user, types, groupFields });
-
- const groupLeavePromises = groupsUserIsMemberOf.map(group => group.leave(user, 'remove-all'));
-
- await Promise.all(groupLeavePromises);
-
- await Tasks.Task.deleteMany({
- userId: user._id,
- }).exec();
-
- await user.deleteOne();
-
if (feedback) {
sendTxn({ email: TECH_ASSISTANCE_EMAIL }, 'admin-feedback', [
{ name: 'PROFILE_NAME', content: user.profile.name },
@@ -323,6 +305,15 @@ api.deleteUser = {
]);
}
+ worker.sendJob('deleteUser', {
+ identifier: user._id,
+ data: {
+ userId: user._id,
+ deleteAccount: true,
+ deleteAmplitude: true,
+ },
+ });
+
res.respond(200, {});
},
};
diff --git a/website/server/controllers/api-v4/members.js b/website/server/controllers/api-v4/members.js
index 9717907433..84177be6c5 100644
--- a/website/server/controllers/api-v4/members.js
+++ b/website/server/controllers/api-v4/members.js
@@ -1,4 +1,4 @@
-import { sendJob } from '../../libs/worker';
+import worker from '../../libs/worker';
import { authWithHeaders } from '../../middlewares/auth';
import { ensurePermission } from '../../middlewares/ensureAccessRight';
import { TransactionModel as Transaction } from '../../models/transaction';
@@ -48,7 +48,8 @@ api.deleteMember = {
req.checkQuery('deleteAmplitude').optional().isIn(['true', 'false']);
const validationErrors = req.validationErrors();
if (validationErrors) throw validationErrors;
- sendJob('delete-user', {
+ await worker.sendJob('deleteUser', {
+ identifier: req.params.memberId,
data: {
userId: req.params.memberId,
deleteAccount: req.query.deleteAccount === 'true',
diff --git a/website/server/controllers/api-v4/status.js b/website/server/controllers/api-v4/status.js
index 980a215d1b..255c5fc7f5 100644
--- a/website/server/controllers/api-v4/status.js
+++ b/website/server/controllers/api-v4/status.js
@@ -24,7 +24,10 @@ api.getReady = {
middlewares: [disableCache],
async handler (req, res) {
// This allows kubernetes to determine if the server is ready to receive traffic
- if (!SERVER_STATUS.MONGODB || !SERVER_STATUS.REDIS || !SERVER_STATUS.EXPRESS) {
+ if (!SERVER_STATUS.MONGODB
+ || !SERVER_STATUS.RATE_LIMITER
+ || !SERVER_STATUS.WORKER
+ || !SERVER_STATUS.EXPRESS) {
res.respond(503, {
status: 'not ready',
});
diff --git a/website/server/libs/email.js b/website/server/libs/email.js
index 41ce24e8ce..9a3ce69b20 100644
--- a/website/server/libs/email.js
+++ b/website/server/libs/email.js
@@ -2,7 +2,7 @@ import nconf from 'nconf';
import { TAVERN_ID } from '../models/group'; // eslint-disable-line import/no-cycle
import { encrypt } from './encryption';
import common from '../../common';
-import { sendJob } from './worker';
+import worker from './worker';
const IS_PROD = nconf.get('IS_PROD');
const BASE_URL = nconf.get('BASE_URL');
@@ -148,7 +148,8 @@ export async function sendTxn (mailingInfoArray, emailType, variables, personalV
}
if (IS_PROD && mailingInfoArray.length > 0) {
- return sendJob('email', {
+ return worker.sendJob('email', {
+ identifier: emailType,
data: {
emailType,
to: mailingInfoArray,
diff --git a/website/server/libs/redis.js b/website/server/libs/redis.js
new file mode 100644
index 0000000000..744c34a537
--- /dev/null
+++ b/website/server/libs/redis.js
@@ -0,0 +1,22 @@
+import IORedis from 'ioredis';
+
+export default function setupRedis (connectionOptions, config) {
+ const redisConfig = { ...config };
+ if (connectionOptions.username) {
+ redisConfig.username = connectionOptions.username;
+ }
+ if (connectionOptions.password) {
+ redisConfig.password = connectionOptions.password;
+ }
+ if (connectionOptions.db) {
+ redisConfig.db = connectionOptions.db;
+ }
+ let connection;
+ const redisUrl = connectionOptions.url;
+ if (redisUrl) {
+ connection = new IORedis(redisUrl, redisConfig);
+ } else {
+ connection = new IORedis(connectionOptions.port, connectionOptions.host, redisConfig);
+ }
+ return connection;
+}
diff --git a/website/server/libs/serverStatus.js b/website/server/libs/serverStatus.js
index ecfb6423b9..c5c1e99396 100644
--- a/website/server/libs/serverStatus.js
+++ b/website/server/libs/serverStatus.js
@@ -1,6 +1,7 @@
const SERVER_STATUS = {
MONGODB: false,
- REDIS: false,
+ RATE_LIMITER: false,
+ WORKER: false,
EXPRESS: false,
};
diff --git a/website/server/libs/worker.js b/website/server/libs/worker.js
index 8f4d518ae8..21b258f538 100644
--- a/website/server/libs/worker.js
+++ b/website/server/libs/worker.js
@@ -1,33 +1,49 @@
-import got from 'got';
import nconf from 'nconf';
-import logger from './logger';
+import { Queue } from 'bullmq';
+import setupRedis from './redis';
+import SERVER_STATUS from './serverStatus';
-const EMAIL_SERVER = {
- url: nconf.get('EMAIL_SERVER_URL'),
- auth: {
- user: nconf.get('EMAIL_SERVER_AUTH_USER'),
- password: nconf.get('EMAIL_SERVER_AUTH_PASSWORD'),
- },
-};
+let redisClient;
+const queues = {};
-export function sendJob (type, config) {
- const { data, options } = config;
- const usedOptions = {
- backoff: { delay: 10 * 60 * 1000, type: 'exponential' },
- ...options,
+if (nconf.get('WORKER_REDIS_URL')) {
+ redisClient = setupRedis({
+ url: nconf.get('WORKER_REDIS_URL'),
+ username: nconf.get('WORKER_REDIS_USERNAME'),
+ password: nconf.get('WORKER_REDIS_PASSWORD'),
+ });
+
+ redisClient.on('ready', () => {
+ SERVER_STATUS.WORKER = true;
+ });
+
+ redisClient.on('reconnecting', () => {
+ SERVER_STATUS.WORKER = false;
+ });
+
+ const queueConfig = {
+ connection: redisClient,
};
+ if (nconf.get('WORKER_REDIS_KEY_PREFIX')) {
+ queueConfig.prefix = nconf.get('WORKER_REDIS_KEY_PREFIX');
+ }
- return got.post(`${EMAIL_SERVER.url}/job`, {
- retry: 5, // retry the http request to the email server 5 times
- timeout: 60000, // wait up to 60s before timing out
- username: EMAIL_SERVER.auth.user,
- password: EMAIL_SERVER.auth.password,
- json: {
- type,
- data,
- options: usedOptions,
- },
- }).json().catch(err => logger.error(err, {
- extraMessage: 'Error while sending an email.',
- }));
+ queues.email = new Queue('emails', queueConfig);
+ queues.deleteUser = new Queue('DeleteUsers', queueConfig);
+} else {
+ SERVER_STATUS.WORKER = true;
}
+
+function sendJob (type, config) {
+ if (!queues[type]) {
+ return Promise.reject(new Error(`Queue ${type} does not exist`));
+ }
+ const { identifier, data } = config;
+ return queues[type].add(identifier, data);
+}
+
+export function getRedisClient () {
+ return redisClient;
+}
+
+export default { sendJob };
diff --git a/website/server/middlewares/rateLimiter.js b/website/server/middlewares/rateLimiter.js
index bc79f75fbe..f19b95038d 100644
--- a/website/server/middlewares/rateLimiter.js
+++ b/website/server/middlewares/rateLimiter.js
@@ -1,5 +1,5 @@
import nconf from 'nconf';
-import redis from 'redis';
+import redis from 'ioredis';
import {
RateLimiterRedis,
RateLimiterMemory,
diff --git a/website/server/models/group.js b/website/server/models/group.js
index 845584b13b..5693ce1231 100644
--- a/website/server/models/group.js
+++ b/website/server/models/group.js
@@ -1457,6 +1457,7 @@ schema.methods.leave = async function leaveGroup (user, keep = 'keep-all', keepC
if (members.length === 0) {
promises.push(group.deleteOne());
+ promises.push(Chat.deleteMany({ groupId: group._id }));
return Promise.all(promises);
}
}
diff --git a/website/server/server.js b/website/server/server.js
index 7d0554ef50..7182fa0e95 100644
--- a/website/server/server.js
+++ b/website/server/server.js
@@ -2,7 +2,7 @@ import nconf from 'nconf';
import express from 'express';
import http from 'http';
import mongoose from 'mongoose';
-import redis from 'redis';
+import redis from 'ioredis';
import logger from './libs/logger';
// Setup translations