Initial commit of working RSS Aggregator build
This commit is contained in:
+21
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2018 taskforcesh
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
+237
@@ -0,0 +1,237 @@
|
||||
<div align="center">
|
||||
<br/>
|
||||
<img src="https://user-images.githubusercontent.com/95200/143832033-32e868df-f3b0-4251-97fb-c64809a43d36.png" width="800" />
|
||||
<br/>
|
||||
<br/>
|
||||
<p>
|
||||
The fastest, most reliable, Redis-based distributed queue for Node. <br/>
|
||||
Carefully written for rock solid stability and atomicity.
|
||||
</p>
|
||||
Read the <a href="https://docs.bullmq.io">documentation</a>
|
||||
<br/>
|
||||
<br/>
|
||||
<p>
|
||||
<a href="https://openbase.com/js/bullmq?utm_source=embedded&utm_medium=badge&utm_campaign=rate-badge">
|
||||
<img src="https://badges.openbase.com/js/rating/bullmq.svg"/>
|
||||
</a>
|
||||
<a href="https://gitter.im/OptimalBits/bull">
|
||||
<img src="https://badges.gitter.im/Join%20Chat.svg"/>
|
||||
</a>
|
||||
<a href="https://badge.fury.io/js/bullmq">
|
||||
<img src="https://badge.fury.io/js/bullmq.svg"/>
|
||||
</a>
|
||||
<a href="https://coveralls.io/github/taskforcesh/bullmq?branch=master">
|
||||
<img src="https://coveralls.io/repos/github/taskforcesh/bullmq/badge.svg?branch=master"/>
|
||||
</a>
|
||||
<a href="https://github.com/semantic-release/semantic-release">
|
||||
<img src="https://img.shields.io/badge/%20%20%F0%9F%93%A6%F0%9F%9A%80-semantic--release-e10079.svg"/>
|
||||
</a>
|
||||
</p>
|
||||
<p>
|
||||
<em>Follow <a href="https://twitter.com/manast">@manast</a> for *important* Bull/BullMQ/BullMQ-Pro news and updates!</em>
|
||||
</p>
|
||||
</div>
|
||||
|
||||
# 🛠 Tutorials
|
||||
|
||||
You can find tutorials and news in this blog: https://blog.taskforce.sh/
|
||||
|
||||
# Official FrontEnd
|
||||
|
||||
[<img src="http://taskforce.sh/assets/logo_square.png" width="150" alt="Taskforce.sh, Inc" style="padding: 200px"/>](https://taskforce.sh)
|
||||
|
||||
Supercharge your queues with a professional front end:
|
||||
|
||||
- Get a complete overview of all your queues.
|
||||
- Inspect jobs, search, retry, or promote delayed jobs.
|
||||
- Metrics and statistics.
|
||||
- and many more features.
|
||||
|
||||
Sign up at [Taskforce.sh](https://taskforce.sh)
|
||||
|
||||
# 🚀 Sponsors 🚀
|
||||
|
||||
<table cellspacing="0" cellpadding="0" border="0">
|
||||
<tr>
|
||||
<td>
|
||||
<a href="https://www.dragonflydb.io/">
|
||||
<img src="https://raw.githubusercontent.com/dragonflydb/dragonfly/main/.github/images/logo-full.svg" width=550 alt="Dragonfly" />
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
Dragonfly is a new Redis™ drop-in replacement that is fully compatible with BullMQ and brings some important advantages over Redis™ such as massive
|
||||
better performance by utilizing all CPU cores available and faster and more memory efficient data structures. Read more <a href="https://www.dragonflydb.io/docs/integrations/bullmq">here</a> on how to use it with BullMQ.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<a href="https://dashboard.memetria.com/new?utm_campaign=BULLMQ">
|
||||
<img src="https://www.memetria.com/images/logo/memetria-logo.svg" width=350 alt="Memetria for Redis" />
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
If you need high quality production Redis instances for your BullMQ project, please consider subscribing to <a href="https://dashboard.memetria.com/new?utm_campaign=BULLMQ">Memetria for Redis</a>, leaders in Redis hosting that works perfectly with BullMQ. Use the promo code "BULLMQ" when signing up to help us sponsor the development of BullMQ!
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
# Used by
|
||||
|
||||
Some notable organizations using BullMQ:
|
||||
|
||||
<table cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td valign="center">
|
||||
<a href="https://github.com/microsoft/lage">
|
||||
<img
|
||||
src="https://files.gitbook.com/v0/b/gitbook-x-prod.appspot.com/o/spaces%2F-LUuDmt_xXMfG66Rn1GA%2Fuploads%2FUvwInTAmk7hxAViDwJzU%2Fclipart1565701.png?alt=media"
|
||||
width="150"
|
||||
alt="Microsoft"
|
||||
/>
|
||||
</a>
|
||||
</td>
|
||||
<td valign="center">
|
||||
<a href="https://github.com/vendure-ecommerce/vendure">
|
||||
<img
|
||||
src="https://files.gitbook.com/v0/b/gitbook-x-prod.appspot.com/o/spaces%2F-LUuDmt_xXMfG66Rn1GA%2Fuploads%2FvT30DUqsi61gL8edn3R2%2Fwordmark-logo.png?alt=media"
|
||||
width="150"
|
||||
alt="Vendure"
|
||||
/>
|
||||
</a>
|
||||
</td>
|
||||
<td valign="center">
|
||||
<a href="https://github.com/datawrapper/datawrapper">
|
||||
<img
|
||||
src="https://files.gitbook.com/v0/b/gitbook-x-prod.appspot.com/o/spaces%2F-LUuDmt_xXMfG66Rn1GA%2Fuploads%2FCJ5XmotpBBsuSgD8CilC%2Fdatawrapper-logo.png?alt=media"
|
||||
width="150"
|
||||
alt="Datawrapper"
|
||||
/>
|
||||
</a>
|
||||
</td>
|
||||
<td valign="center">
|
||||
<a href="https://github.com/nestjs/bull/tree/master/packages/bullmq">
|
||||
<img
|
||||
src="https://876297641-files.gitbook.io/~/files/v0/b/gitbook-x-prod.appspot.com/o/spaces%2F-LUuDmt_xXMfG66Rn1GA%2Fuploads%2FfAcGye182utFUtPKdLqJ%2FScreenshot%202022-02-15%20at%2011.32.39.png?alt=media&token=29feb550-f0bc-467d-a290-f700701d7d15"
|
||||
width="150"
|
||||
alt="Nest"
|
||||
/>
|
||||
</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td valign="center">
|
||||
<a href="https://github.com/teamcurri">
|
||||
<img
|
||||
src="https://user-images.githubusercontent.com/659829/161662129-ae645bc4-c1e9-48ff-997e-4cee281a964a.png"
|
||||
width="150"
|
||||
alt="Curri"
|
||||
/>
|
||||
</a>
|
||||
</td>
|
||||
<td valign="center">
|
||||
<a href="https://novu.co">
|
||||
<img
|
||||
src="https://assets.super.so/1e9f5a51-c4c6-4fca-b6e8-25fa0186f139/images/0f550019-16db-4a65-90d1-1bdb7d3c5f20/novu-logo-gradient-light-background2x.png"
|
||||
width="150"
|
||||
alt="Novu"
|
||||
/>
|
||||
</a>
|
||||
</td>
|
||||
</td>
|
||||
<td valign="center">
|
||||
<a href="https://www.nocodb.com">
|
||||
<img
|
||||
src="https://github.com/nocodb/nocodb/raw/develop/packages/nc-gui/assets/img/icons/512x512.png"
|
||||
width="50"
|
||||
alt="NoCodeDB"
|
||||
/>
|
||||
</a>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
# The gist
|
||||
|
||||
Install:
|
||||
|
||||
```
|
||||
$ yarn add bullmq
|
||||
```
|
||||
|
||||
Add jobs to the queue:
|
||||
|
||||
```ts
|
||||
import { Queue } from 'bullmq';
|
||||
|
||||
const queue = new Queue('Paint');
|
||||
|
||||
queue.add('cars', { color: 'blue' });
|
||||
```
|
||||
|
||||
Process the jobs in your workers:
|
||||
|
||||
```ts
|
||||
import { Worker } from 'bullmq';
|
||||
|
||||
const worker = new Worker('Paint', async job => {
|
||||
if (job.name === 'cars') {
|
||||
await paintCar(job.data.color);
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
Listen to jobs for completion:
|
||||
|
||||
```ts
|
||||
import { QueueEvents } from 'bullmq';
|
||||
|
||||
const queueEvents = new QueueEvents('Paint');
|
||||
|
||||
queueEvents.on('completed', ({ jobId }) => {
|
||||
console.log('done painting');
|
||||
});
|
||||
|
||||
queueEvents.on(
|
||||
'failed',
|
||||
({ jobId, failedReason }: { jobId: string; failedReason: string }) => {
|
||||
console.error('error painting', failedReason);
|
||||
},
|
||||
);
|
||||
```
|
||||
|
||||
This is just scratching the surface, check all the features and more in the official <a href="https://docs.bullmq.io">documentation</a>
|
||||
|
||||
# Feature Comparison
|
||||
|
||||
Since there are a few job queue solutions, here is a table comparing them:
|
||||
|
||||
| Feature | [BullMQ-Pro](https://bullmq.io/#bullmq-pro) | [BullMQ](https://bullmq.io) | Bull | Kue | Bee | Agenda |
|
||||
| :------------------------ | :-------------: | :-------------: | :-------------: | :---: | -------- | ------ |
|
||||
| Backend | redis | redis | redis | redis | redis | mongo |
|
||||
| Observables | ✓ | | | | | |
|
||||
| Group Rate Limit | ✓ | | | | | |
|
||||
| Group Support | ✓ | | | | | |
|
||||
| Batches Support | ✓ | | | | | |
|
||||
| Parent/Child Dependencies | ✓ | ✓ | | | | |
|
||||
| Priorities | ✓ | ✓ | ✓ | ✓ | | ✓ |
|
||||
| Concurrency | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ |
|
||||
| Delayed jobs | ✓ | ✓ | ✓ | ✓ | | ✓ |
|
||||
| Global events | ✓ | ✓ | ✓ | ✓ | | |
|
||||
| Rate Limiter | ✓ | ✓ | ✓ | | | |
|
||||
| Pause/Resume | ✓ | ✓ | ✓ | ✓ | | |
|
||||
| Sandboxed worker | ✓ | ✓ | ✓ | | | |
|
||||
| Repeatable jobs | ✓ | ✓ | ✓ | | | ✓ |
|
||||
| Atomic ops | ✓ | ✓ | ✓ | | ✓ | |
|
||||
| Persistence | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ |
|
||||
| UI | ✓ | ✓ | ✓ | ✓ | | ✓ |
|
||||
| Optimized for | Jobs / Messages | Jobs / Messages | Jobs / Messages | Jobs | Messages | Jobs |
|
||||
|
||||
## Contributing
|
||||
|
||||
Fork the repo, make some changes, submit a pull-request! Here is the [contributing](contributing.md) doc that has more details.
|
||||
|
||||
# Thanks
|
||||
|
||||
Thanks for all the contributors that made this library possible,
|
||||
also a special mention to Leon van Kammen that kindly donated
|
||||
his npm bullmq repo.
|
||||
+97
@@ -0,0 +1,97 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.AsyncFifoQueue = void 0;
|
||||
/**
|
||||
* AsyncFifoQueue
|
||||
*
|
||||
* A minimal FIFO queue for asynchronous operations. Allows adding asynchronous operations
|
||||
* and consume them in the order they are resolved.
|
||||
*
|
||||
* TODO: Optimize using a linked list for the queue instead of an array.
|
||||
* Current implementation requires memory copies when shifting the queue.
|
||||
* For a linked linked implementation, we can exploit the fact that the
|
||||
* maximum number of elements in the list will never exceen the concurrency factor
|
||||
* of the worker, so the nodes of the list could be pre-allocated.
|
||||
*/
|
||||
class AsyncFifoQueue {
|
||||
constructor(ignoreErrors = false) {
|
||||
this.ignoreErrors = ignoreErrors;
|
||||
/**
|
||||
* A queue of completed promises. As the pending
|
||||
* promises are resolved, they are added to this queue.
|
||||
*/
|
||||
this.queue = [];
|
||||
/**
|
||||
* A set of pending promises.
|
||||
*/
|
||||
this.pending = new Set();
|
||||
this.newPromise();
|
||||
}
|
||||
add(promise) {
|
||||
this.pending.add(promise);
|
||||
promise
|
||||
.then(data => {
|
||||
this.pending.delete(promise);
|
||||
if (this.queue.length === 0) {
|
||||
this.resolvePromise(data);
|
||||
}
|
||||
this.queue.push(data);
|
||||
})
|
||||
.catch(err => {
|
||||
// Ignore errors
|
||||
if (this.ignoreErrors) {
|
||||
this.queue.push(undefined);
|
||||
}
|
||||
this.pending.delete(promise);
|
||||
this.rejectPromise(err);
|
||||
});
|
||||
}
|
||||
async waitAll() {
|
||||
await Promise.all(this.pending);
|
||||
}
|
||||
numTotal() {
|
||||
return this.pending.size + this.queue.length;
|
||||
}
|
||||
numPending() {
|
||||
return this.pending.size;
|
||||
}
|
||||
numQueued() {
|
||||
return this.queue.length;
|
||||
}
|
||||
resolvePromise(data) {
|
||||
this.resolve(data);
|
||||
this.newPromise();
|
||||
}
|
||||
rejectPromise(err) {
|
||||
this.reject(err);
|
||||
this.newPromise();
|
||||
}
|
||||
newPromise() {
|
||||
this.nextPromise = new Promise((resolve, reject) => {
|
||||
this.resolve = resolve;
|
||||
this.reject = reject;
|
||||
});
|
||||
}
|
||||
async wait() {
|
||||
return this.nextPromise;
|
||||
}
|
||||
async fetch() {
|
||||
if (this.pending.size === 0 && this.queue.length === 0) {
|
||||
return;
|
||||
}
|
||||
while (this.queue.length === 0) {
|
||||
try {
|
||||
await this.wait();
|
||||
}
|
||||
catch (err) {
|
||||
// Ignore errors
|
||||
if (!this.ignoreErrors) {
|
||||
console.error('Unexpected Error in AsyncFifoQueue', err);
|
||||
}
|
||||
}
|
||||
}
|
||||
return this.queue.shift();
|
||||
}
|
||||
}
|
||||
exports.AsyncFifoQueue = AsyncFifoQueue;
|
||||
//# sourceMappingURL=async-fifo-queue.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"async-fifo-queue.js","sourceRoot":"","sources":["../../../src/classes/async-fifo-queue.ts"],"names":[],"mappings":";;;AAAA;;;;;;;;;;;GAWG;AACH,MAAa,cAAc;IAqBzB,YAAoB,eAAe,KAAK;QAApB,iBAAY,GAAZ,YAAY,CAAQ;QApBxC;;;WAGG;QACK,UAAK,GAAsB,EAAE,CAAC;QAEtC;;WAEG;QACK,YAAO,GAAG,IAAI,GAAG,EAAc,CAAC;QAYtC,IAAI,CAAC,UAAU,EAAE,CAAC;IACpB,CAAC;IAEM,GAAG,CAAC,OAAmB;QAC5B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAE1B,OAAO;aACJ,IAAI,CAAC,IAAI,CAAC,EAAE;YACX,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAE7B,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;gBAC3B,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;aAC3B;YACD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,CAAC,CAAC;aACD,KAAK,CAAC,GAAG,CAAC,EAAE;YACX,gBAAgB;YAChB,IAAI,IAAI,CAAC,YAAY,EAAE;gBACrB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;aAC5B;YACD,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC7B,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;IACP,CAAC;IAEM,KAAK,CAAC,OAAO;QAClB,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAClC,CAAC;IAEM,QAAQ;QACb,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAC/C,CAAC;IAEM,UAAU;QACf,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;IAC3B,CAAC;IAEM,SAAS;QACd,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAC3B,CAAC;IAEO,cAAc,CAAC,IAAO;QAC5B,IAAI,CAAC,OAAQ,CAAC,IAAI,CAAC,CAAC;QACpB,IAAI,CAAC,UAAU,EAAE,CAAC;IACpB,CAAC;IAEO,aAAa,CAAC,GAAQ;QAC5B,IAAI,CAAC,MAAO,CAAC,GAAG,CAAC,CAAC;QAClB,IAAI,CAAC,UAAU,EAAE,CAAC;IACpB,CAAC;IAEO,UAAU;QAChB,IAAI,CAAC,WAAW,GAAG,IAAI,OAAO,CAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAChE,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;YACvB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACvB,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,IAAI;QAChB,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAEM,KAAK,CAAC,KAAK;QAChB,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;YACtD,OAAO;SACR;QACD,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;YAC9B,IAAI;gBACF,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;aACnB;YAAC,OAAO,GAAG,EAAE;gBACZ,gBAAgB;gBAChB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;oBACtB,OAAO,CAAC,KAAK,CAAC,oCAAoC,EAAE,GAAG,CAAC,CAAC;iBAC1D;aACF;SACF;QACD,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IAC5B,CAAC;CACF;AApGD,wCAoGC"}
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.Backoffs = void 0;
|
||||
class Backoffs {
|
||||
static normalize(backoff) {
|
||||
if (Number.isFinite(backoff)) {
|
||||
return {
|
||||
type: 'fixed',
|
||||
delay: backoff,
|
||||
};
|
||||
}
|
||||
else if (backoff) {
|
||||
return backoff;
|
||||
}
|
||||
}
|
||||
static calculate(backoff, attemptsMade, err, job, customStrategy) {
|
||||
if (backoff) {
|
||||
const strategy = lookupStrategy(backoff, customStrategy);
|
||||
return strategy(attemptsMade, backoff.type, err, job);
|
||||
}
|
||||
}
|
||||
}
|
||||
exports.Backoffs = Backoffs;
|
||||
Backoffs.builtinStrategies = {
|
||||
fixed: function (delay) {
|
||||
return function () {
|
||||
return delay;
|
||||
};
|
||||
},
|
||||
exponential: function (delay) {
|
||||
return function (attemptsMade) {
|
||||
return Math.round(Math.pow(2, attemptsMade - 1) * delay);
|
||||
};
|
||||
},
|
||||
};
|
||||
function lookupStrategy(backoff, customStrategy) {
|
||||
if (backoff.type in Backoffs.builtinStrategies) {
|
||||
return Backoffs.builtinStrategies[backoff.type](backoff.delay);
|
||||
}
|
||||
else if (customStrategy) {
|
||||
return customStrategy;
|
||||
}
|
||||
else {
|
||||
throw new Error(`Unknown backoff strategy ${backoff.type}.
|
||||
If a custom backoff strategy is used, specify it when the queue is created.`);
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=backoffs.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"backoffs.js","sourceRoot":"","sources":["../../../src/classes/backoffs.ts"],"names":[],"mappings":";;;AAOA,MAAa,QAAQ;IAenB,MAAM,CAAC,SAAS,CACd,OAAgC;QAEhC,IAAI,MAAM,CAAC,QAAQ,CAAS,OAAO,CAAC,EAAE;YACpC,OAAO;gBACL,IAAI,EAAE,OAAO;gBACb,KAAK,EAAU,OAAO;aACvB,CAAC;SACH;aAAM,IAAI,OAAO,EAAE;YAClB,OAAuB,OAAO,CAAC;SAChC;IACH,CAAC;IAED,MAAM,CAAC,SAAS,CACd,OAAuB,EACvB,YAAoB,EACpB,GAAU,EACV,GAAe,EACf,cAAgC;QAEhC,IAAI,OAAO,EAAE;YACX,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;YAEzD,OAAO,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;SACvD;IACH,CAAC;;AAxCH,4BAyCC;AAxCQ,0BAAiB,GAAsB;IAC5C,KAAK,EAAE,UAAU,KAAa;QAC5B,OAAO;YACL,OAAO,KAAK,CAAC;QACf,CAAC,CAAC;IACJ,CAAC;IAED,WAAW,EAAE,UAAU,KAAa;QAClC,OAAO,UAAU,YAAoB;YACnC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,YAAY,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC;QAC3D,CAAC,CAAC;IACJ,CAAC;CACF,CAAC;AA8BJ,SAAS,cAAc,CACrB,OAAuB,EACvB,cAAgC;IAEhC,IAAI,OAAO,CAAC,IAAI,IAAI,QAAQ,CAAC,iBAAiB,EAAE;QAC9C,OAAO,QAAQ,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAM,CAAC,CAAC;KACjE;SAAM,IAAI,cAAc,EAAE;QACzB,OAAO,cAAc,CAAC;KACvB;SAAM;QACL,MAAM,IAAI,KAAK,CACb,4BAA4B,OAAO,CAAC,IAAI;kFACoC,CAC7E,CAAC;KACH;AACH,CAAC"}
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.ChildPool = void 0;
|
||||
const path = require("path");
|
||||
const child_1 = require("./child");
|
||||
const CHILD_KILL_TIMEOUT = 30000;
|
||||
class ChildPool {
|
||||
constructor({ mainFile = path.join(process.cwd(), 'dist/cjs/classes/main.js'), useWorkerThreads, }) {
|
||||
this.retained = {};
|
||||
this.free = {};
|
||||
this.opts = { mainFile, useWorkerThreads };
|
||||
}
|
||||
async retain(processFile) {
|
||||
let child = this.getFree(processFile).pop();
|
||||
if (child) {
|
||||
this.retained[child.pid] = child;
|
||||
return child;
|
||||
}
|
||||
child = new child_1.Child(this.opts.mainFile, processFile, {
|
||||
useWorkerThreads: this.opts.useWorkerThreads,
|
||||
});
|
||||
child.on('exit', this.remove.bind(this, child));
|
||||
try {
|
||||
await child.init();
|
||||
this.retained[child.pid] = child;
|
||||
return child;
|
||||
}
|
||||
catch (err) {
|
||||
console.error(err);
|
||||
this.release(child);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
release(child) {
|
||||
delete this.retained[child.pid];
|
||||
this.getFree(child.processFile).push(child);
|
||||
}
|
||||
remove(child) {
|
||||
delete this.retained[child.pid];
|
||||
const free = this.getFree(child.processFile);
|
||||
const childIndex = free.indexOf(child);
|
||||
if (childIndex > -1) {
|
||||
free.splice(childIndex, 1);
|
||||
}
|
||||
}
|
||||
async kill(child, signal = 'SIGKILL') {
|
||||
this.remove(child);
|
||||
return child.kill(signal, CHILD_KILL_TIMEOUT);
|
||||
}
|
||||
async clean() {
|
||||
const children = Object.values(this.retained).concat(this.getAllFree());
|
||||
this.retained = {};
|
||||
this.free = {};
|
||||
await Promise.all(children.map(c => this.kill(c, 'SIGTERM')));
|
||||
}
|
||||
getFree(id) {
|
||||
return (this.free[id] = this.free[id] || []);
|
||||
}
|
||||
getAllFree() {
|
||||
return Object.values(this.free).reduce((first, second) => first.concat(second), []);
|
||||
}
|
||||
}
|
||||
exports.ChildPool = ChildPool;
|
||||
//# sourceMappingURL=child-pool.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"child-pool.js","sourceRoot":"","sources":["../../../src/classes/child-pool.ts"],"names":[],"mappings":";;;AAAA,6BAA6B;AAC7B,mCAAgC;AAEhC,MAAM,kBAAkB,GAAG,KAAM,CAAC;AAOlC,MAAa,SAAS;IAKpB,YAAY,EACV,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,0BAA0B,CAAC,EAC/D,gBAAgB,GACF;QAPhB,aAAQ,GAA6B,EAAE,CAAC;QACxC,SAAI,GAA+B,EAAE,CAAC;QAOpC,IAAI,CAAC,IAAI,GAAG,EAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,WAAmB;QAC9B,IAAI,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,GAAG,EAAE,CAAC;QAE5C,IAAI,KAAK,EAAE;YACT,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;YACjC,OAAO,KAAK,CAAC;SACd;QAED,KAAK,GAAG,IAAI,aAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE;YACjD,gBAAgB,EAAE,IAAI,CAAC,IAAI,CAAC,gBAAgB;SAC7C,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;QAEhD,IAAI;YACF,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;YAEjC,OAAO,KAAK,CAAC;SACd;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACnB,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,GAAG,CAAC;SACX;IACH,CAAC;IAED,OAAO,CAAC,KAAY;QAClB,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,CAAC,KAAY;QACjB,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAEhC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QAE7C,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACvC,IAAI,UAAU,GAAG,CAAC,CAAC,EAAE;YACnB,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;SAC5B;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CACR,KAAY,EACZ,SAAgC,SAAS;QAEzC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QACxE,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;QACnB,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC;QAEf,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC;IAChE,CAAC;IAED,OAAO,CAAC,EAAU;QAChB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;IAC/C,CAAC;IAED,UAAU;QACR,OAAO,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,CACpC,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,EACvC,EAAE,CACH,CAAC;IACJ,CAAC;CACF;AA/ED,8BA+EC"}
|
||||
+152
@@ -0,0 +1,152 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.ChildProcessor = void 0;
|
||||
const enums_1 = require("../enums");
|
||||
const utils_1 = require("../utils");
|
||||
var ChildStatus;
|
||||
(function (ChildStatus) {
|
||||
ChildStatus[ChildStatus["Idle"] = 0] = "Idle";
|
||||
ChildStatus[ChildStatus["Started"] = 1] = "Started";
|
||||
ChildStatus[ChildStatus["Terminating"] = 2] = "Terminating";
|
||||
ChildStatus[ChildStatus["Errored"] = 3] = "Errored";
|
||||
})(ChildStatus || (ChildStatus = {}));
|
||||
/**
|
||||
* ChildProcessor
|
||||
*
|
||||
* This class acts as the interface between a child process and it parent process
|
||||
* so that jobs can be processed in different processes.
|
||||
*
|
||||
*/
|
||||
class ChildProcessor {
|
||||
constructor(send) {
|
||||
this.send = send;
|
||||
}
|
||||
async init(processorFile) {
|
||||
let processor;
|
||||
try {
|
||||
const { default: processorFn } = await import(processorFile);
|
||||
processor = processorFn;
|
||||
if (processor.default) {
|
||||
// support es2015 module.
|
||||
processor = processor.default;
|
||||
}
|
||||
if (typeof processor !== 'function') {
|
||||
throw new Error('No function is exported in processor file');
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
this.status = ChildStatus.Errored;
|
||||
return this.send({
|
||||
cmd: enums_1.ParentCommand.InitFailed,
|
||||
err: (0, utils_1.errorToJSON)(err),
|
||||
});
|
||||
}
|
||||
const origProcessor = processor;
|
||||
processor = function (job, token) {
|
||||
try {
|
||||
return Promise.resolve(origProcessor(job, token));
|
||||
}
|
||||
catch (err) {
|
||||
return Promise.reject(err);
|
||||
}
|
||||
};
|
||||
this.processor = processor;
|
||||
this.status = ChildStatus.Idle;
|
||||
await this.send({
|
||||
cmd: enums_1.ParentCommand.InitCompleted,
|
||||
});
|
||||
}
|
||||
async start(jobJson, token) {
|
||||
if (this.status !== ChildStatus.Idle) {
|
||||
return this.send({
|
||||
cmd: enums_1.ParentCommand.Error,
|
||||
err: (0, utils_1.errorToJSON)(new Error('cannot start a not idling child process')),
|
||||
});
|
||||
}
|
||||
this.status = ChildStatus.Started;
|
||||
this.currentJobPromise = (async () => {
|
||||
try {
|
||||
const job = this.wrapJob(jobJson, this.send);
|
||||
const result = await this.processor(job, token);
|
||||
await this.send({
|
||||
cmd: enums_1.ParentCommand.Completed,
|
||||
value: typeof result === 'undefined' ? null : result,
|
||||
});
|
||||
}
|
||||
catch (err) {
|
||||
await this.send({
|
||||
cmd: enums_1.ParentCommand.Failed,
|
||||
value: (0, utils_1.errorToJSON)(!err.message ? new Error(err) : err),
|
||||
});
|
||||
}
|
||||
finally {
|
||||
this.status = ChildStatus.Idle;
|
||||
this.currentJobPromise = undefined;
|
||||
}
|
||||
})();
|
||||
}
|
||||
async stop() { }
|
||||
async waitForCurrentJobAndExit() {
|
||||
this.status = ChildStatus.Terminating;
|
||||
try {
|
||||
await this.currentJobPromise;
|
||||
}
|
||||
finally {
|
||||
process.exit(process.exitCode || 0);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Enhance the given job argument with some functions
|
||||
* that can be called from the sandboxed job processor.
|
||||
*
|
||||
* Note, the `job` argument is a JSON deserialized message
|
||||
* from the main node process to this forked child process,
|
||||
* the functions on the original job object are not in tact.
|
||||
* The wrapped job adds back some of those original functions.
|
||||
*/
|
||||
wrapJob(job, send) {
|
||||
return Object.assign(Object.assign({}, job), { data: JSON.parse(job.data || '{}'), opts: job.opts, returnValue: JSON.parse(job.returnvalue || '{}'),
|
||||
/*
|
||||
* Emulate the real job `updateProgress` function, should works as `progress` function.
|
||||
*/
|
||||
async updateProgress(progress) {
|
||||
// Locally store reference to new progress value
|
||||
// so that we can return it from this process synchronously.
|
||||
this.progress = progress;
|
||||
// Send message to update job progress.
|
||||
await send({
|
||||
cmd: enums_1.ParentCommand.Progress,
|
||||
value: progress,
|
||||
});
|
||||
},
|
||||
/*
|
||||
* Emulate the real job `log` function.
|
||||
*/
|
||||
log: async (row) => {
|
||||
send({
|
||||
cmd: enums_1.ParentCommand.Log,
|
||||
value: row,
|
||||
});
|
||||
},
|
||||
/*
|
||||
* Emulate the real job `moveToDelayed` function.
|
||||
*/
|
||||
moveToDelayed: async (timestamp, token) => {
|
||||
send({
|
||||
cmd: enums_1.ParentCommand.MoveToDelayed,
|
||||
value: { timestamp, token },
|
||||
});
|
||||
},
|
||||
/*
|
||||
* Emulate the real job `updateData` function.
|
||||
*/
|
||||
updateData: async (data) => {
|
||||
send({
|
||||
cmd: enums_1.ParentCommand.Update,
|
||||
value: data,
|
||||
});
|
||||
} });
|
||||
}
|
||||
}
|
||||
exports.ChildProcessor = ChildProcessor;
|
||||
//# sourceMappingURL=child-processor.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"child-processor.js","sourceRoot":"","sources":["../../../src/classes/child-processor.ts"],"names":[],"mappings":";;;AAAA,oCAAyC;AAEzC,oCAAuC;AAEvC,IAAK,WAKJ;AALD,WAAK,WAAW;IACd,6CAAI,CAAA;IACJ,mDAAO,CAAA;IACP,2DAAW,CAAA;IACX,mDAAO,CAAA;AACT,CAAC,EALI,WAAW,KAAX,WAAW,QAKf;AAED;;;;;;GAMG;AACH,MAAa,cAAc;IAKzB,YAAoB,IAAiC;QAAjC,SAAI,GAAJ,IAAI,CAA6B;IAAG,CAAC;IAElD,KAAK,CAAC,IAAI,CAAC,aAAqB;QACrC,IAAI,SAAS,CAAC;QACd,IAAI;YACF,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;YAC7D,SAAS,GAAG,WAAW,CAAC;YAExB,IAAI,SAAS,CAAC,OAAO,EAAE;gBACrB,yBAAyB;gBACzB,SAAS,GAAG,SAAS,CAAC,OAAO,CAAC;aAC/B;YAED,IAAI,OAAO,SAAS,KAAK,UAAU,EAAE;gBACnC,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;aAC9D;SACF;QAAC,OAAO,GAAG,EAAE;YACZ,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC;YAClC,OAAO,IAAI,CAAC,IAAI,CAAC;gBACf,GAAG,EAAE,qBAAa,CAAC,UAAU;gBAC7B,GAAG,EAAE,IAAA,mBAAW,EAAC,GAAG,CAAC;aACtB,CAAC,CAAC;SACJ;QAED,MAAM,aAAa,GAAG,SAAS,CAAC;QAChC,SAAS,GAAG,UAAU,GAAiB,EAAE,KAAc;YACrD,IAAI;gBACF,OAAO,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;aACnD;YAAC,OAAO,GAAG,EAAE;gBACZ,OAAO,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;aAC5B;QACH,CAAC,CAAC;QAEF,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC;QAC/B,MAAM,IAAI,CAAC,IAAI,CAAC;YACd,GAAG,EAAE,qBAAa,CAAC,aAAa;SACjC,CAAC,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,KAAK,CAAC,OAAgB,EAAE,KAAc;QACjD,IAAI,IAAI,CAAC,MAAM,KAAK,WAAW,CAAC,IAAI,EAAE;YACpC,OAAO,IAAI,CAAC,IAAI,CAAC;gBACf,GAAG,EAAE,qBAAa,CAAC,KAAK;gBACxB,GAAG,EAAE,IAAA,mBAAW,EAAC,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;aACvE,CAAC,CAAC;SACJ;QACD,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC;QAClC,IAAI,CAAC,iBAAiB,GAAG,CAAC,KAAK,IAAI,EAAE;YACnC,IAAI;gBACF,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC7C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;gBAChD,MAAM,IAAI,CAAC,IAAI,CAAC;oBACd,GAAG,EAAE,qBAAa,CAAC,SAAS;oBAC5B,KAAK,EAAE,OAAO,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM;iBACrD,CAAC,CAAC;aACJ;YAAC,OAAO,GAAG,EAAE;gBACZ,MAAM,IAAI,CAAC,IAAI,CAAC;oBACd,GAAG,EAAE,qBAAa,CAAC,MAAM;oBACzB,KAAK,EAAE,IAAA,mBAAW,EAAC,CAAS,GAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,KAAK,CAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;iBACtE,CAAC,CAAC;aACJ;oBAAS;gBACR,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC;gBAC/B,IAAI,CAAC,iBAAiB,GAAG,SAAS,CAAC;aACpC;QACH,CAAC,CAAC,EAAE,CAAC;IACP,CAAC;IAEM,KAAK,CAAC,IAAI,KAAmB,CAAC;IAErC,KAAK,CAAC,wBAAwB;QAC5B,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,WAAW,CAAC;QACtC,IAAI;YACF,MAAM,IAAI,CAAC,iBAAiB,CAAC;SAC9B;gBAAS;YACR,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;SACrC;IACH,CAAC;IAED;;;;;;;;OAQG;IACO,OAAO,CACf,GAAY,EACZ,IAAiC;QAEjC,uCACK,GAAG,KACN,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,IAAI,IAAI,CAAC,EAClC,IAAI,EAAE,GAAG,CAAC,IAAI,EACd,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,IAAI,IAAI,CAAC;YAChD;;eAEG;YACH,KAAK,CAAC,cAAc,CAAC,QAAyB;gBAC5C,gDAAgD;gBAChD,4DAA4D;gBAC5D,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;gBACzB,uCAAuC;gBACvC,MAAM,IAAI,CAAC;oBACT,GAAG,EAAE,qBAAa,CAAC,QAAQ;oBAC3B,KAAK,EAAE,QAAQ;iBAChB,CAAC,CAAC;YACL,CAAC;YACD;;eAEG;YACH,GAAG,EAAE,KAAK,EAAE,GAAQ,EAAE,EAAE;gBACtB,IAAI,CAAC;oBACH,GAAG,EAAE,qBAAa,CAAC,GAAG;oBACtB,KAAK,EAAE,GAAG;iBACX,CAAC,CAAC;YACL,CAAC;YACD;;eAEG;YACH,aAAa,EAAE,KAAK,EAAE,SAAiB,EAAE,KAAc,EAAE,EAAE;gBACzD,IAAI,CAAC;oBACH,GAAG,EAAE,qBAAa,CAAC,aAAa;oBAChC,KAAK,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE;iBAC5B,CAAC,CAAC;YACL,CAAC;YACD;;eAEG;YACH,UAAU,EAAE,KAAK,EAAE,IAAS,EAAE,EAAE;gBAC9B,IAAI,CAAC;oBACH,GAAG,EAAE,qBAAa,CAAC,MAAM;oBACzB,KAAK,EAAE,IAAI;iBACZ,CAAC,CAAC;YACL,CAAC,IACD;IACJ,CAAC;CACF;AAhJD,wCAgJC"}
|
||||
+216
@@ -0,0 +1,216 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.Child = void 0;
|
||||
const child_process_1 = require("child_process");
|
||||
const worker_threads_1 = require("worker_threads");
|
||||
const net_1 = require("net");
|
||||
const enums_1 = require("../enums");
|
||||
const events_1 = require("events");
|
||||
/**
|
||||
* @see https://nodejs.org/api/process.html#process_exit_codes
|
||||
*/
|
||||
const exitCodesErrors = {
|
||||
1: 'Uncaught Fatal Exception',
|
||||
2: 'Unused',
|
||||
3: 'Internal JavaScript Parse Error',
|
||||
4: 'Internal JavaScript Evaluation Failure',
|
||||
5: 'Fatal Error',
|
||||
6: 'Non-function Internal Exception Handler',
|
||||
7: 'Internal Exception Handler Run-Time Failure',
|
||||
8: 'Unused',
|
||||
9: 'Invalid Argument',
|
||||
10: 'Internal JavaScript Run-Time Failure',
|
||||
12: 'Invalid Debug Argument',
|
||||
13: 'Unfinished Top-Level Await',
|
||||
};
|
||||
/**
|
||||
* Child class
|
||||
*
|
||||
* This class is used to create a child process or worker thread, and allows using
|
||||
* isolated processes or threads for processing jobs.
|
||||
*
|
||||
*/
|
||||
class Child extends events_1.EventEmitter {
|
||||
constructor(mainFile, processFile, opts = {
|
||||
useWorkerThreads: false,
|
||||
}) {
|
||||
super();
|
||||
this.mainFile = mainFile;
|
||||
this.processFile = processFile;
|
||||
this.opts = opts;
|
||||
this._exitCode = null;
|
||||
this._signalCode = null;
|
||||
this._killed = false;
|
||||
}
|
||||
get pid() {
|
||||
if (this.childProcess) {
|
||||
return this.childProcess.pid;
|
||||
}
|
||||
else if (this.worker) {
|
||||
return this.worker.threadId;
|
||||
}
|
||||
else {
|
||||
throw new Error('No child process or worker thread');
|
||||
}
|
||||
}
|
||||
get exitCode() {
|
||||
return this._exitCode;
|
||||
}
|
||||
get signalCode() {
|
||||
return this._signalCode;
|
||||
}
|
||||
get killed() {
|
||||
if (this.childProcess) {
|
||||
return this.childProcess.killed;
|
||||
}
|
||||
return this._killed;
|
||||
}
|
||||
async init() {
|
||||
const execArgv = await convertExecArgv(process.execArgv);
|
||||
let parent;
|
||||
if (this.opts.useWorkerThreads) {
|
||||
this.worker = parent = new worker_threads_1.Worker(this.mainFile, {
|
||||
execArgv,
|
||||
stdin: true,
|
||||
stdout: true,
|
||||
stderr: true,
|
||||
});
|
||||
}
|
||||
else {
|
||||
this.childProcess = parent = (0, child_process_1.fork)(this.mainFile, [], {
|
||||
execArgv,
|
||||
stdio: 'pipe',
|
||||
});
|
||||
}
|
||||
parent.on('exit', (exitCode, signalCode) => {
|
||||
this._exitCode = exitCode;
|
||||
// Coerce to null if undefined for backwards compatibility
|
||||
signalCode = typeof signalCode === 'undefined' ? null : signalCode;
|
||||
this._signalCode = signalCode;
|
||||
this._killed = true;
|
||||
this.emit('exit', exitCode, signalCode);
|
||||
// Clean all listeners, we do not expect any more events after "exit"
|
||||
parent.removeAllListeners();
|
||||
this.removeAllListeners();
|
||||
});
|
||||
parent.on('error', (...args) => this.emit('error', ...args));
|
||||
parent.on('message', (...args) => this.emit('message', ...args));
|
||||
parent.on('close', (...args) => this.emit('close', ...args));
|
||||
parent.stdout.pipe(process.stdout);
|
||||
parent.stderr.pipe(process.stderr);
|
||||
await this.initChild();
|
||||
}
|
||||
async send(msg) {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (this.childProcess) {
|
||||
this.childProcess.send(msg, (err) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
}
|
||||
else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
}
|
||||
else if (this.worker) {
|
||||
resolve(this.worker.postMessage(msg));
|
||||
}
|
||||
else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
}
|
||||
killProcess(signal = 'SIGKILL') {
|
||||
if (this.childProcess) {
|
||||
this.childProcess.kill(signal);
|
||||
}
|
||||
else if (this.worker) {
|
||||
this.worker.terminate();
|
||||
}
|
||||
}
|
||||
async kill(signal = 'SIGKILL', timeoutMs) {
|
||||
if (this.hasProcessExited()) {
|
||||
return;
|
||||
}
|
||||
const onExit = onExitOnce(this.childProcess || this.worker);
|
||||
this.killProcess(signal);
|
||||
if (timeoutMs !== undefined && (timeoutMs === 0 || isFinite(timeoutMs))) {
|
||||
const timeoutHandle = setTimeout(() => {
|
||||
if (!this.hasProcessExited()) {
|
||||
this.killProcess('SIGKILL');
|
||||
}
|
||||
}, timeoutMs);
|
||||
await onExit;
|
||||
clearTimeout(timeoutHandle);
|
||||
}
|
||||
await onExit;
|
||||
}
|
||||
async initChild() {
|
||||
const onComplete = new Promise((resolve, reject) => {
|
||||
const onMessageHandler = (msg) => {
|
||||
if (msg.cmd === enums_1.ParentCommand.InitCompleted) {
|
||||
resolve();
|
||||
}
|
||||
else if (msg.cmd === enums_1.ParentCommand.InitFailed) {
|
||||
const err = new Error();
|
||||
err.stack = msg.err.stack;
|
||||
err.message = msg.err.message;
|
||||
reject(err);
|
||||
}
|
||||
this.off('message', onMessageHandler);
|
||||
this.off('close', onCloseHandler);
|
||||
};
|
||||
const onCloseHandler = (code, signal) => {
|
||||
if (code > 128) {
|
||||
code -= 128;
|
||||
}
|
||||
const msg = exitCodesErrors[code] || `Unknown exit code ${code}`;
|
||||
reject(new Error(`Error initializing child: ${msg} and signal ${signal}`));
|
||||
this.off('message', onMessageHandler);
|
||||
this.off('close', onCloseHandler);
|
||||
};
|
||||
this.on('message', onMessageHandler);
|
||||
this.on('close', onCloseHandler);
|
||||
});
|
||||
await this.send({
|
||||
cmd: enums_1.ChildCommand.Init,
|
||||
value: this.processFile,
|
||||
});
|
||||
await onComplete;
|
||||
}
|
||||
hasProcessExited() {
|
||||
return !!(this.exitCode !== null || this.signalCode);
|
||||
}
|
||||
}
|
||||
exports.Child = Child;
|
||||
function onExitOnce(child) {
|
||||
return new Promise(resolve => {
|
||||
child.once('exit', () => resolve());
|
||||
});
|
||||
}
|
||||
const getFreePort = async () => {
|
||||
return new Promise(resolve => {
|
||||
const server = (0, net_1.createServer)();
|
||||
server.listen(0, () => {
|
||||
const { port } = server.address();
|
||||
server.close(() => resolve(port));
|
||||
});
|
||||
});
|
||||
};
|
||||
const convertExecArgv = async (execArgv) => {
|
||||
const standard = [];
|
||||
const convertedArgs = [];
|
||||
for (let i = 0; i < execArgv.length; i++) {
|
||||
const arg = execArgv[i];
|
||||
if (arg.indexOf('--inspect') === -1) {
|
||||
standard.push(arg);
|
||||
}
|
||||
else {
|
||||
const argName = arg.split('=')[0];
|
||||
const port = await getFreePort();
|
||||
convertedArgs.push(`${argName}=${port}`);
|
||||
}
|
||||
}
|
||||
return standard.concat(convertedArgs);
|
||||
};
|
||||
//# sourceMappingURL=child.js.map
|
||||
+1
File diff suppressed because one or more lines are too long
+19
@@ -0,0 +1,19 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.DelayedError = void 0;
|
||||
/**
|
||||
* DelayedError
|
||||
*
|
||||
* Error to be thrown when job is moved to delayed state
|
||||
* from job in active state.
|
||||
*
|
||||
*/
|
||||
class DelayedError extends Error {
|
||||
constructor(message) {
|
||||
super(message);
|
||||
this.name = this.constructor.name;
|
||||
Object.setPrototypeOf(this, new.target.prototype);
|
||||
}
|
||||
}
|
||||
exports.DelayedError = DelayedError;
|
||||
//# sourceMappingURL=delayed-error.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"delayed-error.js","sourceRoot":"","sources":["../../../../src/classes/errors/delayed-error.ts"],"names":[],"mappings":";;;AAAA;;;;;;GAMG;AACH,MAAa,YAAa,SAAQ,KAAK;IACrC,YAAY,OAAgB;QAC1B,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;QAClC,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACpD,CAAC;CACF;AAND,oCAMC"}
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
const tslib_1 = require("tslib");
|
||||
tslib_1.__exportStar(require("./delayed-error"), exports);
|
||||
tslib_1.__exportStar(require("./unrecoverable-error"), exports);
|
||||
tslib_1.__exportStar(require("./rate-limit-error"), exports);
|
||||
tslib_1.__exportStar(require("./waiting-children-error"), exports);
|
||||
//# sourceMappingURL=index.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/classes/errors/index.ts"],"names":[],"mappings":";;;AAAA,0DAAgC;AAChC,gEAAsC;AACtC,6DAAmC;AACnC,mEAAyC"}
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.RateLimitError = exports.RATE_LIMIT_ERROR = void 0;
|
||||
exports.RATE_LIMIT_ERROR = 'bullmq:rateLimitExceeded';
|
||||
/**
|
||||
* RateLimitError
|
||||
*
|
||||
* Error to be thrown when queue reaches a rate limit.
|
||||
*
|
||||
*/
|
||||
class RateLimitError extends Error {
|
||||
constructor(message = exports.RATE_LIMIT_ERROR) {
|
||||
super(message);
|
||||
this.name = this.constructor.name;
|
||||
Object.setPrototypeOf(this, new.target.prototype);
|
||||
}
|
||||
}
|
||||
exports.RateLimitError = RateLimitError;
|
||||
//# sourceMappingURL=rate-limit-error.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"rate-limit-error.js","sourceRoot":"","sources":["../../../../src/classes/errors/rate-limit-error.ts"],"names":[],"mappings":";;;AAAa,QAAA,gBAAgB,GAAG,0BAA0B,CAAC;AAE3D;;;;;GAKG;AACH,MAAa,cAAe,SAAQ,KAAK;IACvC,YAAY,UAAkB,wBAAgB;QAC5C,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;QAClC,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACpD,CAAC;CACF;AAND,wCAMC"}
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.UnrecoverableError = void 0;
|
||||
/**
|
||||
* UnrecoverableError
|
||||
*
|
||||
* Error to move a job to failed even if the attemptsMade
|
||||
* are lower than the expected limit.
|
||||
*
|
||||
*/
|
||||
class UnrecoverableError extends Error {
|
||||
constructor(message) {
|
||||
super(message);
|
||||
this.name = this.constructor.name;
|
||||
Object.setPrototypeOf(this, new.target.prototype);
|
||||
}
|
||||
}
|
||||
exports.UnrecoverableError = UnrecoverableError;
|
||||
//# sourceMappingURL=unrecoverable-error.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"unrecoverable-error.js","sourceRoot":"","sources":["../../../../src/classes/errors/unrecoverable-error.ts"],"names":[],"mappings":";;;AAAA;;;;;;GAMG;AACH,MAAa,kBAAmB,SAAQ,KAAK;IAC3C,YAAY,OAAgB;QAC1B,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;QAClC,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACpD,CAAC;CACF;AAND,gDAMC"}
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.WaitingChildrenError = void 0;
|
||||
/**
|
||||
* WaitingChildrenError
|
||||
*
|
||||
* Error to be thrown when job is moved to waiting-children state
|
||||
* from job in active state.
|
||||
*
|
||||
*/
|
||||
class WaitingChildrenError extends Error {
|
||||
constructor(message) {
|
||||
super(message);
|
||||
this.name = this.constructor.name;
|
||||
Object.setPrototypeOf(this, new.target.prototype);
|
||||
}
|
||||
}
|
||||
exports.WaitingChildrenError = WaitingChildrenError;
|
||||
//# sourceMappingURL=waiting-children-error.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"waiting-children-error.js","sourceRoot":"","sources":["../../../../src/classes/errors/waiting-children-error.ts"],"names":[],"mappings":";;;AAAA;;;;;;GAMG;AACH,MAAa,oBAAqB,SAAQ,KAAK;IAC7C,YAAY,OAAgB;QAC1B,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;QAClC,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACpD,CAAC;CACF;AAND,oDAMC"}
|
||||
+309
@@ -0,0 +1,309 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.FlowProducer = void 0;
|
||||
const events_1 = require("events");
|
||||
const lodash_1 = require("lodash");
|
||||
const uuid_1 = require("uuid");
|
||||
const utils_1 = require("../utils");
|
||||
const job_1 = require("./job");
|
||||
const queue_keys_1 = require("./queue-keys");
|
||||
const redis_connection_1 = require("./redis-connection");
|
||||
/**
|
||||
* This class allows to add jobs with dependencies between them in such
|
||||
* a way that it is possible to build complex flows.
|
||||
* Note: A flow is a tree-like structure of jobs that depend on each other.
|
||||
* Whenever the children of a given parent are completed, the parent
|
||||
* will be processed, being able to access the children's result data.
|
||||
* All Jobs can be in different queues, either children or parents,
|
||||
*/
|
||||
class FlowProducer extends events_1.EventEmitter {
|
||||
constructor(opts = {}, Connection = redis_connection_1.RedisConnection) {
|
||||
super();
|
||||
this.opts = opts;
|
||||
this.opts = Object.assign({ prefix: 'bull' }, opts);
|
||||
if (!opts.connection) {
|
||||
console.warn([
|
||||
'BullMQ: DEPRECATION WARNING! Optional instantiation of Queue, Worker, QueueEvents and FlowProducer',
|
||||
'without providing explicitly a connection or connection options is deprecated. This behaviour will',
|
||||
'be removed in the next major release',
|
||||
].join(' '));
|
||||
}
|
||||
this.connection = new Connection(opts.connection, (0, utils_1.isRedisInstance)(opts === null || opts === void 0 ? void 0 : opts.connection), false, opts.skipVersionCheck);
|
||||
this.connection.on('error', (error) => this.emit('error', error));
|
||||
this.connection.on('close', () => {
|
||||
if (!this.closing) {
|
||||
this.emit('ioredis:close');
|
||||
}
|
||||
});
|
||||
this.queueKeys = new queue_keys_1.QueueKeys(opts.prefix);
|
||||
}
|
||||
emit(event, ...args) {
|
||||
return super.emit(event, ...args);
|
||||
}
|
||||
off(eventName, listener) {
|
||||
super.off(eventName, listener);
|
||||
return this;
|
||||
}
|
||||
on(event, listener) {
|
||||
super.on(event, listener);
|
||||
return this;
|
||||
}
|
||||
once(event, listener) {
|
||||
super.once(event, listener);
|
||||
return this;
|
||||
}
|
||||
/**
|
||||
* Returns a promise that resolves to a redis client. Normally used only by subclasses.
|
||||
*/
|
||||
get client() {
|
||||
return this.connection.client;
|
||||
}
|
||||
/**
|
||||
* Helper to easily extend Job class calls.
|
||||
*/
|
||||
get Job() {
|
||||
return job_1.Job;
|
||||
}
|
||||
waitUntilReady() {
|
||||
return this.client;
|
||||
}
|
||||
/**
|
||||
* Adds a flow.
|
||||
*
|
||||
* This call would be atomic, either it fails and no jobs will
|
||||
* be added to the queues, or it succeeds and all jobs will be added.
|
||||
*
|
||||
* @param flow - an object with a tree-like structure where children jobs
|
||||
* will be processed before their parents.
|
||||
* @param opts - options that will be applied to the flow object.
|
||||
*/
|
||||
async add(flow, opts) {
|
||||
var _a;
|
||||
if (this.closing) {
|
||||
return;
|
||||
}
|
||||
const client = await this.connection.client;
|
||||
const multi = client.multi();
|
||||
const parentOpts = (_a = flow === null || flow === void 0 ? void 0 : flow.opts) === null || _a === void 0 ? void 0 : _a.parent;
|
||||
const parentKey = (0, utils_1.getParentKey)(parentOpts);
|
||||
const parentDependenciesKey = parentKey
|
||||
? `${parentKey}:dependencies`
|
||||
: undefined;
|
||||
const jobsTree = this.addNode({
|
||||
multi,
|
||||
node: flow,
|
||||
queuesOpts: opts === null || opts === void 0 ? void 0 : opts.queuesOptions,
|
||||
parent: {
|
||||
parentOpts,
|
||||
parentDependenciesKey,
|
||||
},
|
||||
});
|
||||
await multi.exec();
|
||||
return jobsTree;
|
||||
}
|
||||
/**
|
||||
* Get a flow.
|
||||
*
|
||||
* @param opts - an object with options for getting a JobNode.
|
||||
*/
|
||||
async getFlow(opts) {
|
||||
if (this.closing) {
|
||||
return;
|
||||
}
|
||||
const client = await this.connection.client;
|
||||
const updatedOpts = Object.assign({
|
||||
depth: 10,
|
||||
maxChildren: 20,
|
||||
}, opts);
|
||||
const jobsTree = this.getNode(client, updatedOpts);
|
||||
return jobsTree;
|
||||
}
|
||||
/**
|
||||
* Adds multiple flows.
|
||||
*
|
||||
* A flow is a tree-like structure of jobs that depend on each other.
|
||||
* Whenever the children of a given parent are completed, the parent
|
||||
* will be processed, being able to access the children's result data.
|
||||
*
|
||||
* All Jobs can be in different queues, either children or parents,
|
||||
* however this call would be atomic, either it fails and no jobs will
|
||||
* be added to the queues, or it succeeds and all jobs will be added.
|
||||
*
|
||||
* @param flows - an array of objects with a tree-like structure where children jobs
|
||||
* will be processed before their parents.
|
||||
*/
|
||||
async addBulk(flows) {
|
||||
if (this.closing) {
|
||||
return;
|
||||
}
|
||||
const client = await this.connection.client;
|
||||
const multi = client.multi();
|
||||
const jobsTrees = this.addNodes(multi, flows);
|
||||
await multi.exec();
|
||||
return jobsTrees;
|
||||
}
|
||||
/**
|
||||
* Add a node (job) of a flow to the queue. This method will recursively
|
||||
* add all its children as well. Note that a given job can potentially be
|
||||
* a parent and a child job at the same time depending on where it is located
|
||||
* in the tree hierarchy.
|
||||
*
|
||||
* @param multi - ioredis ChainableCommander
|
||||
* @param node - the node representing a job to be added to some queue
|
||||
* @param parent - parent data sent to children to create the "links" to their parent
|
||||
* @returns
|
||||
*/
|
||||
addNode({ multi, node, parent, queuesOpts }) {
|
||||
var _a;
|
||||
const prefix = node.prefix || this.opts.prefix;
|
||||
const queue = this.queueFromNode(node, new queue_keys_1.QueueKeys(prefix), prefix);
|
||||
const queueOpts = queuesOpts && queuesOpts[node.queueName];
|
||||
const jobsOpts = (0, lodash_1.get)(queueOpts, 'defaultJobOptions');
|
||||
const jobId = ((_a = node.opts) === null || _a === void 0 ? void 0 : _a.jobId) || (0, uuid_1.v4)();
|
||||
const job = new this.Job(queue, node.name, node.data, Object.assign(Object.assign(Object.assign({}, (jobsOpts ? jobsOpts : {})), node.opts), { parent: parent === null || parent === void 0 ? void 0 : parent.parentOpts }), jobId);
|
||||
const parentKey = (0, utils_1.getParentKey)(parent === null || parent === void 0 ? void 0 : parent.parentOpts);
|
||||
if (node.children && node.children.length > 0) {
|
||||
// Create parent job, will be a job in status "waiting-children".
|
||||
const parentId = jobId;
|
||||
const queueKeysParent = new queue_keys_1.QueueKeys(node.prefix || this.opts.prefix);
|
||||
const waitChildrenKey = queueKeysParent.toKey(node.queueName, 'waiting-children');
|
||||
job.addJob(multi, {
|
||||
parentDependenciesKey: parent === null || parent === void 0 ? void 0 : parent.parentDependenciesKey,
|
||||
waitChildrenKey,
|
||||
parentKey,
|
||||
});
|
||||
const parentDependenciesKey = `${queueKeysParent.toKey(node.queueName, parentId)}:dependencies`;
|
||||
const children = this.addChildren({
|
||||
multi,
|
||||
nodes: node.children,
|
||||
parent: {
|
||||
parentOpts: {
|
||||
id: parentId,
|
||||
queue: queueKeysParent.getQueueQualifiedName(node.queueName),
|
||||
},
|
||||
parentDependenciesKey,
|
||||
},
|
||||
queuesOpts,
|
||||
});
|
||||
return { job, children };
|
||||
}
|
||||
else {
|
||||
job.addJob(multi, {
|
||||
parentDependenciesKey: parent === null || parent === void 0 ? void 0 : parent.parentDependenciesKey,
|
||||
parentKey,
|
||||
});
|
||||
return { job };
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Adds nodes (jobs) of multiple flows to the queue. This method will recursively
|
||||
* add all its children as well. Note that a given job can potentially be
|
||||
* a parent and a child job at the same time depending on where it is located
|
||||
* in the tree hierarchy.
|
||||
*
|
||||
* @param multi - ioredis ChainableCommander
|
||||
* @param nodes - the nodes representing jobs to be added to some queue
|
||||
* @returns
|
||||
*/
|
||||
addNodes(multi, nodes) {
|
||||
return nodes.map(node => {
|
||||
var _a;
|
||||
const parentOpts = (_a = node === null || node === void 0 ? void 0 : node.opts) === null || _a === void 0 ? void 0 : _a.parent;
|
||||
const parentKey = (0, utils_1.getParentKey)(parentOpts);
|
||||
const parentDependenciesKey = parentKey
|
||||
? `${parentKey}:dependencies`
|
||||
: undefined;
|
||||
return this.addNode({
|
||||
multi,
|
||||
node,
|
||||
parent: {
|
||||
parentOpts,
|
||||
parentDependenciesKey,
|
||||
},
|
||||
});
|
||||
});
|
||||
}
|
||||
async getNode(client, node) {
|
||||
const queue = this.queueFromNode(node, new queue_keys_1.QueueKeys(node.prefix), node.prefix);
|
||||
const job = await this.Job.fromId(queue, node.id);
|
||||
if (job) {
|
||||
const { processed = {}, unprocessed = [] } = await job.getDependencies({
|
||||
processed: {
|
||||
count: node.maxChildren,
|
||||
},
|
||||
unprocessed: {
|
||||
count: node.maxChildren,
|
||||
},
|
||||
});
|
||||
const processedKeys = Object.keys(processed);
|
||||
const childrenCount = processedKeys.length + unprocessed.length;
|
||||
const newDepth = node.depth - 1;
|
||||
if (childrenCount > 0 && newDepth) {
|
||||
const children = await this.getChildren(client, [...processedKeys, ...unprocessed], newDepth, node.maxChildren);
|
||||
return { job, children };
|
||||
}
|
||||
else {
|
||||
return { job };
|
||||
}
|
||||
}
|
||||
}
|
||||
addChildren({ multi, nodes, parent, queuesOpts }) {
|
||||
return nodes.map(node => this.addNode({ multi, node, parent, queuesOpts }));
|
||||
}
|
||||
getChildren(client, childrenKeys, depth, maxChildren) {
|
||||
const getChild = (key) => {
|
||||
const [prefix, queueName, id] = key.split(':');
|
||||
return this.getNode(client, {
|
||||
id,
|
||||
queueName,
|
||||
prefix,
|
||||
depth,
|
||||
maxChildren,
|
||||
});
|
||||
};
|
||||
return Promise.all([...childrenKeys.map(getChild)]);
|
||||
}
|
||||
/**
|
||||
* Helper factory method that creates a queue-like object
|
||||
* required to create jobs in any queue.
|
||||
*
|
||||
* @param node -
|
||||
* @param queueKeys -
|
||||
* @returns
|
||||
*/
|
||||
queueFromNode(node, queueKeys, prefix) {
|
||||
return {
|
||||
client: this.connection.client,
|
||||
name: node.queueName,
|
||||
keys: queueKeys.getKeys(node.queueName),
|
||||
toKey: (type) => queueKeys.toKey(node.queueName, type),
|
||||
opts: { prefix },
|
||||
qualifiedName: queueKeys.getQueueQualifiedName(node.queueName),
|
||||
closing: this.closing,
|
||||
waitUntilReady: async () => this.connection.client,
|
||||
removeListener: this.removeListener.bind(this),
|
||||
emit: this.emit.bind(this),
|
||||
on: this.on.bind(this),
|
||||
redisVersion: this.connection.redisVersion,
|
||||
};
|
||||
}
|
||||
/**
|
||||
*
|
||||
* Closes the connection and returns a promise that resolves when the connection is closed.
|
||||
*/
|
||||
async close() {
|
||||
if (!this.closing) {
|
||||
this.closing = this.connection.close();
|
||||
}
|
||||
await this.closing;
|
||||
}
|
||||
/**
|
||||
*
|
||||
* Force disconnects a connection.
|
||||
*/
|
||||
disconnect() {
|
||||
return this.connection.disconnect();
|
||||
}
|
||||
}
|
||||
exports.FlowProducer = FlowProducer;
|
||||
//# sourceMappingURL=flow-producer.js.map
|
||||
+1
File diff suppressed because one or more lines are too long
+23
@@ -0,0 +1,23 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
const tslib_1 = require("tslib");
|
||||
tslib_1.__exportStar(require("./async-fifo-queue"), exports);
|
||||
tslib_1.__exportStar(require("./backoffs"), exports);
|
||||
tslib_1.__exportStar(require("./child-pool"), exports);
|
||||
tslib_1.__exportStar(require("./child-processor"), exports);
|
||||
tslib_1.__exportStar(require("./errors"), exports);
|
||||
tslib_1.__exportStar(require("./flow-producer"), exports);
|
||||
tslib_1.__exportStar(require("./job"), exports);
|
||||
// export * from './main'; this file must not be exported
|
||||
// export * from './main-worker'; this file must not be exported
|
||||
tslib_1.__exportStar(require("./queue-base"), exports);
|
||||
tslib_1.__exportStar(require("./queue-events"), exports);
|
||||
tslib_1.__exportStar(require("./queue-getters"), exports);
|
||||
tslib_1.__exportStar(require("./queue-keys"), exports);
|
||||
tslib_1.__exportStar(require("./queue"), exports);
|
||||
tslib_1.__exportStar(require("./redis-connection"), exports);
|
||||
tslib_1.__exportStar(require("./repeat"), exports);
|
||||
tslib_1.__exportStar(require("./sandbox"), exports);
|
||||
tslib_1.__exportStar(require("./scripts"), exports);
|
||||
tslib_1.__exportStar(require("./worker"), exports);
|
||||
//# sourceMappingURL=index.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/classes/index.ts"],"names":[],"mappings":";;;AAAA,6DAAmC;AACnC,qDAA2B;AAC3B,uDAA6B;AAC7B,4DAAkC;AAClC,mDAAyB;AACzB,0DAAgC;AAChC,gDAAsB;AACtB,yDAAyD;AACzD,gEAAgE;AAChE,uDAA6B;AAC7B,yDAA+B;AAC/B,0DAAgC;AAChC,uDAA6B;AAC7B,kDAAwB;AACxB,6DAAmC;AACnC,mDAAyB;AACzB,oDAA0B;AAC1B,oDAA0B;AAC1B,mDAAyB"}
|
||||
+791
@@ -0,0 +1,791 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.Job = exports.PRIORITY_LIMIT = void 0;
|
||||
const tslib_1 = require("tslib");
|
||||
const lodash_1 = require("lodash");
|
||||
const util_1 = require("util");
|
||||
const utils_1 = require("../utils");
|
||||
const backoffs_1 = require("./backoffs");
|
||||
const scripts_1 = require("./scripts");
|
||||
const unrecoverable_error_1 = require("./errors/unrecoverable-error");
|
||||
const logger = (0, util_1.debuglog)('bull');
|
||||
const optsDecodeMap = {
|
||||
fpof: 'failParentOnFailure',
|
||||
kl: 'keepLogs',
|
||||
rdof: 'removeDependencyOnFailure',
|
||||
};
|
||||
const optsEncodeMap = (0, lodash_1.invert)(optsDecodeMap);
|
||||
exports.PRIORITY_LIMIT = 2 ** 21;
|
||||
/**
|
||||
* Job
|
||||
*
|
||||
* This class represents a Job in the queue. Normally job are implicitly created when
|
||||
* you add a job to the queue with methods such as Queue.addJob( ... )
|
||||
*
|
||||
* A Job instance is also passed to the Worker's process function.
|
||||
*
|
||||
* @class Job
|
||||
*/
|
||||
class Job {
|
||||
constructor(queue,
|
||||
/**
|
||||
* The name of the Job
|
||||
*/
|
||||
name,
|
||||
/**
|
||||
* The payload for this job.
|
||||
*/
|
||||
data,
|
||||
/**
|
||||
* The options object for this job.
|
||||
*/
|
||||
opts = {}, id) {
|
||||
this.queue = queue;
|
||||
this.name = name;
|
||||
this.data = data;
|
||||
this.opts = opts;
|
||||
this.id = id;
|
||||
/**
|
||||
* The progress a job has performed so far.
|
||||
* @defaultValue 0
|
||||
*/
|
||||
this.progress = 0;
|
||||
/**
|
||||
* The value returned by the processor when processing this job.
|
||||
* @defaultValue null
|
||||
*/
|
||||
this.returnvalue = null;
|
||||
/**
|
||||
* Stacktrace for the error (for failed jobs).
|
||||
* @defaultValue null
|
||||
*/
|
||||
this.stacktrace = null;
|
||||
/**
|
||||
* Number of attempts after the job has failed.
|
||||
* @defaultValue 0
|
||||
*/
|
||||
this.attemptsMade = 0;
|
||||
const _a = this.opts, { repeatJobKey } = _a, restOpts = tslib_1.__rest(_a, ["repeatJobKey"]);
|
||||
this.opts = Object.assign({
|
||||
attempts: 0,
|
||||
delay: 0,
|
||||
}, restOpts);
|
||||
this.delay = this.opts.delay;
|
||||
this.repeatJobKey = repeatJobKey;
|
||||
this.timestamp = opts.timestamp ? opts.timestamp : Date.now();
|
||||
this.opts.backoff = backoffs_1.Backoffs.normalize(opts.backoff);
|
||||
this.parentKey = (0, utils_1.getParentKey)(opts.parent);
|
||||
this.parent = opts.parent
|
||||
? { id: opts.parent.id, queueKey: opts.parent.queue }
|
||||
: undefined;
|
||||
this.toKey = queue.toKey.bind(queue);
|
||||
this.scripts = new scripts_1.Scripts(queue);
|
||||
this.queueQualifiedName = queue.qualifiedName;
|
||||
}
|
||||
/**
|
||||
* Creates a new job and adds it to the queue.
|
||||
*
|
||||
* @param queue - the queue where to add the job.
|
||||
* @param name - the name of the job.
|
||||
* @param data - the payload of the job.
|
||||
* @param opts - the options bag for this job.
|
||||
* @returns
|
||||
*/
|
||||
static async create(queue, name, data, opts) {
|
||||
const client = await queue.client;
|
||||
const job = new this(queue, name, data, opts, opts && opts.jobId);
|
||||
job.id = await job.addJob(client, {
|
||||
parentKey: job.parentKey,
|
||||
parentDependenciesKey: job.parentKey
|
||||
? `${job.parentKey}:dependencies`
|
||||
: '',
|
||||
});
|
||||
return job;
|
||||
}
|
||||
/**
|
||||
* Creates a bulk of jobs and adds them atomically to the given queue.
|
||||
*
|
||||
* @param queue -the queue were to add the jobs.
|
||||
* @param jobs - an array of jobs to be added to the queue.
|
||||
* @returns
|
||||
*/
|
||||
static async createBulk(queue, jobs) {
|
||||
const client = await queue.client;
|
||||
const jobInstances = jobs.map(job => { var _a; return new this(queue, job.name, job.data, job.opts, (_a = job.opts) === null || _a === void 0 ? void 0 : _a.jobId); });
|
||||
const multi = client.multi();
|
||||
for (const job of jobInstances) {
|
||||
job.addJob(multi, {
|
||||
parentKey: job.parentKey,
|
||||
parentDependenciesKey: job.parentKey
|
||||
? `${job.parentKey}:dependencies`
|
||||
: '',
|
||||
});
|
||||
}
|
||||
const results = (await multi.exec());
|
||||
for (let index = 0; index < results.length; ++index) {
|
||||
const [err, id] = results[index];
|
||||
if (err) {
|
||||
throw err;
|
||||
}
|
||||
jobInstances[index].id = id;
|
||||
}
|
||||
return jobInstances;
|
||||
}
|
||||
/**
|
||||
* Instantiates a Job from a JobJsonRaw object (coming from a deserialized JSON object)
|
||||
*
|
||||
* @param queue - the queue where the job belongs to.
|
||||
* @param json - the plain object containing the job.
|
||||
* @param jobId - an optional job id (overrides the id coming from the JSON object)
|
||||
* @returns
|
||||
*/
|
||||
static fromJSON(queue, json, jobId) {
|
||||
const data = JSON.parse(json.data || '{}');
|
||||
const opts = Job.optsFromJSON(json.opts);
|
||||
const job = new this(queue, json.name, data, opts, json.id || jobId);
|
||||
job.progress = JSON.parse(json.progress || '0');
|
||||
job.delay = parseInt(json.delay);
|
||||
job.timestamp = parseInt(json.timestamp);
|
||||
if (json.finishedOn) {
|
||||
job.finishedOn = parseInt(json.finishedOn);
|
||||
}
|
||||
if (json.processedOn) {
|
||||
job.processedOn = parseInt(json.processedOn);
|
||||
}
|
||||
if (json.rjk) {
|
||||
job.repeatJobKey = json.rjk;
|
||||
}
|
||||
job.failedReason = json.failedReason;
|
||||
job.attemptsMade = parseInt(json.attemptsMade || '0');
|
||||
job.stacktrace = getTraces(json.stacktrace);
|
||||
if (typeof json.returnvalue === 'string') {
|
||||
job.returnvalue = getReturnValue(json.returnvalue);
|
||||
}
|
||||
if (json.parentKey) {
|
||||
job.parentKey = json.parentKey;
|
||||
}
|
||||
if (json.parent) {
|
||||
job.parent = JSON.parse(json.parent);
|
||||
}
|
||||
return job;
|
||||
}
|
||||
static optsFromJSON(rawOpts) {
|
||||
const opts = JSON.parse(rawOpts || '{}');
|
||||
const optionEntries = Object.entries(opts);
|
||||
const options = {};
|
||||
for (const item of optionEntries) {
|
||||
const [attributeName, value] = item;
|
||||
if (optsDecodeMap[attributeName]) {
|
||||
options[optsDecodeMap[attributeName]] =
|
||||
value;
|
||||
}
|
||||
else {
|
||||
options[attributeName] = value;
|
||||
}
|
||||
}
|
||||
return options;
|
||||
}
|
||||
/**
|
||||
* Fetches a Job from the queue given the passed job id.
|
||||
*
|
||||
* @param queue - the queue where the job belongs to.
|
||||
* @param jobId - the job id.
|
||||
* @returns
|
||||
*/
|
||||
static async fromId(queue, jobId) {
|
||||
// jobId can be undefined if moveJob returns undefined
|
||||
if (jobId) {
|
||||
const client = await queue.client;
|
||||
const jobData = await client.hgetall(queue.toKey(jobId));
|
||||
return (0, utils_1.isEmpty)(jobData)
|
||||
? undefined
|
||||
: this.fromJSON(queue, jobData, jobId);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* addJobLog
|
||||
*
|
||||
* @param queue Queue instance
|
||||
* @param jobId Job id
|
||||
* @param logRow Log row
|
||||
* @param keepLogs optional maximum number of logs to keep
|
||||
*
|
||||
* @returns The total number of log entries for this job so far.
|
||||
*/
|
||||
static async addJobLog(queue, jobId, logRow, keepLogs) {
|
||||
const client = await queue.client;
|
||||
const logsKey = queue.toKey(jobId) + ':logs';
|
||||
const multi = client.multi();
|
||||
multi.rpush(logsKey, logRow);
|
||||
if (keepLogs) {
|
||||
multi.ltrim(logsKey, -keepLogs, -1);
|
||||
}
|
||||
const result = (await multi.exec());
|
||||
return keepLogs ? Math.min(keepLogs, result[0][1]) : result[0][1];
|
||||
}
|
||||
toJSON() {
|
||||
const _a = this, { queue, scripts } = _a, withoutQueueAndScripts = tslib_1.__rest(_a, ["queue", "scripts"]);
|
||||
return withoutQueueAndScripts;
|
||||
}
|
||||
/**
|
||||
* Prepares a job to be serialized for storage in Redis.
|
||||
* @returns
|
||||
*/
|
||||
asJSON() {
|
||||
return {
|
||||
id: this.id,
|
||||
name: this.name,
|
||||
data: JSON.stringify(typeof this.data === 'undefined' ? {} : this.data),
|
||||
opts: this.optsAsJSON(this.opts),
|
||||
parent: this.parent ? Object.assign({}, this.parent) : undefined,
|
||||
parentKey: this.parentKey,
|
||||
progress: this.progress,
|
||||
attemptsMade: this.attemptsMade,
|
||||
finishedOn: this.finishedOn,
|
||||
processedOn: this.processedOn,
|
||||
timestamp: this.timestamp,
|
||||
failedReason: JSON.stringify(this.failedReason),
|
||||
stacktrace: JSON.stringify(this.stacktrace),
|
||||
repeatJobKey: this.repeatJobKey,
|
||||
returnvalue: JSON.stringify(this.returnvalue),
|
||||
};
|
||||
}
|
||||
optsAsJSON(opts = {}) {
|
||||
const optionEntries = Object.entries(opts);
|
||||
const options = {};
|
||||
for (const item of optionEntries) {
|
||||
const [attributeName, value] = item;
|
||||
if (optsEncodeMap[attributeName]) {
|
||||
options[optsEncodeMap[attributeName]] =
|
||||
value;
|
||||
}
|
||||
else {
|
||||
options[attributeName] = value;
|
||||
}
|
||||
}
|
||||
return options;
|
||||
}
|
||||
/**
|
||||
* Prepares a job to be passed to Sandbox.
|
||||
* @returns
|
||||
*/
|
||||
asJSONSandbox() {
|
||||
return Object.assign(Object.assign({}, this.asJSON()), { queueName: this.queueName, prefix: this.prefix });
|
||||
}
|
||||
/**
|
||||
* Updates a job's data
|
||||
*
|
||||
* @param data - the data that will replace the current jobs data.
|
||||
*/
|
||||
updateData(data) {
|
||||
this.data = data;
|
||||
return this.scripts.updateData(this, data);
|
||||
}
|
||||
/**
|
||||
* Updates a job's progress
|
||||
*
|
||||
* @param progress - number or object to be saved as progress.
|
||||
*/
|
||||
async updateProgress(progress) {
|
||||
this.progress = progress;
|
||||
await this.scripts.updateProgress(this.id, progress);
|
||||
this.queue.emit('progress', this, progress);
|
||||
}
|
||||
/**
|
||||
* Logs one row of log data.
|
||||
*
|
||||
* @param logRow - string with log data to be logged.
|
||||
* @returns The total number of log entries for this job so far.
|
||||
*/
|
||||
async log(logRow) {
|
||||
return Job.addJobLog(this.queue, this.id, logRow, this.opts.keepLogs);
|
||||
}
|
||||
/**
|
||||
* Clears job's logs
|
||||
*
|
||||
* @param keepLogs - the amount of log entries to preserve
|
||||
*/
|
||||
async clearLogs(keepLogs) {
|
||||
const client = await this.queue.client;
|
||||
const logsKey = this.toKey(this.id) + ':logs';
|
||||
if (keepLogs) {
|
||||
await client.ltrim(logsKey, -keepLogs, -1);
|
||||
}
|
||||
else {
|
||||
await client.del(logsKey);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Completely remove the job from the queue.
|
||||
* Note, this call will throw an exception if the job
|
||||
* is being processed when the call is performed.
|
||||
*
|
||||
* @param opts - Options to remove a job
|
||||
*/
|
||||
async remove({ removeChildren = true } = {}) {
|
||||
await this.queue.waitUntilReady();
|
||||
const queue = this.queue;
|
||||
const job = this;
|
||||
const removed = await this.scripts.remove(job.id, removeChildren);
|
||||
if (removed) {
|
||||
queue.emit('removed', job);
|
||||
}
|
||||
else {
|
||||
throw new Error(`Job ${this.id} could not be removed because it is locked by another worker`);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Extend the lock for this job.
|
||||
*
|
||||
* @param token - unique token for the lock
|
||||
* @param duration - lock duration in milliseconds
|
||||
*/
|
||||
extendLock(token, duration) {
|
||||
return this.scripts.extendLock(this.id, token, duration);
|
||||
}
|
||||
/**
|
||||
* Moves a job to the completed queue.
|
||||
* Returned job to be used with Queue.prototype.nextJobFromJobData.
|
||||
*
|
||||
* @param returnValue - The jobs success message.
|
||||
* @param token - Worker token used to acquire completed job.
|
||||
* @param fetchNext - True when wanting to fetch the next job.
|
||||
* @returns Returns the jobData of the next job in the waiting queue.
|
||||
*/
|
||||
async moveToCompleted(returnValue, token, fetchNext = true) {
|
||||
await this.queue.waitUntilReady();
|
||||
this.returnvalue = returnValue || void 0;
|
||||
const stringifiedReturnValue = (0, utils_1.tryCatch)(JSON.stringify, JSON, [
|
||||
returnValue,
|
||||
]);
|
||||
if (stringifiedReturnValue === utils_1.errorObject) {
|
||||
throw utils_1.errorObject.value;
|
||||
}
|
||||
const args = this.scripts.moveToCompletedArgs(this, stringifiedReturnValue, this.opts.removeOnComplete, token, fetchNext);
|
||||
const result = await this.scripts.moveToFinished(this.id, args);
|
||||
this.finishedOn = args[14];
|
||||
return result;
|
||||
}
|
||||
/**
|
||||
* Moves a job to the failed queue.
|
||||
*
|
||||
* @param err - the jobs error message.
|
||||
* @param token - token to check job is locked by current worker
|
||||
* @param fetchNext - true when wanting to fetch the next job
|
||||
* @returns void
|
||||
*/
|
||||
async moveToFailed(err, token, fetchNext = false) {
|
||||
const client = await this.queue.client;
|
||||
const message = err === null || err === void 0 ? void 0 : err.message;
|
||||
const queue = this.queue;
|
||||
this.failedReason = message;
|
||||
let command;
|
||||
const multi = client.multi();
|
||||
this.saveStacktrace(multi, err);
|
||||
//
|
||||
// Check if an automatic retry should be performed
|
||||
//
|
||||
let moveToFailed = false;
|
||||
let finishedOn, delay;
|
||||
if (this.attemptsMade < this.opts.attempts &&
|
||||
!this.discarded &&
|
||||
!(err instanceof unrecoverable_error_1.UnrecoverableError || err.name == 'UnrecoverableError')) {
|
||||
const opts = queue.opts;
|
||||
// Check if backoff is needed
|
||||
delay = await backoffs_1.Backoffs.calculate(this.opts.backoff, this.attemptsMade, err, this, opts.settings && opts.settings.backoffStrategy);
|
||||
if (delay === -1) {
|
||||
moveToFailed = true;
|
||||
}
|
||||
else if (delay) {
|
||||
const args = this.scripts.moveToDelayedArgs(this.id, Date.now() + delay, token, delay);
|
||||
this.scripts.execCommand(multi, 'moveToDelayed', args);
|
||||
command = 'delayed';
|
||||
}
|
||||
else {
|
||||
// Retry immediately
|
||||
this.scripts.execCommand(multi, 'retryJob', this.scripts.retryJobArgs(this.id, this.opts.lifo, token));
|
||||
command = 'retryJob';
|
||||
}
|
||||
}
|
||||
else {
|
||||
// If not, move to failed
|
||||
moveToFailed = true;
|
||||
}
|
||||
if (moveToFailed) {
|
||||
const args = this.scripts.moveToFailedArgs(this, message, this.opts.removeOnFail, token, fetchNext);
|
||||
this.scripts.execCommand(multi, 'moveToFinished', args);
|
||||
finishedOn = args[14];
|
||||
command = 'failed';
|
||||
}
|
||||
const results = await multi.exec();
|
||||
const anyError = results.find(result => result[0]);
|
||||
if (anyError) {
|
||||
throw new Error(`Error "moveToFailed" with command ${command}: ${anyError}`);
|
||||
}
|
||||
const code = results[results.length - 1][1];
|
||||
if (code < 0) {
|
||||
throw this.scripts.finishedErrors(code, this.id, command, 'active');
|
||||
}
|
||||
if (finishedOn && typeof finishedOn === 'number') {
|
||||
this.finishedOn = finishedOn;
|
||||
}
|
||||
if (delay && typeof delay === 'number') {
|
||||
this.delay = delay;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* @returns true if the job has completed.
|
||||
*/
|
||||
isCompleted() {
|
||||
return this.isInZSet('completed');
|
||||
}
|
||||
/**
|
||||
* @returns true if the job has failed.
|
||||
*/
|
||||
isFailed() {
|
||||
return this.isInZSet('failed');
|
||||
}
|
||||
/**
|
||||
* @returns true if the job is delayed.
|
||||
*/
|
||||
isDelayed() {
|
||||
return this.isInZSet('delayed');
|
||||
}
|
||||
/**
|
||||
* @returns true if the job is waiting for children.
|
||||
*/
|
||||
isWaitingChildren() {
|
||||
return this.isInZSet('waiting-children');
|
||||
}
|
||||
/**
|
||||
* @returns true of the job is active.
|
||||
*/
|
||||
isActive() {
|
||||
return this.isInList('active');
|
||||
}
|
||||
/**
|
||||
* @returns true if the job is waiting.
|
||||
*/
|
||||
async isWaiting() {
|
||||
return (await this.isInList('wait')) || (await this.isInList('paused'));
|
||||
}
|
||||
/**
|
||||
* @returns the queue name this job belongs to.
|
||||
*/
|
||||
get queueName() {
|
||||
return this.queue.name;
|
||||
}
|
||||
/**
|
||||
* @returns the prefix that is used.
|
||||
*/
|
||||
get prefix() {
|
||||
return this.queue.opts.prefix;
|
||||
}
|
||||
/**
|
||||
* Get current state.
|
||||
*
|
||||
* @returns Returns one of these values:
|
||||
* 'completed', 'failed', 'delayed', 'active', 'waiting', 'waiting-children', 'unknown'.
|
||||
*/
|
||||
getState() {
|
||||
return this.scripts.getState(this.id);
|
||||
}
|
||||
/**
|
||||
* Change delay of a delayed job.
|
||||
*
|
||||
* @param delay - milliseconds to be added to current time.
|
||||
* @returns void
|
||||
*/
|
||||
async changeDelay(delay) {
|
||||
await this.scripts.changeDelay(this.id, delay);
|
||||
this.delay = delay;
|
||||
}
|
||||
/**
|
||||
* Change job priority.
|
||||
*
|
||||
* @returns void
|
||||
*/
|
||||
async changePriority(opts) {
|
||||
await this.scripts.changePriority(this.id, opts.priority, opts.lifo);
|
||||
}
|
||||
/**
|
||||
* Get this jobs children result values if any.
|
||||
*
|
||||
* @returns Object mapping children job keys with their values.
|
||||
*/
|
||||
async getChildrenValues() {
|
||||
const client = await this.queue.client;
|
||||
const result = (await client.hgetall(this.toKey(`${this.id}:processed`)));
|
||||
if (result) {
|
||||
return (0, utils_1.parseObjectValues)(result);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Get children job keys if this job is a parent and has children.
|
||||
* @remarks
|
||||
* Count options before Redis v7.2 works as expected with any quantity of entries
|
||||
* on processed/unprocessed dependencies, since v7.2 you must consider that count
|
||||
* won't have any effect until processed/unprocessed dependencies have a length
|
||||
* greater than 127
|
||||
* @see https://redis.io/docs/management/optimization/memory-optimization/#redis--72
|
||||
* @returns dependencies separated by processed and unprocessed.
|
||||
*/
|
||||
async getDependencies(opts = {}) {
|
||||
const client = await this.queue.client;
|
||||
const multi = client.multi();
|
||||
if (!opts.processed && !opts.unprocessed) {
|
||||
multi.hgetall(this.toKey(`${this.id}:processed`));
|
||||
multi.smembers(this.toKey(`${this.id}:dependencies`));
|
||||
const [[err1, processed], [err2, unprocessed]] = (await multi.exec());
|
||||
const transformedProcessed = (0, utils_1.parseObjectValues)(processed);
|
||||
return { processed: transformedProcessed, unprocessed };
|
||||
}
|
||||
else {
|
||||
const defaultOpts = {
|
||||
cursor: 0,
|
||||
count: 20,
|
||||
};
|
||||
if (opts.processed) {
|
||||
const processedOpts = Object.assign(Object.assign({}, defaultOpts), opts.processed);
|
||||
multi.hscan(this.toKey(`${this.id}:processed`), processedOpts.cursor, 'COUNT', processedOpts.count);
|
||||
}
|
||||
if (opts.unprocessed) {
|
||||
const unprocessedOpts = Object.assign(Object.assign({}, defaultOpts), opts.unprocessed);
|
||||
multi.sscan(this.toKey(`${this.id}:dependencies`), unprocessedOpts.cursor, 'COUNT', unprocessedOpts.count);
|
||||
}
|
||||
const [result1, result2] = (await multi.exec());
|
||||
const [processedCursor, processed = []] = opts.processed
|
||||
? result1[1]
|
||||
: [];
|
||||
const [unprocessedCursor, unprocessed = []] = opts.unprocessed
|
||||
? opts.processed
|
||||
? result2[1]
|
||||
: result1[1]
|
||||
: [];
|
||||
const transformedProcessed = {};
|
||||
for (let index = 0; index < processed.length; ++index) {
|
||||
if (index % 2) {
|
||||
transformedProcessed[processed[index - 1]] = JSON.parse(processed[index]);
|
||||
}
|
||||
}
|
||||
return Object.assign(Object.assign({}, (processedCursor
|
||||
? {
|
||||
processed: transformedProcessed,
|
||||
nextProcessedCursor: Number(processedCursor),
|
||||
}
|
||||
: {})), (unprocessedCursor
|
||||
? { unprocessed, nextUnprocessedCursor: Number(unprocessedCursor) }
|
||||
: {}));
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Get children job counts if this job is a parent and has children.
|
||||
*
|
||||
* @returns dependencies count separated by processed and unprocessed.
|
||||
*/
|
||||
async getDependenciesCount(opts = {}) {
|
||||
const client = await this.queue.client;
|
||||
const multi = client.multi();
|
||||
const updatedOpts = !opts.processed && !opts.unprocessed
|
||||
? { processed: true, unprocessed: true }
|
||||
: opts;
|
||||
if (updatedOpts.processed) {
|
||||
multi.hlen(this.toKey(`${this.id}:processed`));
|
||||
}
|
||||
if (updatedOpts.unprocessed) {
|
||||
multi.scard(this.toKey(`${this.id}:dependencies`));
|
||||
}
|
||||
const [[err1, result1] = [], [err2, result2] = []] = (await multi.exec());
|
||||
const processed = updatedOpts.processed ? result1 : undefined;
|
||||
const unprocessed = updatedOpts.unprocessed
|
||||
? updatedOpts.processed
|
||||
? result2
|
||||
: result1
|
||||
: undefined;
|
||||
return Object.assign(Object.assign({}, (updatedOpts.processed
|
||||
? {
|
||||
processed,
|
||||
}
|
||||
: {})), (updatedOpts.unprocessed ? { unprocessed } : {}));
|
||||
}
|
||||
/**
|
||||
* Returns a promise the resolves when the job has completed (containing the return value of the job),
|
||||
* or rejects when the job has failed (containing the failedReason).
|
||||
*
|
||||
* @param queueEvents - Instance of QueueEvents.
|
||||
* @param ttl - Time in milliseconds to wait for job to finish before timing out.
|
||||
*/
|
||||
async waitUntilFinished(queueEvents, ttl) {
|
||||
await this.queue.waitUntilReady();
|
||||
const jobId = this.id;
|
||||
return new Promise(async (resolve, reject) => {
|
||||
let timeout;
|
||||
if (ttl) {
|
||||
timeout = setTimeout(() => onFailed(
|
||||
/* eslint-disable max-len */
|
||||
`Job wait ${this.name} timed out before finishing, no finish notification arrived after ${ttl}ms (id=${jobId})`), ttl);
|
||||
}
|
||||
function onCompleted(args) {
|
||||
removeListeners();
|
||||
resolve(args.returnvalue);
|
||||
}
|
||||
function onFailed(args) {
|
||||
removeListeners();
|
||||
reject(new Error(args.failedReason || args));
|
||||
}
|
||||
const completedEvent = `completed:${jobId}`;
|
||||
const failedEvent = `failed:${jobId}`;
|
||||
queueEvents.on(completedEvent, onCompleted);
|
||||
queueEvents.on(failedEvent, onFailed);
|
||||
this.queue.on('closing', onFailed);
|
||||
const removeListeners = () => {
|
||||
clearInterval(timeout);
|
||||
queueEvents.removeListener(completedEvent, onCompleted);
|
||||
queueEvents.removeListener(failedEvent, onFailed);
|
||||
this.queue.removeListener('closing', onFailed);
|
||||
};
|
||||
// Poll once right now to see if the job has already finished. The job may have been completed before we were able
|
||||
// to register the event handlers on the QueueEvents, so we check here to make sure we're not waiting for an event
|
||||
// that has already happened. We block checking the job until the queue events object is actually listening to
|
||||
// Redis so there's no chance that it will miss events.
|
||||
await queueEvents.waitUntilReady();
|
||||
const [status, result] = (await this.scripts.isFinished(jobId, true));
|
||||
const finished = status != 0;
|
||||
if (finished) {
|
||||
if (status == -1 || status == 2) {
|
||||
onFailed({ failedReason: result });
|
||||
}
|
||||
else {
|
||||
onCompleted({ returnvalue: getReturnValue(result) });
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Moves the job to the delay set.
|
||||
*
|
||||
* @param timestamp - timestamp where the job should be moved back to "wait"
|
||||
* @param token - token to check job is locked by current worker
|
||||
* @returns
|
||||
*/
|
||||
moveToDelayed(timestamp, token) {
|
||||
const delay = timestamp - Date.now();
|
||||
return this.scripts.moveToDelayed(this.id, timestamp, delay > 0 ? delay : 0, token);
|
||||
}
|
||||
/**
|
||||
* Moves the job to the waiting-children set.
|
||||
*
|
||||
* @param token - Token to check job is locked by current worker
|
||||
* @param opts - The options bag for moving a job to waiting-children.
|
||||
* @returns true if the job was moved
|
||||
*/
|
||||
moveToWaitingChildren(token, opts = {}) {
|
||||
return this.scripts.moveToWaitingChildren(this.id, token, opts);
|
||||
}
|
||||
/**
|
||||
* Promotes a delayed job so that it starts to be processed as soon as possible.
|
||||
*/
|
||||
async promote() {
|
||||
const jobId = this.id;
|
||||
await this.scripts.promote(jobId);
|
||||
this.delay = 0;
|
||||
}
|
||||
/**
|
||||
* Attempts to retry the job. Only a job that has failed or completed can be retried.
|
||||
*
|
||||
* @param state - completed / failed
|
||||
* @returns If resolved and return code is 1, then the queue emits a waiting event
|
||||
* otherwise the operation was not a success and throw the corresponding error. If the promise
|
||||
* rejects, it indicates that the script failed to execute
|
||||
*/
|
||||
retry(state = 'failed') {
|
||||
this.failedReason = null;
|
||||
this.finishedOn = null;
|
||||
this.processedOn = null;
|
||||
this.returnvalue = null;
|
||||
return this.scripts.reprocessJob(this, state);
|
||||
}
|
||||
/**
|
||||
* Marks a job to not be retried if it fails (even if attempts has been configured)
|
||||
*/
|
||||
discard() {
|
||||
this.discarded = true;
|
||||
}
|
||||
async isInZSet(set) {
|
||||
const client = await this.queue.client;
|
||||
const score = await client.zscore(this.queue.toKey(set), this.id);
|
||||
return score !== null;
|
||||
}
|
||||
async isInList(list) {
|
||||
return this.scripts.isJobInList(this.queue.toKey(list), this.id);
|
||||
}
|
||||
/**
|
||||
* Adds the job to Redis.
|
||||
*
|
||||
* @param client -
|
||||
* @param parentOpts -
|
||||
* @returns
|
||||
*/
|
||||
addJob(client, parentOpts) {
|
||||
const jobData = this.asJSON();
|
||||
this.validateOptions(jobData);
|
||||
return this.scripts.addJob(client, jobData, jobData.opts, this.id, parentOpts);
|
||||
}
|
||||
validateOptions(jobData) {
|
||||
var _a;
|
||||
const exceedLimit = this.opts.sizeLimit &&
|
||||
(0, utils_1.lengthInUtf8Bytes)(jobData.data) > this.opts.sizeLimit;
|
||||
if (exceedLimit) {
|
||||
throw new Error(`The size of job ${this.name} exceeds the limit ${this.opts.sizeLimit} bytes`);
|
||||
}
|
||||
if (this.opts.delay && this.opts.repeat && !((_a = this.opts.repeat) === null || _a === void 0 ? void 0 : _a.count)) {
|
||||
throw new Error(`Delay and repeat options could not be used together`);
|
||||
}
|
||||
if (this.opts.removeDependencyOnFailure && this.opts.failParentOnFailure) {
|
||||
throw new Error(`RemoveDependencyOnFailure and failParentOnFailure options can not be used together`);
|
||||
}
|
||||
if (`${parseInt(this.id, 10)}` === this.id) {
|
||||
//TODO: throw an error in next breaking change
|
||||
console.warn('Custom Ids should not be integers: https://github.com/taskforcesh/bullmq/pull/1569');
|
||||
}
|
||||
if (this.opts.priority) {
|
||||
if (Math.trunc(this.opts.priority) !== this.opts.priority) {
|
||||
throw new Error(`Priority should not be float`);
|
||||
}
|
||||
if (this.opts.priority > exports.PRIORITY_LIMIT) {
|
||||
throw new Error(`Priority should be between 0 and ${exports.PRIORITY_LIMIT}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
saveStacktrace(multi, err) {
|
||||
this.stacktrace = this.stacktrace || [];
|
||||
if (err === null || err === void 0 ? void 0 : err.stack) {
|
||||
this.stacktrace.push(err.stack);
|
||||
if (this.opts.stackTraceLimit) {
|
||||
this.stacktrace = this.stacktrace.slice(0, this.opts.stackTraceLimit);
|
||||
}
|
||||
}
|
||||
const args = this.scripts.saveStacktraceArgs(this.id, JSON.stringify(this.stacktrace), err === null || err === void 0 ? void 0 : err.message);
|
||||
this.scripts.execCommand(multi, 'saveStacktrace', args);
|
||||
}
|
||||
}
|
||||
exports.Job = Job;
|
||||
function getTraces(stacktrace) {
|
||||
const traces = (0, utils_1.tryCatch)(JSON.parse, JSON, [stacktrace]);
|
||||
if (traces === utils_1.errorObject || !(traces instanceof Array)) {
|
||||
return [];
|
||||
}
|
||||
else {
|
||||
return traces;
|
||||
}
|
||||
}
|
||||
function getReturnValue(_value) {
|
||||
const value = (0, utils_1.tryCatch)(JSON.parse, JSON, [_value]);
|
||||
if (value !== utils_1.errorObject) {
|
||||
return value;
|
||||
}
|
||||
else {
|
||||
logger('corrupted returnvalue: ' + _value, value);
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=job.js.map
|
||||
+1
File diff suppressed because one or more lines are too long
+45
@@ -0,0 +1,45 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
/**
|
||||
* Wrapper for sandboxing.
|
||||
*
|
||||
*/
|
||||
const lodash_1 = require("lodash");
|
||||
const child_processor_1 = require("./child-processor");
|
||||
const enums_1 = require("../enums");
|
||||
const utils_1 = require("../utils");
|
||||
exports.default = (send, receiver) => {
|
||||
const childProcessor = new child_processor_1.ChildProcessor(send);
|
||||
receiver === null || receiver === void 0 ? void 0 : receiver.on('message', async (msg) => {
|
||||
try {
|
||||
switch (msg.cmd) {
|
||||
case enums_1.ChildCommand.Init:
|
||||
await childProcessor.init(msg.value);
|
||||
break;
|
||||
case enums_1.ChildCommand.Start:
|
||||
await childProcessor.start(msg.job, msg === null || msg === void 0 ? void 0 : msg.token);
|
||||
break;
|
||||
case enums_1.ChildCommand.Stop:
|
||||
break;
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
console.error('Error handling child message');
|
||||
}
|
||||
});
|
||||
process.on('SIGTERM', () => childProcessor.waitForCurrentJobAndExit());
|
||||
process.on('SIGINT', () => childProcessor.waitForCurrentJobAndExit());
|
||||
process.on('uncaughtException', async (err) => {
|
||||
if (!err.message) {
|
||||
err = new Error((0, lodash_1.toString)(err));
|
||||
}
|
||||
await send({
|
||||
cmd: enums_1.ParentCommand.Failed,
|
||||
value: (0, utils_1.errorToJSON)(err),
|
||||
});
|
||||
// An uncaughException leaves this process in a potentially undetermined state so
|
||||
// we must exit
|
||||
process.exit();
|
||||
});
|
||||
};
|
||||
//# sourceMappingURL=main-base.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"main-base.js","sourceRoot":"","sources":["../../../src/classes/main-base.ts"],"names":[],"mappings":";;AAAA;;;GAGG;AACH,mCAAkC;AAClC,uDAAmD;AACnD,oCAAuD;AACvD,oCAAuC;AAEvC,kBAAe,CACb,IAAiC,EACjC,QAAkE,EAClE,EAAE;IACF,MAAM,cAAc,GAAG,IAAI,gCAAc,CAAC,IAAI,CAAC,CAAC;IAEhD,QAAQ,aAAR,QAAQ,uBAAR,QAAQ,CAAE,EAAE,CAAC,SAAS,EAAE,KAAK,EAAC,GAAG,EAAC,EAAE;QAClC,IAAI;YACF,QAAQ,GAAG,CAAC,GAAmB,EAAE;gBAC/B,KAAK,oBAAY,CAAC,IAAI;oBACpB,MAAM,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;oBACrC,MAAM;gBACR,KAAK,oBAAY,CAAC,KAAK;oBACrB,MAAM,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,KAAK,CAAC,CAAC;oBAChD,MAAM;gBACR,KAAK,oBAAY,CAAC,IAAI;oBACpB,MAAM;aACT;SACF;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;SAC/C;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,cAAc,CAAC,wBAAwB,EAAE,CAAC,CAAC;IACvE,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,cAAc,CAAC,wBAAwB,EAAE,CAAC,CAAC;IAEtE,OAAO,CAAC,EAAE,CAAC,mBAAmB,EAAE,KAAK,EAAE,GAAU,EAAE,EAAE;QACnD,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE;YAChB,GAAG,GAAG,IAAI,KAAK,CAAC,IAAA,iBAAQ,EAAC,GAAG,CAAC,CAAC,CAAC;SAChC;QACD,MAAM,IAAI,CAAC;YACT,GAAG,EAAE,qBAAa,CAAC,MAAM;YACzB,KAAK,EAAE,IAAA,mBAAW,EAAC,GAAG,CAAC;SACxB,CAAC,CAAC;QAEH,iFAAiF;QACjF,eAAe;QACf,OAAO,CAAC,IAAI,EAAE,CAAC;IACjB,CAAC,CAAC,CAAC;AACL,CAAC,CAAC"}
|
||||
+1
@@ -0,0 +1 @@
|
||||
export {};
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
/**
|
||||
* Worker Thread wrapper for sandboxing
|
||||
*
|
||||
*/
|
||||
const worker_threads_1 = require("worker_threads");
|
||||
const main_base_1 = require("./main-base");
|
||||
(0, main_base_1.default)(async (msg) => worker_threads_1.parentPort.postMessage(msg), worker_threads_1.parentPort);
|
||||
//# sourceMappingURL=main-worker.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"main-worker.js","sourceRoot":"","sources":["../../../src/classes/main-worker.ts"],"names":[],"mappings":";;AAAA;;;GAGG;AACH,mDAA4C;AAC5C,2CAAmC;AAEnC,IAAA,mBAAQ,EAAC,KAAK,EAAE,GAAQ,EAAE,EAAE,CAAC,2BAAU,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,2BAAU,CAAC,CAAC"}
|
||||
+1
@@ -0,0 +1 @@
|
||||
export {};
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
/**
|
||||
* Child process wrapper for sandboxing.
|
||||
*
|
||||
*/
|
||||
const utils_1 = require("../utils");
|
||||
const main_base_1 = require("./main-base");
|
||||
(0, main_base_1.default)((msg) => (0, utils_1.childSend)(process, msg), process);
|
||||
//# sourceMappingURL=main.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"main.js","sourceRoot":"","sources":["../../../src/classes/main.ts"],"names":[],"mappings":";;AAAA;;;GAGG;AACH,oCAAqC;AACrC,2CAAmC;AAEnC,IAAA,mBAAQ,EAAC,CAAC,GAAQ,EAAE,EAAE,CAAC,IAAA,iBAAS,EAAC,OAAO,EAAE,GAAG,CAAC,EAAE,OAAO,CAAC,CAAC"}
|
||||
+141
@@ -0,0 +1,141 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.QueueBase = void 0;
|
||||
const events_1 = require("events");
|
||||
const utils_1 = require("../utils");
|
||||
const redis_connection_1 = require("./redis-connection");
|
||||
const job_1 = require("./job");
|
||||
const queue_keys_1 = require("./queue-keys");
|
||||
const scripts_1 = require("./scripts");
|
||||
/**
|
||||
* @class QueueBase
|
||||
* @extends EventEmitter
|
||||
*
|
||||
* @description Base class for all classes that need to interact with queues.
|
||||
* This class is normally not used directly, but extended by the other classes.
|
||||
*
|
||||
*/
|
||||
class QueueBase extends events_1.EventEmitter {
|
||||
/**
|
||||
*
|
||||
* @param name - The name of the queue.
|
||||
* @param opts - Options for the queue.
|
||||
* @param Connection - An optional "Connection" class used to instantiate a Connection. This is useful for
|
||||
* testing with mockups and/or extending the Connection class and passing an alternate implementation.
|
||||
*/
|
||||
constructor(name, opts = {}, Connection = redis_connection_1.RedisConnection) {
|
||||
super();
|
||||
this.name = name;
|
||||
this.opts = opts;
|
||||
this.closed = false;
|
||||
this.opts = Object.assign({ prefix: 'bull' }, opts);
|
||||
if (!name) {
|
||||
throw new Error('Queue name must be provided');
|
||||
}
|
||||
if (!opts.connection) {
|
||||
console.warn([
|
||||
'BullMQ: DEPRECATION WARNING! Optional instantiation of Queue, Worker, QueueEvents and FlowProducer',
|
||||
'without providing explicitly a connection or connection options is deprecated. This behaviour will',
|
||||
'be removed in the next major release',
|
||||
].join(' '));
|
||||
}
|
||||
this.connection = new Connection(opts.connection, (0, utils_1.isRedisInstance)(opts === null || opts === void 0 ? void 0 : opts.connection), opts.blockingConnection, opts.skipVersionCheck);
|
||||
this.connection.on('error', (error) => this.emit('error', error));
|
||||
this.connection.on('close', () => {
|
||||
if (!this.closing) {
|
||||
this.emit('ioredis:close');
|
||||
}
|
||||
});
|
||||
const queueKeys = new queue_keys_1.QueueKeys(opts.prefix);
|
||||
this.qualifiedName = queueKeys.getQueueQualifiedName(name);
|
||||
this.keys = queueKeys.getKeys(name);
|
||||
this.toKey = (type) => queueKeys.toKey(name, type);
|
||||
this.scripts = new scripts_1.Scripts(this);
|
||||
}
|
||||
/**
|
||||
* Returns a promise that resolves to a redis client. Normally used only by subclasses.
|
||||
*/
|
||||
get client() {
|
||||
return this.connection.client;
|
||||
}
|
||||
/**
|
||||
* Returns the version of the Redis instance the client is connected to,
|
||||
*/
|
||||
get redisVersion() {
|
||||
return this.connection.redisVersion;
|
||||
}
|
||||
/**
|
||||
* Helper to easily extend Job class calls.
|
||||
*/
|
||||
get Job() {
|
||||
return job_1.Job;
|
||||
}
|
||||
/**
|
||||
* Emits an event. Normally used by subclasses to emit events.
|
||||
*
|
||||
* @param event - The emitted event.
|
||||
* @param args -
|
||||
* @returns
|
||||
*/
|
||||
emit(event, ...args) {
|
||||
try {
|
||||
return super.emit(event, ...args);
|
||||
}
|
||||
catch (err) {
|
||||
try {
|
||||
return super.emit('error', err);
|
||||
}
|
||||
catch (err) {
|
||||
// We give up if the error event also throws an exception.
|
||||
console.error(err);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
waitUntilReady() {
|
||||
return this.client;
|
||||
}
|
||||
base64Name() {
|
||||
return Buffer.from(this.name).toString('base64');
|
||||
}
|
||||
clientName(suffix = '') {
|
||||
const queueNameBase64 = this.base64Name();
|
||||
return `${this.opts.prefix}:${queueNameBase64}${suffix}`;
|
||||
}
|
||||
/**
|
||||
*
|
||||
* Closes the connection and returns a promise that resolves when the connection is closed.
|
||||
*/
|
||||
async close() {
|
||||
if (!this.closing) {
|
||||
this.closing = this.connection.close();
|
||||
}
|
||||
await this.closing;
|
||||
this.closed = true;
|
||||
}
|
||||
/**
|
||||
*
|
||||
* Force disconnects a connection.
|
||||
*/
|
||||
disconnect() {
|
||||
return this.connection.disconnect();
|
||||
}
|
||||
async checkConnectionError(fn, delayInMs = utils_1.DELAY_TIME_5) {
|
||||
try {
|
||||
return await fn();
|
||||
}
|
||||
catch (error) {
|
||||
if ((0, utils_1.isNotConnectionError)(error)) {
|
||||
this.emit('error', error);
|
||||
}
|
||||
if (!this.closing && delayInMs) {
|
||||
await (0, utils_1.delay)(delayInMs);
|
||||
}
|
||||
else {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
exports.QueueBase = QueueBase;
|
||||
//# sourceMappingURL=queue-base.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"queue-base.js","sourceRoot":"","sources":["../../../src/classes/queue-base.ts"],"names":[],"mappings":";;;AAAA,mCAAsC;AAGtC,oCAKkB;AAClB,yDAAqD;AACrD,+BAA4B;AAC5B,6CAAkD;AAClD,uCAAoC;AAEpC;;;;;;;GAOG;AACH,MAAa,SAAU,SAAQ,qBAAY;IAUzC;;;;;;OAMG;IACH,YACkB,IAAY,EACrB,OAAyB,EAAE,EAClC,aAAqC,kCAAe;QAEpD,KAAK,EAAE,CAAC;QAJQ,SAAI,GAAJ,IAAI,CAAQ;QACrB,SAAI,GAAJ,IAAI,CAAuB;QAd1B,WAAM,GAAY,KAAK,CAAC;QAmBhC,IAAI,CAAC,IAAI,mBACP,MAAM,EAAE,MAAM,IACX,IAAI,CACR,CAAC;QAEF,IAAI,CAAC,IAAI,EAAE;YACT,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;SAChD;QAED,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE;YACpB,OAAO,CAAC,IAAI,CACV;gBACE,oGAAoG;gBACpG,oGAAoG;gBACpG,sCAAsC;aACvC,CAAC,IAAI,CAAC,GAAG,CAAC,CACZ,CAAC;SACH;QAED,IAAI,CAAC,UAAU,GAAG,IAAI,UAAU,CAC9B,IAAI,CAAC,UAAU,EACf,IAAA,uBAAe,EAAC,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,UAAU,CAAC,EACjC,IAAI,CAAC,kBAAkB,EACvB,IAAI,CAAC,gBAAgB,CACtB,CAAC;QAEF,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAY,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;QACzE,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YAC/B,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;gBACjB,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;aAC5B;QACH,CAAC,CAAC,CAAC;QAEH,MAAM,SAAS,GAAG,IAAI,sBAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC7C,IAAI,CAAC,aAAa,GAAG,SAAS,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAC3D,IAAI,CAAC,IAAI,GAAG,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,CAAC,KAAK,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC3D,IAAI,CAAC,OAAO,GAAG,IAAI,iBAAO,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC;IAED;;OAEG;IACH,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,IAAc,GAAG;QACf,OAAO,SAAG,CAAC;IACb,CAAC;IAED;;;;;;OAMG;IACH,IAAI,CAAC,KAAsB,EAAE,GAAG,IAAW;QACzC,IAAI;YACF,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,CAAC;SACnC;QAAC,OAAO,GAAG,EAAE;YACZ,IAAI;gBACF,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;aACjC;YAAC,OAAO,GAAG,EAAE;gBACZ,0DAA0D;gBAC1D,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACnB,OAAO,KAAK,CAAC;aACd;SACF;IACH,CAAC;IAED,cAAc;QACZ,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAES,UAAU;QAClB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACnD,CAAC;IAES,UAAU,CAAC,MAAM,GAAG,EAAE;QAC9B,MAAM,eAAe,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;QAC1C,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,eAAe,GAAG,MAAM,EAAE,CAAC;IAC3D,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YACjB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;SACxC;QACD,MAAM,IAAI,CAAC,OAAO,CAAC;QACnB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;IACrB,CAAC;IAED;;;OAGG;IACH,UAAU;QACR,OAAO,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,CAAC;IACtC,CAAC;IAES,KAAK,CAAC,oBAAoB,CAClC,EAAoB,EACpB,SAAS,GAAG,oBAAY;QAExB,IAAI;YACF,OAAO,MAAM,EAAE,EAAE,CAAC;SACnB;QAAC,OAAO,KAAK,EAAE;YACd,IAAI,IAAA,4BAAoB,EAAC,KAAc,CAAC,EAAE;gBACxC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAS,KAAK,CAAC,CAAC;aAClC;YAED,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,SAAS,EAAE;gBAC9B,MAAM,IAAA,aAAK,EAAC,SAAS,CAAC,CAAC;aACxB;iBAAM;gBACL,OAAO;aACR;SACF;IACH,CAAC;CACF;AA7JD,8BA6JC"}
|
||||
+120
@@ -0,0 +1,120 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.QueueEvents = void 0;
|
||||
const tslib_1 = require("tslib");
|
||||
const utils_1 = require("../utils");
|
||||
const queue_base_1 = require("./queue-base");
|
||||
/**
|
||||
* The QueueEvents class is used for listening to the global events
|
||||
* emitted by a given queue.
|
||||
*
|
||||
* This class requires a dedicated redis connection.
|
||||
*
|
||||
*/
|
||||
class QueueEvents extends queue_base_1.QueueBase {
|
||||
constructor(name, _a = {}, Connection) {
|
||||
var { connection, autorun = true } = _a, opts = tslib_1.__rest(_a, ["connection", "autorun"]);
|
||||
super(name, Object.assign(Object.assign({}, opts), { connection: (0, utils_1.isRedisInstance)(connection)
|
||||
? connection.duplicate()
|
||||
: connection, blockingConnection: true }), Connection);
|
||||
this.running = false;
|
||||
this.opts = Object.assign({
|
||||
blockingTimeout: 10000,
|
||||
}, this.opts);
|
||||
if (autorun) {
|
||||
this.run().catch(error => this.emit('error', error));
|
||||
}
|
||||
}
|
||||
emit(event, ...args) {
|
||||
return super.emit(event, ...args);
|
||||
}
|
||||
off(eventName, listener) {
|
||||
super.off(eventName, listener);
|
||||
return this;
|
||||
}
|
||||
on(event, listener) {
|
||||
super.on(event, listener);
|
||||
return this;
|
||||
}
|
||||
once(event, listener) {
|
||||
super.once(event, listener);
|
||||
return this;
|
||||
}
|
||||
/**
|
||||
* Manually starts running the event consumming loop. This shall be used if you do not
|
||||
* use the default "autorun" option on the constructor.
|
||||
*/
|
||||
async run() {
|
||||
if (!this.running) {
|
||||
try {
|
||||
this.running = true;
|
||||
const client = await this.client;
|
||||
try {
|
||||
await client.client('SETNAME', this.clientName(utils_1.QUEUE_EVENT_SUFFIX));
|
||||
}
|
||||
catch (err) {
|
||||
if (!utils_1.clientCommandMessageReg.test(err.message)) {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
await this.consumeEvents(client);
|
||||
}
|
||||
catch (error) {
|
||||
this.running = false;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
else {
|
||||
throw new Error('Queue Events is already running.');
|
||||
}
|
||||
}
|
||||
async consumeEvents(client) {
|
||||
const opts = this.opts;
|
||||
const key = this.keys.events;
|
||||
let id = opts.lastEventId || '$';
|
||||
while (!this.closing) {
|
||||
// Cast to actual return type, see: https://github.com/DefinitelyTyped/DefinitelyTyped/issues/44301
|
||||
const data = await this.checkConnectionError(() => client.xread('BLOCK', opts.blockingTimeout, 'STREAMS', key, id));
|
||||
if (data) {
|
||||
const stream = data[0];
|
||||
const events = stream[1];
|
||||
for (let i = 0; i < events.length; i++) {
|
||||
id = events[i][0];
|
||||
const args = (0, utils_1.array2obj)(events[i][1]);
|
||||
//
|
||||
// TODO: we may need to have a separate xtream for progress data
|
||||
// to avoid this hack.
|
||||
switch (args.event) {
|
||||
case 'progress':
|
||||
args.data = JSON.parse(args.data);
|
||||
break;
|
||||
case 'completed':
|
||||
args.returnvalue = JSON.parse(args.returnvalue);
|
||||
break;
|
||||
}
|
||||
const { event } = args, restArgs = tslib_1.__rest(args, ["event"]);
|
||||
if (event === 'drained') {
|
||||
this.emit(event, id);
|
||||
}
|
||||
else {
|
||||
this.emit(event, restArgs, id);
|
||||
this.emit(`${event}:${restArgs.jobId}`, restArgs, id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Stops consuming events and close the underlying Redis connection if necessary.
|
||||
*
|
||||
* @returns
|
||||
*/
|
||||
close() {
|
||||
if (!this.closing) {
|
||||
this.closing = this.disconnect();
|
||||
}
|
||||
return this.closing;
|
||||
}
|
||||
}
|
||||
exports.QueueEvents = QueueEvents;
|
||||
//# sourceMappingURL=queue-events.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"queue-events.js","sourceRoot":"","sources":["../../../src/classes/queue-events.ts"],"names":[],"mappings":";;;;AAMA,oCAKkB;AAClB,6CAAyC;AAgJzC;;;;;;GAMG;AACH,MAAa,WAAY,SAAQ,sBAAS;IAGxC,YACE,IAAY,EACZ,KAA8D,EAAE,EAChE,UAAmC;YADnC,EAAE,UAAU,EAAE,OAAO,GAAG,IAAI,OAAoC,EAA/B,IAAI,sBAArC,yBAAuC,CAAF;QAGrC,KAAK,CACH,IAAI,kCAEC,IAAI,KACP,UAAU,EAAE,IAAA,uBAAe,EAAC,UAAU,CAAC;gBACrC,CAAC,CAAe,UAAW,CAAC,SAAS,EAAE;gBACvC,CAAC,CAAC,UAAU,EACd,kBAAkB,EAAE,IAAI,KAE1B,UAAU,CACX,CAAC;QAjBI,YAAO,GAAG,KAAK,CAAC;QAmBtB,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,MAAM,CACvB;YACE,eAAe,EAAE,KAAK;SACvB,EACD,IAAI,CAAC,IAAI,CACV,CAAC;QAEF,IAAI,OAAO,EAAE;YACX,IAAI,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;SACtD;IACH,CAAC;IAED,IAAI,CACF,KAAQ,EACR,GAAG,IAAwC;QAE3C,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,CAAC;IACpC,CAAC;IAED,GAAG,CACD,SAAY,EACZ,QAAgC;QAEhC,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAC/B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,EAAE,CACA,KAAQ,EACR,QAAgC;QAEhC,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAC1B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CACF,KAAQ,EACR,QAAgC;QAEhC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAC5B,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,GAAG;QACP,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YACjB,IAAI;gBACF,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;gBACpB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC;gBAEjC,IAAI;oBACF,MAAM,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,UAAU,CAAC,0BAAkB,CAAC,CAAC,CAAC;iBACrE;gBAAC,OAAO,GAAG,EAAE;oBACZ,IAAI,CAAC,+BAAuB,CAAC,IAAI,CAAS,GAAI,CAAC,OAAO,CAAC,EAAE;wBACvD,MAAM,GAAG,CAAC;qBACX;iBACF;gBAED,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;aAClC;YAAC,OAAO,KAAK,EAAE;gBACd,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;gBACrB,MAAM,KAAK,CAAC;aACb;SACF;aAAM;YACL,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;SACrD;IACH,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,MAAmB;QAC7C,MAAM,IAAI,GAAuB,IAAI,CAAC,IAAI,CAAC;QAE3C,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;QAC7B,IAAI,EAAE,GAAG,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;QAEjC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE;YACpB,mGAAmG;YACnG,MAAM,IAAI,GAAkB,MAAM,IAAI,CAAC,oBAAoB,CAAC,GAAG,EAAE,CAC/D,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,eAAgB,EAAE,SAAS,EAAE,GAAG,EAAE,EAAE,CAAC,CACjE,CAAC;YACF,IAAI,IAAI,EAAE;gBACR,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;gBACvB,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;oBACtC,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBAClB,MAAM,IAAI,GAAG,IAAA,iBAAS,EAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBAErC,EAAE;oBACF,gEAAgE;oBAChE,sBAAsB;oBACtB,QAAQ,IAAI,CAAC,KAAK,EAAE;wBAClB,KAAK,UAAU;4BACb,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;4BAClC,MAAM;wBACR,KAAK,WAAW;4BACd,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;4BAChD,MAAM;qBACT;oBAED,MAAM,EAAE,KAAK,KAAkB,IAAI,EAAjB,QAAQ,kBAAK,IAAI,EAA7B,SAAsB,CAAO,CAAC;oBAEpC,IAAI,KAAK,KAAK,SAAS,EAAE;wBACvB,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;qBACtB;yBAAM;wBACL,IAAI,CAAC,IAAI,CAAC,KAAY,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC;wBACtC,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,IAAI,QAAQ,CAAC,KAAK,EAAS,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC;qBAC9D;iBACF;aACF;SACF;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YACjB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;SAClC;QACD,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;CACF;AAlJD,kCAkJC"}
|
||||
+405
@@ -0,0 +1,405 @@
|
||||
/*eslint-env node */
|
||||
'use strict';
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.QueueGetters = void 0;
|
||||
const queue_base_1 = require("./queue-base");
|
||||
const job_1 = require("./job");
|
||||
const utils_1 = require("../utils");
|
||||
/**
|
||||
*
|
||||
* @class QueueGetters
|
||||
* @extends QueueBase
|
||||
*
|
||||
* @description Provides different getters for different aspects of a queue.
|
||||
*/
|
||||
class QueueGetters extends queue_base_1.QueueBase {
|
||||
getJob(jobId) {
|
||||
return this.Job.fromId(this, jobId);
|
||||
}
|
||||
commandByType(types, count, callback) {
|
||||
return types.map((type) => {
|
||||
type = type === 'waiting' ? 'wait' : type; // alias
|
||||
const key = this.toKey(type);
|
||||
switch (type) {
|
||||
case 'completed':
|
||||
case 'failed':
|
||||
case 'delayed':
|
||||
case 'prioritized':
|
||||
case 'repeat':
|
||||
case 'waiting-children':
|
||||
return callback(key, count ? 'zcard' : 'zrange');
|
||||
case 'active':
|
||||
case 'wait':
|
||||
case 'paused':
|
||||
return callback(key, count ? 'llen' : 'lrange');
|
||||
}
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Helper to easily extend Job class calls.
|
||||
*/
|
||||
get Job() {
|
||||
return job_1.Job;
|
||||
}
|
||||
sanitizeJobTypes(types) {
|
||||
const currentTypes = typeof types === 'string' ? [types] : types;
|
||||
if (Array.isArray(currentTypes) && currentTypes.length > 0) {
|
||||
const sanitizedTypes = [...currentTypes];
|
||||
if (sanitizedTypes.indexOf('waiting') !== -1) {
|
||||
sanitizedTypes.push('paused');
|
||||
}
|
||||
return [...new Set(sanitizedTypes)];
|
||||
}
|
||||
return [
|
||||
'active',
|
||||
'completed',
|
||||
'delayed',
|
||||
'failed',
|
||||
'paused',
|
||||
'prioritized',
|
||||
'waiting',
|
||||
'waiting-children',
|
||||
];
|
||||
}
|
||||
/**
|
||||
Returns the number of jobs waiting to be processed. This includes jobs that are
|
||||
"waiting" or "delayed" or "prioritized" or "waiting-children".
|
||||
*/
|
||||
async count() {
|
||||
const count = await this.getJobCountByTypes('waiting', 'paused', 'delayed', 'prioritized', 'waiting-children');
|
||||
return count;
|
||||
}
|
||||
/**
|
||||
* Returns the time to live for a rate limited key in milliseconds.
|
||||
* @returns -2 if the key does not exist.
|
||||
* -1 if the key exists but has no associated expire.
|
||||
* @see {@link https://redis.io/commands/pttl/}
|
||||
*/
|
||||
async getRateLimitTtl() {
|
||||
const client = await this.client;
|
||||
return client.pttl(this.keys.limiter);
|
||||
}
|
||||
/**
|
||||
* Job counts by type
|
||||
*
|
||||
* Queue#getJobCountByTypes('completed') => completed count
|
||||
* Queue#getJobCountByTypes('completed,failed') => completed + failed count
|
||||
* Queue#getJobCountByTypes('completed', 'failed') => completed + failed count
|
||||
* Queue#getJobCountByTypes('completed', 'waiting', 'failed') => completed + waiting + failed count
|
||||
*/
|
||||
async getJobCountByTypes(...types) {
|
||||
const result = await this.getJobCounts(...types);
|
||||
return Object.values(result).reduce((sum, count) => sum + count, 0);
|
||||
}
|
||||
/**
|
||||
* Returns the job counts for each type specified or every list/set in the queue by default.
|
||||
*
|
||||
* @returns An object, key (type) and value (count)
|
||||
*/
|
||||
async getJobCounts(...types) {
|
||||
const currentTypes = this.sanitizeJobTypes(types);
|
||||
const responses = await this.scripts.getCounts(currentTypes);
|
||||
const counts = {};
|
||||
responses.forEach((res, index) => {
|
||||
counts[currentTypes[index]] = res || 0;
|
||||
});
|
||||
return counts;
|
||||
}
|
||||
/**
|
||||
* Get current job state.
|
||||
*
|
||||
* @returns Returns one of these values:
|
||||
* 'completed', 'failed', 'delayed', 'active', 'waiting', 'waiting-children', 'unknown'.
|
||||
*/
|
||||
getJobState(jobId) {
|
||||
return this.scripts.getState(jobId);
|
||||
}
|
||||
/**
|
||||
* Returns the number of jobs in completed status.
|
||||
*/
|
||||
getCompletedCount() {
|
||||
return this.getJobCountByTypes('completed');
|
||||
}
|
||||
/**
|
||||
* Returns the number of jobs in failed status.
|
||||
*/
|
||||
getFailedCount() {
|
||||
return this.getJobCountByTypes('failed');
|
||||
}
|
||||
/**
|
||||
* Returns the number of jobs in delayed status.
|
||||
*/
|
||||
getDelayedCount() {
|
||||
return this.getJobCountByTypes('delayed');
|
||||
}
|
||||
/**
|
||||
* Returns the number of jobs in active status.
|
||||
*/
|
||||
getActiveCount() {
|
||||
return this.getJobCountByTypes('active');
|
||||
}
|
||||
/**
|
||||
* Returns the number of jobs in prioritized status.
|
||||
*/
|
||||
getPrioritizedCount() {
|
||||
return this.getJobCountByTypes('prioritized');
|
||||
}
|
||||
/**
|
||||
* Returns the number of jobs in waiting or paused statuses.
|
||||
*/
|
||||
getWaitingCount() {
|
||||
return this.getJobCountByTypes('waiting');
|
||||
}
|
||||
/**
|
||||
* Returns the number of jobs in waiting-children status.
|
||||
*/
|
||||
getWaitingChildrenCount() {
|
||||
return this.getJobCountByTypes('waiting-children');
|
||||
}
|
||||
/**
|
||||
* Returns the jobs that are in the "waiting" status.
|
||||
* @param start - zero based index from where to start returning jobs.
|
||||
* @param end - zero based index where to stop returning jobs.
|
||||
*/
|
||||
getWaiting(start = 0, end = -1) {
|
||||
return this.getJobs(['waiting'], start, end, true);
|
||||
}
|
||||
/**
|
||||
* Returns the jobs that are in the "waiting-children" status.
|
||||
* I.E. parent jobs that have at least one child that has not completed yet.
|
||||
* @param start - zero based index from where to start returning jobs.
|
||||
* @param end - zero based index where to stop returning jobs.
|
||||
*/
|
||||
getWaitingChildren(start = 0, end = -1) {
|
||||
return this.getJobs(['waiting-children'], start, end, true);
|
||||
}
|
||||
/**
|
||||
* Returns the jobs that are in the "active" status.
|
||||
* @param start - zero based index from where to start returning jobs.
|
||||
* @param end - zero based index where to stop returning jobs.
|
||||
*/
|
||||
getActive(start = 0, end = -1) {
|
||||
return this.getJobs(['active'], start, end, true);
|
||||
}
|
||||
/**
|
||||
* Returns the jobs that are in the "delayed" status.
|
||||
* @param start - zero based index from where to start returning jobs.
|
||||
* @param end - zero based index where to stop returning jobs.
|
||||
*/
|
||||
getDelayed(start = 0, end = -1) {
|
||||
return this.getJobs(['delayed'], start, end, true);
|
||||
}
|
||||
/**
|
||||
* Returns the jobs that are in the "prioritized" status.
|
||||
* @param start - zero based index from where to start returning jobs.
|
||||
* @param end - zero based index where to stop returning jobs.
|
||||
*/
|
||||
getPrioritized(start = 0, end = -1) {
|
||||
return this.getJobs(['prioritized'], start, end, true);
|
||||
}
|
||||
/**
|
||||
* Returns the jobs that are in the "completed" status.
|
||||
* @param start - zero based index from where to start returning jobs.
|
||||
* @param end - zero based index where to stop returning jobs.
|
||||
*/
|
||||
getCompleted(start = 0, end = -1) {
|
||||
return this.getJobs(['completed'], start, end, false);
|
||||
}
|
||||
/**
|
||||
* Returns the jobs that are in the "failed" status.
|
||||
* @param start - zero based index from where to start returning jobs.
|
||||
* @param end - zero based index where to stop returning jobs.
|
||||
*/
|
||||
getFailed(start = 0, end = -1) {
|
||||
return this.getJobs(['failed'], start, end, false);
|
||||
}
|
||||
/**
|
||||
* Returns the qualified job ids and the raw job data (if available) of the
|
||||
* children jobs of the given parent job.
|
||||
* It is possible to get either the already processed children, in this case
|
||||
* an array of qualified job ids and their result values will be returned,
|
||||
* or the pending children, in this case an array of qualified job ids will
|
||||
* be returned.
|
||||
* A qualified job id is a string representing the job id in a given queue,
|
||||
* for example: "bull:myqueue:jobid".
|
||||
*
|
||||
* @param parentId The id of the parent job
|
||||
* @param type "processed" | "pending"
|
||||
* @param opts
|
||||
*
|
||||
* @returns { items: { id: string, v?: any, err?: string } [], jobs: JobJsonRaw[], total: number}
|
||||
*/
|
||||
async getDependencies(parentId, type, start, end) {
|
||||
const key = this.toKey(type == 'processed'
|
||||
? `${parentId}:processed`
|
||||
: `${parentId}:dependencies`);
|
||||
const { items, total, jobs } = await this.scripts.paginate(key, {
|
||||
start,
|
||||
end,
|
||||
fetchJobs: true,
|
||||
});
|
||||
return {
|
||||
items,
|
||||
jobs,
|
||||
total,
|
||||
};
|
||||
}
|
||||
async getRanges(types, start = 0, end = 1, asc = false) {
|
||||
const multiCommands = [];
|
||||
this.commandByType(types, false, (key, command) => {
|
||||
switch (command) {
|
||||
case 'lrange':
|
||||
multiCommands.push('lrange');
|
||||
break;
|
||||
case 'zrange':
|
||||
multiCommands.push('zrange');
|
||||
break;
|
||||
}
|
||||
});
|
||||
const responses = await this.scripts.getRanges(types, start, end, asc);
|
||||
let results = [];
|
||||
responses.forEach((response, index) => {
|
||||
const result = response || [];
|
||||
if (asc && multiCommands[index] === 'lrange') {
|
||||
results = results.concat(result.reverse());
|
||||
}
|
||||
else {
|
||||
results = results.concat(result);
|
||||
}
|
||||
});
|
||||
return [...new Set(results)];
|
||||
}
|
||||
/**
|
||||
* Returns the jobs that are on the given statuses (note that JobType is synonym for job status)
|
||||
* @param types - the statuses of the jobs to return.
|
||||
* @param start - zero based index from where to start returning jobs.
|
||||
* @param end - zero based index where to stop returning jobs.
|
||||
* @param asc - if true, the jobs will be returned in ascending order.
|
||||
*/
|
||||
async getJobs(types, start = 0, end = -1, asc = false) {
|
||||
const currentTypes = this.sanitizeJobTypes(types);
|
||||
const jobIds = await this.getRanges(currentTypes, start, end, asc);
|
||||
return Promise.all(jobIds.map(jobId => this.Job.fromId(this, jobId)));
|
||||
}
|
||||
/**
|
||||
* Returns the logs for a given Job.
|
||||
* @param jobId - the id of the job to get the logs for.
|
||||
* @param start - zero based index from where to start returning jobs.
|
||||
* @param end - zero based index where to stop returning jobs.
|
||||
* @param asc - if true, the jobs will be returned in ascending order.
|
||||
*/
|
||||
async getJobLogs(jobId, start = 0, end = -1, asc = true) {
|
||||
const client = await this.client;
|
||||
const multi = client.multi();
|
||||
const logsKey = this.toKey(jobId + ':logs');
|
||||
if (asc) {
|
||||
multi.lrange(logsKey, start, end);
|
||||
}
|
||||
else {
|
||||
multi.lrange(logsKey, -(end + 1), -(start + 1));
|
||||
}
|
||||
multi.llen(logsKey);
|
||||
const result = (await multi.exec());
|
||||
if (!asc) {
|
||||
result[0][1].reverse();
|
||||
}
|
||||
return {
|
||||
logs: result[0][1],
|
||||
count: result[1][1],
|
||||
};
|
||||
}
|
||||
async baseGetClients(suffix) {
|
||||
const client = await this.client;
|
||||
const clients = (await client.client('LIST'));
|
||||
try {
|
||||
const list = this.parseClientList(clients, suffix);
|
||||
return list;
|
||||
}
|
||||
catch (err) {
|
||||
if (!utils_1.clientCommandMessageReg.test(err.message)) {
|
||||
throw err;
|
||||
}
|
||||
return [];
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Get the worker list related to the queue. i.e. all the known
|
||||
* workers that are available to process jobs for this queue.
|
||||
* Note: GCP does not support SETNAME, so this call will not work
|
||||
*
|
||||
* @returns - Returns an array with workers info.
|
||||
*/
|
||||
getWorkers() {
|
||||
return this.baseGetClients(utils_1.WORKER_SUFFIX);
|
||||
}
|
||||
/**
|
||||
* Get queue events list related to the queue.
|
||||
* Note: GCP does not support SETNAME, so this call will not work
|
||||
*
|
||||
* @returns - Returns an array with queue events info.
|
||||
*/
|
||||
async getQueueEvents() {
|
||||
return this.baseGetClients(utils_1.QUEUE_EVENT_SUFFIX);
|
||||
}
|
||||
/**
|
||||
* Get queue metrics related to the queue.
|
||||
*
|
||||
* This method returns the gathered metrics for the queue.
|
||||
* The metrics are represented as an array of job counts
|
||||
* per unit of time (1 minute).
|
||||
*
|
||||
* @param start - Start point of the metrics, where 0
|
||||
* is the newest point to be returned.
|
||||
* @param end - End point of the metrics, where -1 is the
|
||||
* oldest point to be returned.
|
||||
*
|
||||
* @returns - Returns an object with queue metrics.
|
||||
*/
|
||||
async getMetrics(type, start = 0, end = -1) {
|
||||
const client = await this.client;
|
||||
const metricsKey = this.toKey(`metrics:${type}`);
|
||||
const dataKey = `${metricsKey}:data`;
|
||||
const multi = client.multi();
|
||||
multi.hmget(metricsKey, 'count', 'prevTS', 'prevCount');
|
||||
multi.lrange(dataKey, start, end);
|
||||
multi.llen(dataKey);
|
||||
const [hmget, range, len] = (await multi.exec());
|
||||
const [err, [count, prevTS, prevCount]] = hmget;
|
||||
const [err2, data] = range;
|
||||
const [err3, numPoints] = len;
|
||||
if (err || err2) {
|
||||
throw err || err2 || err3;
|
||||
}
|
||||
return {
|
||||
meta: {
|
||||
count: parseInt(count || '0', 10),
|
||||
prevTS: parseInt(prevTS || '0', 10),
|
||||
prevCount: parseInt(prevCount || '0', 10),
|
||||
},
|
||||
data,
|
||||
count: numPoints,
|
||||
};
|
||||
}
|
||||
parseClientList(list, suffix = '') {
|
||||
const lines = list.split('\n');
|
||||
const clients = [];
|
||||
lines.forEach((line) => {
|
||||
const client = {};
|
||||
const keyValues = line.split(' ');
|
||||
keyValues.forEach(function (keyValue) {
|
||||
const index = keyValue.indexOf('=');
|
||||
const key = keyValue.substring(0, index);
|
||||
const value = keyValue.substring(index + 1);
|
||||
client[key] = value;
|
||||
});
|
||||
const name = client['name'];
|
||||
if (name && name === `${this.clientName()}${suffix ? `${suffix}` : ''}`) {
|
||||
client['name'] = this.name;
|
||||
clients.push(client);
|
||||
}
|
||||
});
|
||||
return clients;
|
||||
}
|
||||
}
|
||||
exports.QueueGetters = QueueGetters;
|
||||
//# sourceMappingURL=queue-getters.js.map
|
||||
+1
File diff suppressed because one or more lines are too long
+41
@@ -0,0 +1,41 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.QueueKeys = void 0;
|
||||
class QueueKeys {
|
||||
constructor(prefix = 'bull') {
|
||||
this.prefix = prefix;
|
||||
}
|
||||
getKeys(name) {
|
||||
const keys = {};
|
||||
[
|
||||
'',
|
||||
'active',
|
||||
'wait',
|
||||
'waiting-children',
|
||||
'paused',
|
||||
'id',
|
||||
'delayed',
|
||||
'prioritized',
|
||||
'stalled-check',
|
||||
'completed',
|
||||
'failed',
|
||||
'stalled',
|
||||
'repeat',
|
||||
'limiter',
|
||||
'meta',
|
||||
'events',
|
||||
'pc',
|
||||
].forEach(key => {
|
||||
keys[key] = this.toKey(name, key);
|
||||
});
|
||||
return keys;
|
||||
}
|
||||
toKey(name, type) {
|
||||
return `${this.getQueueQualifiedName(name)}:${type}`;
|
||||
}
|
||||
getQueueQualifiedName(name) {
|
||||
return `${this.prefix}:${name}`;
|
||||
}
|
||||
}
|
||||
exports.QueueKeys = QueueKeys;
|
||||
//# sourceMappingURL=queue-keys.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"queue-keys.js","sourceRoot":"","sources":["../../../src/classes/queue-keys.ts"],"names":[],"mappings":";;;AAEA,MAAa,SAAS;IACpB,YAA4B,SAAiB,MAAM;QAAvB,WAAM,GAAN,MAAM,CAAiB;IAAG,CAAC;IAEvD,OAAO,CAAC,IAAY;QAClB,MAAM,IAAI,GAAgC,EAAE,CAAC;QAC7C;YACE,EAAE;YACF,QAAQ;YACR,MAAM;YACN,kBAAkB;YAClB,QAAQ;YACR,IAAI;YACJ,SAAS;YACT,aAAa;YACb,eAAe;YACf,WAAW;YACX,QAAQ;YACR,SAAS;YACT,QAAQ;YACR,SAAS;YACT,MAAM;YACN,QAAQ;YACR,IAAI;SACL,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;YACd,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,IAAY,EAAE,IAAY;QAC9B,OAAO,GAAG,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;IACvD,CAAC;IAED,qBAAqB,CAAC,IAAY;QAChC,OAAO,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC;IAClC,CAAC;CACF;AArCD,8BAqCC"}
|
||||
+344
@@ -0,0 +1,344 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.Queue = void 0;
|
||||
const lodash_1 = require("lodash");
|
||||
const uuid_1 = require("uuid");
|
||||
const job_1 = require("./job");
|
||||
const queue_getters_1 = require("./queue-getters");
|
||||
const repeat_1 = require("./repeat");
|
||||
const version_1 = require("../version");
|
||||
/**
|
||||
* Queue
|
||||
*
|
||||
* This class provides methods to add jobs to a queue and some othe high-level
|
||||
* administration such as pausing or deleting queues.
|
||||
*
|
||||
*/
|
||||
class Queue extends queue_getters_1.QueueGetters {
|
||||
constructor(name, opts, Connection) {
|
||||
var _a;
|
||||
super(name, Object.assign({ blockingConnection: false }, opts), Connection);
|
||||
this.token = (0, uuid_1.v4)();
|
||||
this.libName = 'bullmq';
|
||||
this.jobsOpts = (_a = (0, lodash_1.get)(opts, 'defaultJobOptions')) !== null && _a !== void 0 ? _a : {};
|
||||
this.waitUntilReady()
|
||||
.then(client => {
|
||||
if (!this.closing && !(opts === null || opts === void 0 ? void 0 : opts.skipMetasUpdate)) {
|
||||
return client.hmset(this.keys.meta, this.metaValues);
|
||||
}
|
||||
})
|
||||
.catch(err => {
|
||||
// We ignore this error to avoid warnings. The error can still
|
||||
// be received by listening to event 'error'
|
||||
});
|
||||
}
|
||||
emit(event, ...args) {
|
||||
return super.emit(event, ...args);
|
||||
}
|
||||
off(eventName, listener) {
|
||||
super.off(eventName, listener);
|
||||
return this;
|
||||
}
|
||||
on(event, listener) {
|
||||
super.on(event, listener);
|
||||
return this;
|
||||
}
|
||||
once(event, listener) {
|
||||
super.once(event, listener);
|
||||
return this;
|
||||
}
|
||||
/**
|
||||
* Returns this instance current default job options.
|
||||
*/
|
||||
get defaultJobOptions() {
|
||||
return Object.assign({}, this.jobsOpts);
|
||||
}
|
||||
get metaValues() {
|
||||
var _a, _b, _c, _d;
|
||||
return {
|
||||
'opts.maxLenEvents': (_d = (_c = (_b = (_a = this.opts) === null || _a === void 0 ? void 0 : _a.streams) === null || _b === void 0 ? void 0 : _b.events) === null || _c === void 0 ? void 0 : _c.maxLen) !== null && _d !== void 0 ? _d : 10000,
|
||||
version: `${this.libName}:${version_1.version}`,
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Get library version.
|
||||
*
|
||||
* @returns the content of the meta.library field.
|
||||
*/
|
||||
async getVersion() {
|
||||
const client = await this.client;
|
||||
return await client.hget(this.keys.meta, 'version');
|
||||
}
|
||||
get repeat() {
|
||||
return new Promise(async (resolve) => {
|
||||
if (!this._repeat) {
|
||||
this._repeat = new repeat_1.Repeat(this.name, Object.assign(Object.assign({}, this.opts), { connection: await this.client }));
|
||||
this._repeat.on('error', e => this.emit.bind(this, e));
|
||||
}
|
||||
resolve(this._repeat);
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Adds a new job to the queue.
|
||||
*
|
||||
* @param name - Name of the job to be added to the queue,.
|
||||
* @param data - Arbitrary data to append to the job.
|
||||
* @param opts - Job options that affects how the job is going to be processed.
|
||||
*/
|
||||
async add(name, data, opts) {
|
||||
if (opts && opts.repeat) {
|
||||
return (await this.repeat).addNextRepeatableJob(name, data, Object.assign(Object.assign({}, this.jobsOpts), opts), true);
|
||||
}
|
||||
else {
|
||||
const jobId = opts === null || opts === void 0 ? void 0 : opts.jobId;
|
||||
if (jobId == '0' || (jobId === null || jobId === void 0 ? void 0 : jobId.startsWith('0:'))) {
|
||||
throw new Error("JobId cannot be '0' or start with 0:");
|
||||
}
|
||||
const job = await this.Job.create(this, name, data, Object.assign(Object.assign(Object.assign({}, this.jobsOpts), opts), { jobId }));
|
||||
this.emit('waiting', job);
|
||||
return job;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Adds an array of jobs to the queue. This method may be faster than adding
|
||||
* one job at a time in a sequence.
|
||||
*
|
||||
* @param jobs - The array of jobs to add to the queue. Each job is defined by 3
|
||||
* properties, 'name', 'data' and 'opts'. They follow the same signature as 'Queue.add'.
|
||||
*/
|
||||
addBulk(jobs) {
|
||||
return this.Job.createBulk(this, jobs.map(job => {
|
||||
var _a;
|
||||
return ({
|
||||
name: job.name,
|
||||
data: job.data,
|
||||
opts: Object.assign(Object.assign(Object.assign({}, this.jobsOpts), job.opts), { jobId: (_a = job.opts) === null || _a === void 0 ? void 0 : _a.jobId }),
|
||||
});
|
||||
}));
|
||||
}
|
||||
/**
|
||||
* Pauses the processing of this queue globally.
|
||||
*
|
||||
* We use an atomic RENAME operation on the wait queue. Since
|
||||
* we have blocking calls with BRPOPLPUSH on the wait queue, as long as the queue
|
||||
* is renamed to 'paused', no new jobs will be processed (the current ones
|
||||
* will run until finalized).
|
||||
*
|
||||
* Adding jobs requires a LUA script to check first if the paused list exist
|
||||
* and in that case it will add it there instead of the wait list.
|
||||
*/
|
||||
async pause() {
|
||||
await this.scripts.pause(true);
|
||||
this.emit('paused');
|
||||
}
|
||||
/**
|
||||
* Close the queue instance.
|
||||
*
|
||||
*/
|
||||
async close() {
|
||||
if (!this.closing) {
|
||||
if (this._repeat) {
|
||||
await this._repeat.close();
|
||||
}
|
||||
}
|
||||
return super.close();
|
||||
}
|
||||
/**
|
||||
* Resumes the processing of this queue globally.
|
||||
*
|
||||
* The method reverses the pause operation by resuming the processing of the
|
||||
* queue.
|
||||
*/
|
||||
async resume() {
|
||||
await this.scripts.pause(false);
|
||||
this.emit('resumed');
|
||||
}
|
||||
/**
|
||||
* Returns true if the queue is currently paused.
|
||||
*/
|
||||
async isPaused() {
|
||||
const client = await this.client;
|
||||
const pausedKeyExists = await client.hexists(this.keys.meta, 'paused');
|
||||
return pausedKeyExists === 1;
|
||||
}
|
||||
/**
|
||||
* Get all repeatable meta jobs.
|
||||
*
|
||||
* @param start - Offset of first job to return.
|
||||
* @param end - Offset of last job to return.
|
||||
* @param asc - Determine the order in which jobs are returned based on their
|
||||
* next execution time.
|
||||
*/
|
||||
async getRepeatableJobs(start, end, asc) {
|
||||
return (await this.repeat).getRepeatableJobs(start, end, asc);
|
||||
}
|
||||
/**
|
||||
* Removes a repeatable job.
|
||||
*
|
||||
* Note: you need to use the exact same repeatOpts when deleting a repeatable job
|
||||
* than when adding it.
|
||||
*
|
||||
* @see removeRepeatableByKey
|
||||
*
|
||||
* @param name - job name
|
||||
* @param repeatOpts -
|
||||
* @param jobId -
|
||||
* @returns
|
||||
*/
|
||||
async removeRepeatable(name, repeatOpts, jobId) {
|
||||
const repeat = await this.repeat;
|
||||
const removed = await repeat.removeRepeatable(name, repeatOpts, jobId);
|
||||
return !removed;
|
||||
}
|
||||
/**
|
||||
* Removes a repeatable job by its key. Note that the key is the one used
|
||||
* to store the repeatable job metadata and not one of the job iterations
|
||||
* themselves. You can use "getRepeatableJobs" in order to get the keys.
|
||||
*
|
||||
* @see getRepeatableJobs
|
||||
*
|
||||
* @param repeatJobKey - to the repeatable job.
|
||||
* @returns
|
||||
*/
|
||||
async removeRepeatableByKey(key) {
|
||||
const repeat = await this.repeat;
|
||||
const removed = await repeat.removeRepeatableByKey(key);
|
||||
return !removed;
|
||||
}
|
||||
/**
|
||||
* Removes the given job from the queue as well as all its
|
||||
* dependencies.
|
||||
*
|
||||
* @param jobId - The id of the job to remove
|
||||
* @param opts - Options to remove a job
|
||||
* @returns 1 if it managed to remove the job or 0 if the job or
|
||||
* any of its dependencies were locked.
|
||||
*/
|
||||
remove(jobId, { removeChildren = true } = {}) {
|
||||
return this.scripts.remove(jobId, removeChildren);
|
||||
}
|
||||
/**
|
||||
* Updates the given job's progress.
|
||||
*
|
||||
* @param jobId - The id of the job to update
|
||||
* @param progress - number or object to be saved as progress.
|
||||
*/
|
||||
async updateJobProgress(jobId, progress) {
|
||||
return this.scripts.updateProgress(jobId, progress);
|
||||
}
|
||||
/**
|
||||
* Logs one row of job's log data.
|
||||
*
|
||||
* @param jobId - The job id to log against.
|
||||
* @param logRow - string with log data to be logged.
|
||||
* @param keepLogs - max number of log entries to keep (0 for unlimited).
|
||||
*
|
||||
* @returns The total number of log entries for this job so far.
|
||||
*/
|
||||
async addJobLog(jobId, logRow, keepLogs) {
|
||||
return job_1.Job.addJobLog(this, jobId, logRow, keepLogs);
|
||||
}
|
||||
/**
|
||||
* Drains the queue, i.e., removes all jobs that are waiting
|
||||
* or delayed, but not active, completed or failed.
|
||||
*
|
||||
* @param delayed - Pass true if it should also clean the
|
||||
* delayed jobs.
|
||||
*/
|
||||
drain(delayed = false) {
|
||||
return this.scripts.drain(delayed);
|
||||
}
|
||||
/**
|
||||
* Cleans jobs from a queue. Similar to drain but keeps jobs within a certain
|
||||
* grace period.
|
||||
*
|
||||
* @param grace - The grace period
|
||||
* @param limit - Max number of jobs to clean
|
||||
* @param type - The type of job to clean
|
||||
* Possible values are completed, wait, active, paused, delayed, failed. Defaults to completed.
|
||||
* @returns Id jobs from the deleted records
|
||||
*/
|
||||
async clean(grace, limit, type = 'completed') {
|
||||
const maxCount = limit || Infinity;
|
||||
const maxCountPerCall = Math.min(10000, maxCount);
|
||||
const timestamp = Date.now() - grace;
|
||||
let deletedCount = 0;
|
||||
const deletedJobsIds = [];
|
||||
while (deletedCount < maxCount) {
|
||||
const jobsIds = await this.scripts.cleanJobsInSet(type, timestamp, maxCountPerCall);
|
||||
this.emit('cleaned', jobsIds, type);
|
||||
deletedCount += jobsIds.length;
|
||||
deletedJobsIds.push(...jobsIds);
|
||||
if (jobsIds.length < maxCountPerCall) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return deletedJobsIds;
|
||||
}
|
||||
/**
|
||||
* Completely destroys the queue and all of its contents irreversibly.
|
||||
* This method will the *pause* the queue and requires that there are no
|
||||
* active jobs. It is possible to bypass this requirement, i.e. not
|
||||
* having active jobs using the "force" option.
|
||||
*
|
||||
* Note: This operation requires to iterate on all the jobs stored in the queue
|
||||
* and can be slow for very large queues.
|
||||
*
|
||||
* @param opts - Obliterate options.
|
||||
*/
|
||||
async obliterate(opts) {
|
||||
await this.pause();
|
||||
let cursor = 0;
|
||||
do {
|
||||
cursor = await this.scripts.obliterate(Object.assign({ force: false, count: 1000 }, opts));
|
||||
} while (cursor);
|
||||
}
|
||||
/**
|
||||
* Retry all the failed jobs.
|
||||
*
|
||||
* @param opts: { count: number; state: FinishedStatus; timestamp: number}
|
||||
* - count number to limit how many jobs will be moved to wait status per iteration,
|
||||
* - state failed by default or completed.
|
||||
* - timestamp from which timestamp to start moving jobs to wait status, default Date.now().
|
||||
*
|
||||
* @returns
|
||||
*/
|
||||
async retryJobs(opts = {}) {
|
||||
let cursor = 0;
|
||||
do {
|
||||
cursor = await this.scripts.retryJobs(opts.state, opts.count, opts.timestamp);
|
||||
} while (cursor);
|
||||
}
|
||||
/**
|
||||
* Promote all the delayed jobs.
|
||||
*
|
||||
* @param opts: { count: number }
|
||||
* - count number to limit how many jobs will be moved to wait status per iteration
|
||||
*
|
||||
* @returns
|
||||
*/
|
||||
async promoteJobs(opts = {}) {
|
||||
let cursor = 0;
|
||||
do {
|
||||
cursor = await this.scripts.promoteJobs(opts.count);
|
||||
} while (cursor);
|
||||
}
|
||||
/**
|
||||
* Trim the event stream to an approximately maxLength.
|
||||
*
|
||||
* @param maxLength -
|
||||
*/
|
||||
async trimEvents(maxLength) {
|
||||
const client = await this.client;
|
||||
return client.xtrim(this.keys.events, 'MAXLEN', '~', maxLength);
|
||||
}
|
||||
/**
|
||||
* Delete old priority helper key.
|
||||
*/
|
||||
async removeDeprecatedPriorityKey() {
|
||||
const client = await this.client;
|
||||
return client.del(this.toKey('priority'));
|
||||
}
|
||||
}
|
||||
exports.Queue = Queue;
|
||||
//# sourceMappingURL=queue.js.map
|
||||
+1
File diff suppressed because one or more lines are too long
+233
@@ -0,0 +1,233 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.RedisConnection = void 0;
|
||||
const events_1 = require("events");
|
||||
const ioredis_1 = require("ioredis");
|
||||
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
||||
// @ts-ignore
|
||||
const utils_1 = require("ioredis/built/utils");
|
||||
const utils_2 = require("../utils");
|
||||
const version_1 = require("../version");
|
||||
const scripts = require("../scripts");
|
||||
const overrideMessage = [
|
||||
'BullMQ: WARNING! Your redis options maxRetriesPerRequest must be null',
|
||||
'and will be overridden by BullMQ.',
|
||||
].join(' ');
|
||||
const deprecationMessage = [
|
||||
'BullMQ: DEPRECATION WARNING! Your redis options maxRetriesPerRequest must be null.',
|
||||
'On the next versions having this settings will throw an exception',
|
||||
].join(' ');
|
||||
class RedisConnection extends events_1.EventEmitter {
|
||||
constructor(opts, shared = false, blocking = true, skipVersionCheck = false) {
|
||||
super();
|
||||
this.shared = shared;
|
||||
this.blocking = blocking;
|
||||
this.capabilities = {
|
||||
canDoubleTimeout: false,
|
||||
};
|
||||
if (!(0, utils_2.isRedisInstance)(opts)) {
|
||||
this.checkBlockingOptions(overrideMessage, opts);
|
||||
this.opts = Object.assign({ port: 6379, host: '127.0.0.1', retryStrategy: function (times) {
|
||||
return Math.max(Math.min(Math.exp(times), 20000), 1000);
|
||||
} }, opts);
|
||||
if (this.blocking) {
|
||||
this.opts.maxRetriesPerRequest = null;
|
||||
}
|
||||
}
|
||||
else {
|
||||
this._client = opts;
|
||||
// Test if the redis instance is using keyPrefix
|
||||
// and if so, throw an error.
|
||||
if (this._client.options.keyPrefix) {
|
||||
throw new Error('BullMQ: ioredis does not support ioredis prefixes, use the prefix option instead.');
|
||||
}
|
||||
if ((0, utils_2.isRedisCluster)(this._client)) {
|
||||
this.opts = this._client.options.redisOptions;
|
||||
}
|
||||
else {
|
||||
this.opts = this._client.options;
|
||||
}
|
||||
this.checkBlockingOptions(deprecationMessage, this.opts);
|
||||
}
|
||||
this.skipVersionCheck =
|
||||
skipVersionCheck || !!(this.opts && this.opts.skipVersionCheck);
|
||||
this.handleClientError = (err) => {
|
||||
this.emit('error', err);
|
||||
};
|
||||
this.handleClientClose = () => {
|
||||
this.emit('close');
|
||||
};
|
||||
this.handleClientReady = () => {
|
||||
this.emit('ready');
|
||||
};
|
||||
this.initializing = this.init();
|
||||
this.initializing.catch(err => this.emit('error', err));
|
||||
}
|
||||
checkBlockingOptions(msg, options) {
|
||||
if (this.blocking && options && options.maxRetriesPerRequest) {
|
||||
console.error(msg);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Waits for a redis client to be ready.
|
||||
* @param redis - client
|
||||
*/
|
||||
static async waitUntilReady(client) {
|
||||
if (client.status === 'ready') {
|
||||
return;
|
||||
}
|
||||
if (client.status === 'wait') {
|
||||
return client.connect();
|
||||
}
|
||||
if (client.status === 'end') {
|
||||
throw new Error(utils_1.CONNECTION_CLOSED_ERROR_MSG);
|
||||
}
|
||||
let handleReady;
|
||||
let handleEnd;
|
||||
let handleError;
|
||||
try {
|
||||
await new Promise((resolve, reject) => {
|
||||
let lastError;
|
||||
handleError = (err) => {
|
||||
lastError = err;
|
||||
};
|
||||
handleReady = () => {
|
||||
resolve();
|
||||
};
|
||||
handleEnd = () => {
|
||||
reject(lastError || new Error(utils_1.CONNECTION_CLOSED_ERROR_MSG));
|
||||
};
|
||||
(0, utils_2.increaseMaxListeners)(client, 3);
|
||||
client.once('ready', handleReady);
|
||||
client.on('end', handleEnd);
|
||||
client.once('error', handleError);
|
||||
});
|
||||
}
|
||||
finally {
|
||||
client.removeListener('end', handleEnd);
|
||||
client.removeListener('error', handleError);
|
||||
client.removeListener('ready', handleReady);
|
||||
(0, utils_2.decreaseMaxListeners)(client, 3);
|
||||
}
|
||||
}
|
||||
get client() {
|
||||
return this.initializing;
|
||||
}
|
||||
loadCommands(version, providedScripts) {
|
||||
const finalScripts = providedScripts || scripts;
|
||||
for (const property in finalScripts) {
|
||||
// Only define the command if not already defined
|
||||
const commandName = `${finalScripts[property].name}:${version}`;
|
||||
if (!this._client[commandName]) {
|
||||
this._client.defineCommand(commandName, {
|
||||
numberOfKeys: finalScripts[property].keys,
|
||||
lua: finalScripts[property].content,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
async init() {
|
||||
if (!this._client) {
|
||||
this._client = new ioredis_1.default(this.opts);
|
||||
}
|
||||
(0, utils_2.increaseMaxListeners)(this._client, 3);
|
||||
this._client.on('error', this.handleClientError);
|
||||
// ioredis treats connection errors as a different event ('close')
|
||||
this._client.on('close', this.handleClientClose);
|
||||
this._client.on('ready', this.handleClientReady);
|
||||
await RedisConnection.waitUntilReady(this._client);
|
||||
this.loadCommands(version_1.version);
|
||||
this.version = await this.getRedisVersion();
|
||||
if (this.skipVersionCheck !== true && !this.closing) {
|
||||
if ((0, utils_2.isRedisVersionLowerThan)(this.version, RedisConnection.minimumVersion)) {
|
||||
throw new Error(`Redis version needs to be greater or equal than ${RedisConnection.minimumVersion} Current: ${this.version}`);
|
||||
}
|
||||
if ((0, utils_2.isRedisVersionLowerThan)(this.version, RedisConnection.recommendedMinimumVersion)) {
|
||||
console.warn(`It is highly recommended to use a minimum Redis version of ${RedisConnection.recommendedMinimumVersion}
|
||||
Current: ${this.version}`);
|
||||
}
|
||||
}
|
||||
this.capabilities = {
|
||||
canDoubleTimeout: !(0, utils_2.isRedisVersionLowerThan)(this.version, '6.0.0'),
|
||||
};
|
||||
return this._client;
|
||||
}
|
||||
async disconnect(wait = true) {
|
||||
const client = await this.client;
|
||||
if (client.status !== 'end') {
|
||||
let _resolve, _reject;
|
||||
if (!wait) {
|
||||
return client.disconnect();
|
||||
}
|
||||
const disconnecting = new Promise((resolve, reject) => {
|
||||
(0, utils_2.increaseMaxListeners)(client, 2);
|
||||
client.once('end', resolve);
|
||||
client.once('error', reject);
|
||||
_resolve = resolve;
|
||||
_reject = reject;
|
||||
});
|
||||
client.disconnect();
|
||||
try {
|
||||
await disconnecting;
|
||||
}
|
||||
finally {
|
||||
(0, utils_2.decreaseMaxListeners)(client, 2);
|
||||
client.removeListener('end', _resolve);
|
||||
client.removeListener('error', _reject);
|
||||
}
|
||||
}
|
||||
}
|
||||
async reconnect() {
|
||||
const client = await this.client;
|
||||
return client.connect();
|
||||
}
|
||||
async close() {
|
||||
if (!this.closing) {
|
||||
this.closing = true;
|
||||
try {
|
||||
await this.initializing;
|
||||
if (!this.shared) {
|
||||
await this._client.quit();
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
if ((0, utils_2.isNotConnectionError)(error)) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
finally {
|
||||
this._client.off('error', this.handleClientError);
|
||||
this._client.off('close', this.handleClientClose);
|
||||
this._client.off('ready', this.handleClientReady);
|
||||
(0, utils_2.decreaseMaxListeners)(this._client, 3);
|
||||
this.removeAllListeners();
|
||||
}
|
||||
}
|
||||
}
|
||||
async getRedisVersion() {
|
||||
const doc = await this._client.info();
|
||||
const redisPrefix = 'redis_version:';
|
||||
const maxMemoryPolicyPrefix = 'maxmemory_policy:';
|
||||
const lines = doc.split('\r\n');
|
||||
let redisVersion;
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
if (lines[i].indexOf(maxMemoryPolicyPrefix) === 0) {
|
||||
const maxMemoryPolicy = lines[i].substr(maxMemoryPolicyPrefix.length);
|
||||
if (maxMemoryPolicy !== 'noeviction') {
|
||||
console.warn(`IMPORTANT! Eviction policy is ${maxMemoryPolicy}. It should be "noeviction"`);
|
||||
}
|
||||
}
|
||||
if (lines[i].indexOf(redisPrefix) === 0) {
|
||||
redisVersion = lines[i].substr(redisPrefix.length);
|
||||
}
|
||||
}
|
||||
return redisVersion;
|
||||
}
|
||||
get redisVersion() {
|
||||
return this.version;
|
||||
}
|
||||
}
|
||||
exports.RedisConnection = RedisConnection;
|
||||
RedisConnection.minimumVersion = '5.0.0';
|
||||
RedisConnection.recommendedMinimumVersion = '6.2.0';
|
||||
//# sourceMappingURL=redis-connection.js.map
|
||||
+1
File diff suppressed because one or more lines are too long
+151
@@ -0,0 +1,151 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.getNextMillis = exports.Repeat = void 0;
|
||||
const tslib_1 = require("tslib");
|
||||
const cron_parser_1 = require("cron-parser");
|
||||
const crypto_1 = require("crypto");
|
||||
const queue_base_1 = require("./queue-base");
|
||||
class Repeat extends queue_base_1.QueueBase {
|
||||
constructor(name, opts, Connection) {
|
||||
super(name, opts, Connection);
|
||||
this.repeatStrategy =
|
||||
(opts.settings && opts.settings.repeatStrategy) || exports.getNextMillis;
|
||||
this.repeatKeyHashAlgorithm =
|
||||
(opts.settings && opts.settings.repeatKeyHashAlgorithm) || 'md5';
|
||||
}
|
||||
async addNextRepeatableJob(name, data, opts, skipCheckExists) {
|
||||
var _a;
|
||||
// HACK: This is a temporary fix to enable easy migration from bullmq <3.0.0
|
||||
// to >= 3.0.0. It should be removed when moving to 4.x.
|
||||
const repeatOpts = Object.assign({}, opts.repeat);
|
||||
(_a = repeatOpts.pattern) !== null && _a !== void 0 ? _a : (repeatOpts.pattern = repeatOpts.cron);
|
||||
delete repeatOpts.cron;
|
||||
const prevMillis = opts.prevMillis || 0;
|
||||
const currentCount = repeatOpts.count ? repeatOpts.count + 1 : 1;
|
||||
if (typeof repeatOpts.limit !== 'undefined' &&
|
||||
currentCount > repeatOpts.limit) {
|
||||
return;
|
||||
}
|
||||
let now = Date.now();
|
||||
if (!(typeof repeatOpts.endDate === undefined) &&
|
||||
now > new Date(repeatOpts.endDate).getTime()) {
|
||||
return;
|
||||
}
|
||||
now = prevMillis < now ? now : prevMillis;
|
||||
const nextMillis = await this.repeatStrategy(now, repeatOpts, name);
|
||||
const pattern = repeatOpts.pattern;
|
||||
const hasImmediately = Boolean((repeatOpts.every || pattern) && repeatOpts.immediately);
|
||||
const offset = hasImmediately ? now - nextMillis : undefined;
|
||||
if (nextMillis) {
|
||||
// We store the undecorated opts.jobId into the repeat options
|
||||
if (!prevMillis && opts.jobId) {
|
||||
repeatOpts.jobId = opts.jobId;
|
||||
}
|
||||
const repeatJobKey = getRepeatKey(name, repeatOpts);
|
||||
let repeatableExists = true;
|
||||
if (!skipCheckExists) {
|
||||
// Check that the repeatable job hasn't been removed
|
||||
// TODO: a lua script would be better here
|
||||
const client = await this.client;
|
||||
repeatableExists = !!(await client.zscore(this.keys.repeat, repeatJobKey));
|
||||
}
|
||||
const { immediately } = repeatOpts, filteredRepeatOpts = tslib_1.__rest(repeatOpts, ["immediately"]);
|
||||
// The job could have been deleted since this check
|
||||
if (repeatableExists) {
|
||||
return this.createNextJob(name, nextMillis, repeatJobKey, Object.assign(Object.assign({}, opts), { repeat: Object.assign({ offset }, filteredRepeatOpts) }), data, currentCount, hasImmediately);
|
||||
}
|
||||
}
|
||||
}
|
||||
async createNextJob(name, nextMillis, repeatJobKey, opts, data, currentCount, hasImmediately) {
|
||||
const client = await this.client;
|
||||
//
|
||||
// Generate unique job id for this iteration.
|
||||
//
|
||||
const jobId = this.getRepeatJobId(name, nextMillis, this.hash(repeatJobKey), opts.repeat.jobId);
|
||||
const now = Date.now();
|
||||
const delay = nextMillis + (opts.repeat.offset ? opts.repeat.offset : 0) - now;
|
||||
const mergedOpts = Object.assign(Object.assign({}, opts), { jobId, delay: delay < 0 || hasImmediately ? 0 : delay, timestamp: now, prevMillis: nextMillis, repeatJobKey });
|
||||
mergedOpts.repeat = Object.assign(Object.assign({}, opts.repeat), { count: currentCount });
|
||||
await client.zadd(this.keys.repeat, nextMillis.toString(), repeatJobKey);
|
||||
return this.Job.create(this, name, data, mergedOpts);
|
||||
}
|
||||
async removeRepeatable(name, repeat, jobId) {
|
||||
const repeatJobKey = getRepeatKey(name, Object.assign(Object.assign({}, repeat), { jobId }));
|
||||
const repeatJobId = this.getRepeatJobId(name, '', this.hash(repeatJobKey), jobId || repeat.jobId);
|
||||
return this.scripts.removeRepeatable(repeatJobId, repeatJobKey);
|
||||
}
|
||||
async removeRepeatableByKey(repeatJobKey) {
|
||||
const data = this.keyToData(repeatJobKey);
|
||||
const repeatJobId = this.getRepeatJobId(data.name, '', this.hash(repeatJobKey), data.id);
|
||||
return this.scripts.removeRepeatable(repeatJobId, repeatJobKey);
|
||||
}
|
||||
keyToData(key, next) {
|
||||
const data = key.split(':');
|
||||
const pattern = data.slice(4).join(':') || null;
|
||||
return {
|
||||
key,
|
||||
name: data[0],
|
||||
id: data[1] || null,
|
||||
endDate: parseInt(data[2]) || null,
|
||||
tz: data[3] || null,
|
||||
pattern,
|
||||
next,
|
||||
};
|
||||
}
|
||||
async getRepeatableJobs(start = 0, end = -1, asc = false) {
|
||||
const client = await this.client;
|
||||
const key = this.keys.repeat;
|
||||
const result = asc
|
||||
? await client.zrange(key, start, end, 'WITHSCORES')
|
||||
: await client.zrevrange(key, start, end, 'WITHSCORES');
|
||||
const jobs = [];
|
||||
for (let i = 0; i < result.length; i += 2) {
|
||||
jobs.push(this.keyToData(result[i], parseInt(result[i + 1])));
|
||||
}
|
||||
return jobs;
|
||||
}
|
||||
async getRepeatableCount() {
|
||||
const client = await this.client;
|
||||
return client.zcard(this.toKey('repeat'));
|
||||
}
|
||||
hash(str) {
|
||||
return (0, crypto_1.createHash)(this.repeatKeyHashAlgorithm).update(str).digest('hex');
|
||||
}
|
||||
getRepeatJobId(name, nextMillis, namespace, jobId) {
|
||||
const checksum = this.hash(`${name}${jobId || ''}${namespace}`);
|
||||
return `repeat:${checksum}:${nextMillis}`;
|
||||
// return `repeat:${jobId || ''}:${name}:${namespace}:${nextMillis}`;
|
||||
//return `repeat:${name}:${namespace}:${nextMillis}`;
|
||||
}
|
||||
}
|
||||
exports.Repeat = Repeat;
|
||||
function getRepeatKey(name, repeat) {
|
||||
const endDate = repeat.endDate ? new Date(repeat.endDate).getTime() : '';
|
||||
const tz = repeat.tz || '';
|
||||
const pattern = repeat.pattern;
|
||||
const suffix = (pattern ? pattern : String(repeat.every)) || '';
|
||||
const jobId = repeat.jobId ? repeat.jobId : '';
|
||||
return `${name}:${jobId}:${endDate}:${tz}:${suffix}`;
|
||||
}
|
||||
const getNextMillis = (millis, opts) => {
|
||||
const pattern = opts.pattern;
|
||||
if (pattern && opts.every) {
|
||||
throw new Error('Both .pattern and .every options are defined for this repeatable job');
|
||||
}
|
||||
if (opts.every) {
|
||||
return (Math.floor(millis / opts.every) * opts.every +
|
||||
(opts.immediately ? 0 : opts.every));
|
||||
}
|
||||
const currentDate = opts.startDate && new Date(opts.startDate) > new Date(millis)
|
||||
? new Date(opts.startDate)
|
||||
: new Date(millis);
|
||||
const interval = (0, cron_parser_1.parseExpression)(pattern, Object.assign(Object.assign({}, opts), { currentDate }));
|
||||
try {
|
||||
return interval.next().getTime();
|
||||
}
|
||||
catch (e) {
|
||||
// Ignore error
|
||||
}
|
||||
};
|
||||
exports.getNextMillis = getNextMillis;
|
||||
//# sourceMappingURL=repeat.js.map
|
||||
+1
File diff suppressed because one or more lines are too long
+65
@@ -0,0 +1,65 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
const enums_1 = require("../enums");
|
||||
const sandbox = (processFile, childPool) => {
|
||||
return async function process(job, token) {
|
||||
const child = await childPool.retain(processFile);
|
||||
let msgHandler;
|
||||
let exitHandler;
|
||||
await child.send({
|
||||
cmd: enums_1.ChildCommand.Start,
|
||||
job: job.asJSONSandbox(),
|
||||
token,
|
||||
});
|
||||
const done = new Promise((resolve, reject) => {
|
||||
msgHandler = async (msg) => {
|
||||
var _a, _b;
|
||||
switch (msg.cmd) {
|
||||
case enums_1.ParentCommand.Completed:
|
||||
resolve(msg.value);
|
||||
break;
|
||||
case enums_1.ParentCommand.Failed:
|
||||
case enums_1.ParentCommand.Error: {
|
||||
const err = new Error();
|
||||
Object.assign(err, msg.value);
|
||||
reject(err);
|
||||
break;
|
||||
}
|
||||
case enums_1.ParentCommand.Progress:
|
||||
await job.updateProgress(msg.value);
|
||||
break;
|
||||
case enums_1.ParentCommand.Log:
|
||||
await job.log(msg.value);
|
||||
break;
|
||||
case enums_1.ParentCommand.MoveToDelayed:
|
||||
await job.moveToDelayed((_a = msg.value) === null || _a === void 0 ? void 0 : _a.timestamp, (_b = msg.value) === null || _b === void 0 ? void 0 : _b.token);
|
||||
break;
|
||||
case enums_1.ParentCommand.Update:
|
||||
await job.updateData(msg.value);
|
||||
break;
|
||||
}
|
||||
};
|
||||
exitHandler = (exitCode, signal) => {
|
||||
reject(new Error('Unexpected exit code: ' + exitCode + ' signal: ' + signal));
|
||||
};
|
||||
child.on('message', msgHandler);
|
||||
child.on('exit', exitHandler);
|
||||
});
|
||||
try {
|
||||
await done;
|
||||
return done;
|
||||
}
|
||||
finally {
|
||||
child.off('message', msgHandler);
|
||||
child.off('exit', exitHandler);
|
||||
if (child.exitCode !== null || /SIG.*/.test(`${child.signalCode}`)) {
|
||||
childPool.remove(child);
|
||||
}
|
||||
else {
|
||||
childPool.release(child);
|
||||
}
|
||||
}
|
||||
};
|
||||
};
|
||||
exports.default = sandbox;
|
||||
//# sourceMappingURL=sandbox.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"sandbox.js","sourceRoot":"","sources":["../../../src/classes/sandbox.ts"],"names":[],"mappings":";;AAAA,oCAAuD;AAKvD,MAAM,OAAO,GAAG,CACd,WAAgB,EAChB,SAAoB,EACpB,EAAE;IACF,OAAO,KAAK,UAAU,OAAO,CAAC,GAAiB,EAAE,KAAc;QAC7D,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QAClD,IAAI,UAAe,CAAC;QACpB,IAAI,WAAgB,CAAC;QAErB,MAAM,KAAK,CAAC,IAAI,CAAC;YACf,GAAG,EAAE,oBAAY,CAAC,KAAK;YACvB,GAAG,EAAE,GAAG,CAAC,aAAa,EAAE;YACxB,KAAK;SACN,CAAC,CAAC;QAEH,MAAM,IAAI,GAAe,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACvD,UAAU,GAAG,KAAK,EAAE,GAAiB,EAAE,EAAE;;gBACvC,QAAQ,GAAG,CAAC,GAAG,EAAE;oBACf,KAAK,qBAAa,CAAC,SAAS;wBAC1B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;wBACnB,MAAM;oBACR,KAAK,qBAAa,CAAC,MAAM,CAAC;oBAC1B,KAAK,qBAAa,CAAC,KAAK,CAAC,CAAC;wBACxB,MAAM,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC;wBACxB,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;wBAC9B,MAAM,CAAC,GAAG,CAAC,CAAC;wBACZ,MAAM;qBACP;oBACD,KAAK,qBAAa,CAAC,QAAQ;wBACzB,MAAM,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;wBACpC,MAAM;oBACR,KAAK,qBAAa,CAAC,GAAG;wBACpB,MAAM,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;wBACzB,MAAM;oBACR,KAAK,qBAAa,CAAC,aAAa;wBAC9B,MAAM,GAAG,CAAC,aAAa,CAAC,MAAA,GAAG,CAAC,KAAK,0CAAE,SAAS,EAAE,MAAA,GAAG,CAAC,KAAK,0CAAE,KAAK,CAAC,CAAC;wBAChE,MAAM;oBACR,KAAK,qBAAa,CAAC,MAAM;wBACvB,MAAM,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;wBAChC,MAAM;iBACT;YACH,CAAC,CAAC;YAEF,WAAW,GAAG,CAAC,QAAa,EAAE,MAAW,EAAE,EAAE;gBAC3C,MAAM,CACJ,IAAI,KAAK,CAAC,wBAAwB,GAAG,QAAQ,GAAG,WAAW,GAAG,MAAM,CAAC,CACtE,CAAC;YACJ,CAAC,CAAC;YAEF,KAAK,CAAC,EAAE,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;YAChC,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,IAAI;YACF,MAAM,IAAI,CAAC;YACX,OAAO,IAAI,CAAC;SACb;gBAAS;YACR,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;YACjC,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;YAE/B,IAAI,KAAK,CAAC,QAAQ,KAAK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,UAAU,EAAE,CAAC,EAAE;gBAClE,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;aACzB;iBAAM;gBACL,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;aAC1B;SACF;IACH,CAAC,CAAC;AACJ,CAAC,CAAC;AAEF,kBAAe,OAAO,CAAC"}
|
||||
+782
@@ -0,0 +1,782 @@
|
||||
/**
|
||||
* Includes all the scripts needed by the queue and jobs.
|
||||
*/
|
||||
/*eslint-env node */
|
||||
'use strict';
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.raw2NextJobData = exports.Scripts = void 0;
|
||||
const msgpackr_1 = require("msgpackr");
|
||||
const packer = new msgpackr_1.Packr({
|
||||
useRecords: false,
|
||||
encodeUndefinedAsNil: true,
|
||||
});
|
||||
const pack = packer.pack;
|
||||
const enums_1 = require("../enums");
|
||||
const utils_1 = require("../utils");
|
||||
const version_1 = require("../version");
|
||||
class Scripts {
|
||||
constructor(queue) {
|
||||
this.queue = queue;
|
||||
const queueKeys = this.queue.keys;
|
||||
this.version = version_1.version;
|
||||
this.moveToFinishedKeys = [
|
||||
queueKeys.wait,
|
||||
queueKeys.active,
|
||||
queueKeys.prioritized,
|
||||
queueKeys.events,
|
||||
queueKeys.stalled,
|
||||
queueKeys.limiter,
|
||||
queueKeys.delayed,
|
||||
queueKeys.paused,
|
||||
queueKeys.meta,
|
||||
queueKeys.pc,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
];
|
||||
}
|
||||
execCommand(client, commandName, args) {
|
||||
const commandNameWithVersion = `${commandName}:${this.version}`;
|
||||
return client[commandNameWithVersion](args);
|
||||
}
|
||||
async isJobInList(listKey, jobId) {
|
||||
const client = await this.queue.client;
|
||||
let result;
|
||||
if ((0, utils_1.isRedisVersionLowerThan)(this.queue.redisVersion, '6.0.6')) {
|
||||
result = await this.execCommand(client, 'isJobInList', [listKey, jobId]);
|
||||
}
|
||||
else {
|
||||
result = await client.lpos(listKey, jobId);
|
||||
}
|
||||
return Number.isInteger(result);
|
||||
}
|
||||
async addDelayedJob(client, job, encodedOpts, args) {
|
||||
const queueKeys = this.queue.keys;
|
||||
const keys = [
|
||||
queueKeys.wait,
|
||||
queueKeys.paused,
|
||||
queueKeys.meta,
|
||||
queueKeys.id,
|
||||
queueKeys.delayed,
|
||||
queueKeys.completed,
|
||||
queueKeys.events,
|
||||
];
|
||||
keys.push(pack(args), job.data, encodedOpts);
|
||||
return this.execCommand(client, 'addDelayedJob', keys);
|
||||
}
|
||||
async addPrioritizedJob(client, job, encodedOpts, args) {
|
||||
const queueKeys = this.queue.keys;
|
||||
const keys = [
|
||||
queueKeys.wait,
|
||||
queueKeys.paused,
|
||||
queueKeys.meta,
|
||||
queueKeys.id,
|
||||
queueKeys.prioritized,
|
||||
queueKeys.completed,
|
||||
queueKeys.events,
|
||||
queueKeys.pc,
|
||||
];
|
||||
keys.push(pack(args), job.data, encodedOpts);
|
||||
return this.execCommand(client, 'addPrioritizedJob', keys);
|
||||
}
|
||||
async addParentJob(client, job, encodedOpts, args) {
|
||||
const queueKeys = this.queue.keys;
|
||||
const keys = [
|
||||
queueKeys.meta,
|
||||
queueKeys.id,
|
||||
queueKeys.completed,
|
||||
queueKeys.events,
|
||||
];
|
||||
keys.push(pack(args), job.data, encodedOpts);
|
||||
return this.execCommand(client, 'addParentJob', keys);
|
||||
}
|
||||
async addJob(client, job, opts, jobId, parentOpts = {}) {
|
||||
const queueKeys = this.queue.keys;
|
||||
const parent = job.parent
|
||||
? Object.assign(Object.assign({}, job.parent), { fpof: opts.fpof, rdof: opts.rdof }) : null;
|
||||
const args = [
|
||||
queueKeys[''],
|
||||
typeof jobId !== 'undefined' ? jobId : '',
|
||||
job.name,
|
||||
job.timestamp,
|
||||
job.parentKey || null,
|
||||
parentOpts.waitChildrenKey || null,
|
||||
parentOpts.parentDependenciesKey || null,
|
||||
parent,
|
||||
job.repeatJobKey,
|
||||
];
|
||||
let encodedOpts;
|
||||
if (opts.repeat) {
|
||||
const repeat = Object.assign({}, opts.repeat);
|
||||
if (repeat.startDate) {
|
||||
repeat.startDate = +new Date(repeat.startDate);
|
||||
}
|
||||
if (repeat.endDate) {
|
||||
repeat.endDate = +new Date(repeat.endDate);
|
||||
}
|
||||
encodedOpts = pack(Object.assign(Object.assign({}, opts), { repeat }));
|
||||
}
|
||||
else {
|
||||
encodedOpts = pack(opts);
|
||||
}
|
||||
let result;
|
||||
if (parentOpts.waitChildrenKey) {
|
||||
result = await this.addParentJob(client, job, encodedOpts, args);
|
||||
}
|
||||
else if (opts.delay) {
|
||||
result = await this.addDelayedJob(client, job, encodedOpts, args);
|
||||
}
|
||||
else if (opts.priority) {
|
||||
result = await this.addPrioritizedJob(client, job, encodedOpts, args);
|
||||
}
|
||||
else {
|
||||
const keys = [
|
||||
queueKeys.wait,
|
||||
queueKeys.paused,
|
||||
queueKeys.meta,
|
||||
queueKeys.id,
|
||||
queueKeys.completed,
|
||||
queueKeys.events,
|
||||
];
|
||||
keys.push(pack(args), job.data, encodedOpts);
|
||||
result = await this.execCommand(client, 'addStandardJob', keys);
|
||||
}
|
||||
if (result < 0) {
|
||||
throw this.finishedErrors(result, parentOpts.parentKey, 'addJob');
|
||||
}
|
||||
return result;
|
||||
}
|
||||
async pause(pause) {
|
||||
const client = await this.queue.client;
|
||||
let src = 'wait', dst = 'paused';
|
||||
if (!pause) {
|
||||
src = 'paused';
|
||||
dst = 'wait';
|
||||
}
|
||||
const keys = [src, dst, 'meta', 'prioritized'].map((name) => this.queue.toKey(name));
|
||||
keys.push(this.queue.keys.events);
|
||||
return this.execCommand(client, 'pause', keys.concat([pause ? 'paused' : 'resumed']));
|
||||
}
|
||||
removeRepeatableArgs(repeatJobId, repeatJobKey) {
|
||||
const queueKeys = this.queue.keys;
|
||||
const keys = [queueKeys.repeat, queueKeys.delayed];
|
||||
const args = [repeatJobId, repeatJobKey, queueKeys['']];
|
||||
return keys.concat(args);
|
||||
}
|
||||
async removeRepeatable(repeatJobId, repeatJobKey) {
|
||||
const client = await this.queue.client;
|
||||
const args = this.removeRepeatableArgs(repeatJobId, repeatJobKey);
|
||||
return this.execCommand(client, 'removeRepeatable', args);
|
||||
}
|
||||
async remove(jobId, removeChildren) {
|
||||
const client = await this.queue.client;
|
||||
const keys = [''].map(name => this.queue.toKey(name));
|
||||
return this.execCommand(client, 'removeJob', keys.concat([jobId, removeChildren ? 1 : 0]));
|
||||
}
|
||||
async extendLock(jobId, token, duration, client) {
|
||||
client = client || (await this.queue.client);
|
||||
const args = [
|
||||
this.queue.toKey(jobId) + ':lock',
|
||||
this.queue.keys.stalled,
|
||||
token,
|
||||
duration,
|
||||
jobId,
|
||||
];
|
||||
return this.execCommand(client, 'extendLock', args);
|
||||
}
|
||||
async updateData(job, data) {
|
||||
const client = await this.queue.client;
|
||||
const keys = [this.queue.toKey(job.id)];
|
||||
const dataJson = JSON.stringify(data);
|
||||
const result = await this.execCommand(client, 'updateData', keys.concat([dataJson]));
|
||||
if (result < 0) {
|
||||
throw this.finishedErrors(result, job.id, 'updateData');
|
||||
}
|
||||
}
|
||||
async updateProgress(jobId, progress) {
|
||||
const client = await this.queue.client;
|
||||
const keys = [
|
||||
this.queue.toKey(jobId),
|
||||
this.queue.keys.events,
|
||||
this.queue.keys.meta,
|
||||
];
|
||||
const progressJson = JSON.stringify(progress);
|
||||
const result = await this.execCommand(client, 'updateProgress', keys.concat([jobId, progressJson]));
|
||||
if (result < 0) {
|
||||
throw this.finishedErrors(result, jobId, 'updateProgress');
|
||||
}
|
||||
}
|
||||
moveToFinishedArgs(job, val, propVal, shouldRemove, target, token, timestamp, fetchNext = true) {
|
||||
var _a, _b, _c, _d, _e;
|
||||
const queueKeys = this.queue.keys;
|
||||
const opts = this.queue.opts;
|
||||
const workerKeepJobs = target === 'completed' ? opts.removeOnComplete : opts.removeOnFail;
|
||||
const metricsKey = this.queue.toKey(`metrics:${target}`);
|
||||
const keys = this.moveToFinishedKeys;
|
||||
keys[10] = queueKeys[target];
|
||||
keys[11] = this.queue.toKey((_a = job.id) !== null && _a !== void 0 ? _a : '');
|
||||
keys[12] = metricsKey;
|
||||
const keepJobs = this.getKeepJobs(shouldRemove, workerKeepJobs);
|
||||
const args = [
|
||||
job.id,
|
||||
timestamp,
|
||||
propVal,
|
||||
typeof val === 'undefined' ? 'null' : val,
|
||||
target,
|
||||
JSON.stringify({ jobId: job.id, val: val }),
|
||||
!fetchNext || this.queue.closing ? 0 : 1,
|
||||
queueKeys[''],
|
||||
pack({
|
||||
token,
|
||||
keepJobs,
|
||||
limiter: opts.limiter,
|
||||
lockDuration: opts.lockDuration,
|
||||
attempts: job.opts.attempts,
|
||||
attemptsMade: job.attemptsMade,
|
||||
maxMetricsSize: ((_b = opts.metrics) === null || _b === void 0 ? void 0 : _b.maxDataPoints)
|
||||
? (_c = opts.metrics) === null || _c === void 0 ? void 0 : _c.maxDataPoints
|
||||
: '',
|
||||
fpof: !!((_d = job.opts) === null || _d === void 0 ? void 0 : _d.failParentOnFailure),
|
||||
rdof: !!((_e = job.opts) === null || _e === void 0 ? void 0 : _e.removeDependencyOnFailure),
|
||||
}),
|
||||
];
|
||||
return keys.concat(args);
|
||||
}
|
||||
getKeepJobs(shouldRemove, workerKeepJobs) {
|
||||
if (typeof shouldRemove === 'undefined') {
|
||||
return workerKeepJobs || { count: shouldRemove ? 0 : -1 };
|
||||
}
|
||||
return typeof shouldRemove === 'object'
|
||||
? shouldRemove
|
||||
: typeof shouldRemove === 'number'
|
||||
? { count: shouldRemove }
|
||||
: { count: shouldRemove ? 0 : -1 };
|
||||
}
|
||||
async moveToFinished(jobId, args) {
|
||||
const client = await this.queue.client;
|
||||
const result = await this.execCommand(client, 'moveToFinished', args);
|
||||
if (result < 0) {
|
||||
throw this.finishedErrors(result, jobId, 'moveToFinished', 'active');
|
||||
}
|
||||
else {
|
||||
if (typeof result !== 'undefined') {
|
||||
return raw2NextJobData(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
finishedErrors(code, jobId, command, state) {
|
||||
switch (code) {
|
||||
case enums_1.ErrorCode.JobNotExist:
|
||||
return new Error(`Missing key for job ${jobId}. ${command}`);
|
||||
case enums_1.ErrorCode.JobLockNotExist:
|
||||
return new Error(`Missing lock for job ${jobId}. ${command}`);
|
||||
case enums_1.ErrorCode.JobNotInState:
|
||||
return new Error(`Job ${jobId} is not in the ${state} state. ${command}`);
|
||||
case enums_1.ErrorCode.JobPendingDependencies:
|
||||
return new Error(`Job ${jobId} has pending dependencies. ${command}`);
|
||||
case enums_1.ErrorCode.ParentJobNotExist:
|
||||
return new Error(`Missing key for parent job ${jobId}. ${command}`);
|
||||
case enums_1.ErrorCode.JobLockMismatch:
|
||||
return new Error(`Lock mismatch for job ${jobId}. Cmd ${command} from ${state}`);
|
||||
default:
|
||||
return new Error(`Unknown code ${code} error for ${jobId}. ${command}`);
|
||||
}
|
||||
}
|
||||
drainArgs(delayed) {
|
||||
const queueKeys = this.queue.keys;
|
||||
const keys = [
|
||||
queueKeys.wait,
|
||||
queueKeys.paused,
|
||||
delayed ? queueKeys.delayed : '',
|
||||
queueKeys.prioritized,
|
||||
];
|
||||
const args = [queueKeys['']];
|
||||
return keys.concat(args);
|
||||
}
|
||||
async drain(delayed) {
|
||||
const client = await this.queue.client;
|
||||
const args = this.drainArgs(delayed);
|
||||
return this.execCommand(client, 'drain', args);
|
||||
}
|
||||
getRangesArgs(types, start, end, asc) {
|
||||
const queueKeys = this.queue.keys;
|
||||
const transformedTypes = types.map(type => {
|
||||
return type === 'waiting' ? 'wait' : type;
|
||||
});
|
||||
const keys = [queueKeys['']];
|
||||
const args = [start, end, asc ? '1' : '0', ...transformedTypes];
|
||||
return keys.concat(args);
|
||||
}
|
||||
async getRanges(types, start = 0, end = 1, asc = false) {
|
||||
const client = await this.queue.client;
|
||||
const args = this.getRangesArgs(types, start, end, asc);
|
||||
return this.execCommand(client, 'getRanges', args);
|
||||
}
|
||||
getCountsArgs(types) {
|
||||
const queueKeys = this.queue.keys;
|
||||
const transformedTypes = types.map(type => {
|
||||
return type === 'waiting' ? 'wait' : type;
|
||||
});
|
||||
const keys = [queueKeys['']];
|
||||
const args = [...transformedTypes];
|
||||
return keys.concat(args);
|
||||
}
|
||||
async getCounts(types) {
|
||||
const client = await this.queue.client;
|
||||
const args = this.getCountsArgs(types);
|
||||
return this.execCommand(client, 'getCounts', args);
|
||||
}
|
||||
moveToCompletedArgs(job, returnvalue, removeOnComplete, token, fetchNext = false) {
|
||||
const timestamp = Date.now();
|
||||
return this.moveToFinishedArgs(job, returnvalue, 'returnvalue', removeOnComplete, 'completed', token, timestamp, fetchNext);
|
||||
}
|
||||
moveToFailedArgs(job, failedReason, removeOnFailed, token, fetchNext = false) {
|
||||
const timestamp = Date.now();
|
||||
return this.moveToFinishedArgs(job, failedReason, 'failedReason', removeOnFailed, 'failed', token, timestamp, fetchNext);
|
||||
}
|
||||
async isFinished(jobId, returnValue = false) {
|
||||
const client = await this.queue.client;
|
||||
const keys = ['completed', 'failed', jobId].map((key) => {
|
||||
return this.queue.toKey(key);
|
||||
});
|
||||
return this.execCommand(client, 'isFinished', keys.concat([jobId, returnValue ? '1' : '']));
|
||||
}
|
||||
async getState(jobId) {
|
||||
const client = await this.queue.client;
|
||||
const keys = [
|
||||
'completed',
|
||||
'failed',
|
||||
'delayed',
|
||||
'active',
|
||||
'wait',
|
||||
'paused',
|
||||
'waiting-children',
|
||||
'prioritized',
|
||||
].map((key) => {
|
||||
return this.queue.toKey(key);
|
||||
});
|
||||
if ((0, utils_1.isRedisVersionLowerThan)(this.queue.redisVersion, '6.0.6')) {
|
||||
return this.execCommand(client, 'getState', keys.concat([jobId]));
|
||||
}
|
||||
return this.execCommand(client, 'getStateV2', keys.concat([jobId]));
|
||||
}
|
||||
async changeDelay(jobId, delay) {
|
||||
const client = await this.queue.client;
|
||||
const args = this.changeDelayArgs(jobId, delay);
|
||||
const result = await this.execCommand(client, 'changeDelay', args);
|
||||
if (result < 0) {
|
||||
throw this.finishedErrors(result, jobId, 'changeDelay', 'delayed');
|
||||
}
|
||||
}
|
||||
changeDelayArgs(jobId, delay) {
|
||||
//
|
||||
// Bake in the job id first 12 bits into the timestamp
|
||||
// to guarantee correct execution order of delayed jobs
|
||||
// (up to 4096 jobs per given timestamp or 4096 jobs apart per timestamp)
|
||||
//
|
||||
// WARNING: Jobs that are so far apart that they wrap around will cause FIFO to fail
|
||||
//
|
||||
let timestamp = Date.now() + delay;
|
||||
if (timestamp > 0) {
|
||||
timestamp = timestamp * 0x1000 + (+jobId & 0xfff);
|
||||
}
|
||||
const keys = ['delayed', jobId].map(name => {
|
||||
return this.queue.toKey(name);
|
||||
});
|
||||
keys.push.apply(keys, [this.queue.keys.events]);
|
||||
return keys.concat([delay, JSON.stringify(timestamp), jobId]);
|
||||
}
|
||||
async changePriority(jobId, priority = 0, lifo = false) {
|
||||
const client = await this.queue.client;
|
||||
const args = this.changePriorityArgs(jobId, priority, lifo);
|
||||
const result = await this.execCommand(client, 'changePriority', args);
|
||||
if (result < 0) {
|
||||
throw this.finishedErrors(result, jobId, 'changePriority');
|
||||
}
|
||||
}
|
||||
changePriorityArgs(jobId, priority = 0, lifo = false) {
|
||||
const keys = [
|
||||
this.queue.keys.wait,
|
||||
this.queue.keys.paused,
|
||||
this.queue.keys.meta,
|
||||
this.queue.keys.prioritized,
|
||||
this.queue.keys.pc,
|
||||
];
|
||||
return keys.concat([
|
||||
priority,
|
||||
this.queue.toKey(jobId),
|
||||
jobId,
|
||||
lifo ? 1 : 0,
|
||||
]);
|
||||
}
|
||||
// Note: We have an issue here with jobs using custom job ids
|
||||
moveToDelayedArgs(jobId, timestamp, token, delay) {
|
||||
//
|
||||
// Bake in the job id first 12 bits into the timestamp
|
||||
// to guarantee correct execution order of delayed jobs
|
||||
// (up to 4096 jobs per given timestamp or 4096 jobs apart per timestamp)
|
||||
//
|
||||
// WARNING: Jobs that are so far apart that they wrap around will cause FIFO to fail
|
||||
//
|
||||
timestamp = Math.max(0, timestamp !== null && timestamp !== void 0 ? timestamp : 0);
|
||||
if (timestamp > 0) {
|
||||
timestamp = timestamp * 0x1000 + (+jobId & 0xfff);
|
||||
}
|
||||
const keys = [
|
||||
'wait',
|
||||
'active',
|
||||
'prioritized',
|
||||
'delayed',
|
||||
jobId,
|
||||
].map(name => {
|
||||
return this.queue.toKey(name);
|
||||
});
|
||||
keys.push.apply(keys, [
|
||||
this.queue.keys.events,
|
||||
this.queue.keys.paused,
|
||||
this.queue.keys.meta,
|
||||
]);
|
||||
return keys.concat([
|
||||
this.queue.keys[''],
|
||||
Date.now(),
|
||||
JSON.stringify(timestamp),
|
||||
jobId,
|
||||
token,
|
||||
delay,
|
||||
]);
|
||||
}
|
||||
saveStacktraceArgs(jobId, stacktrace, failedReason) {
|
||||
const keys = [this.queue.toKey(jobId)];
|
||||
return keys.concat([stacktrace, failedReason]);
|
||||
}
|
||||
moveToWaitingChildrenArgs(jobId, token, opts) {
|
||||
const timestamp = Date.now();
|
||||
const childKey = (0, utils_1.getParentKey)(opts.child);
|
||||
const keys = [`${jobId}:lock`, 'active', 'waiting-children', jobId].map(name => {
|
||||
return this.queue.toKey(name);
|
||||
});
|
||||
return keys.concat([
|
||||
token,
|
||||
childKey !== null && childKey !== void 0 ? childKey : '',
|
||||
JSON.stringify(timestamp),
|
||||
jobId,
|
||||
]);
|
||||
}
|
||||
async moveToDelayed(jobId, timestamp, delay, token = '0') {
|
||||
const client = await this.queue.client;
|
||||
const args = this.moveToDelayedArgs(jobId, timestamp, token, delay);
|
||||
const result = await this.execCommand(client, 'moveToDelayed', args);
|
||||
if (result < 0) {
|
||||
throw this.finishedErrors(result, jobId, 'moveToDelayed', 'active');
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Move parent job to waiting-children state.
|
||||
*
|
||||
* @returns true if job is successfully moved, false if there are pending dependencies.
|
||||
* @throws JobNotExist
|
||||
* This exception is thrown if jobId is missing.
|
||||
* @throws JobLockNotExist
|
||||
* This exception is thrown if job lock is missing.
|
||||
* @throws JobNotInState
|
||||
* This exception is thrown if job is not in active state.
|
||||
*/
|
||||
async moveToWaitingChildren(jobId, token, opts = {}) {
|
||||
const client = await this.queue.client;
|
||||
const args = this.moveToWaitingChildrenArgs(jobId, token, opts);
|
||||
const result = await this.execCommand(client, 'moveToWaitingChildren', args);
|
||||
switch (result) {
|
||||
case 0:
|
||||
return true;
|
||||
case 1:
|
||||
return false;
|
||||
default:
|
||||
throw this.finishedErrors(result, jobId, 'moveToWaitingChildren', 'active');
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Remove jobs in a specific state.
|
||||
*
|
||||
* @returns Id jobs from the deleted records.
|
||||
*/
|
||||
async cleanJobsInSet(set, timestamp, limit = 0) {
|
||||
const client = await this.queue.client;
|
||||
return this.execCommand(client, 'cleanJobsInSet', [
|
||||
this.queue.toKey(set),
|
||||
this.queue.toKey('events'),
|
||||
this.queue.toKey(''),
|
||||
timestamp,
|
||||
limit,
|
||||
set,
|
||||
]);
|
||||
}
|
||||
retryJobArgs(jobId, lifo, token) {
|
||||
const keys = [
|
||||
'active',
|
||||
'wait',
|
||||
'paused',
|
||||
jobId,
|
||||
'meta',
|
||||
].map(name => {
|
||||
return this.queue.toKey(name);
|
||||
});
|
||||
keys.push(this.queue.keys.events, this.queue.keys.delayed, this.queue.keys.prioritized, this.queue.keys.pc);
|
||||
const pushCmd = (lifo ? 'R' : 'L') + 'PUSH';
|
||||
return keys.concat([
|
||||
this.queue.toKey(''),
|
||||
Date.now(),
|
||||
pushCmd,
|
||||
jobId,
|
||||
token,
|
||||
]);
|
||||
}
|
||||
moveJobsToWaitArgs(state, count, timestamp) {
|
||||
const keys = [
|
||||
this.queue.toKey(''),
|
||||
this.queue.keys.events,
|
||||
this.queue.toKey(state),
|
||||
this.queue.toKey('wait'),
|
||||
this.queue.toKey('paused'),
|
||||
this.queue.toKey('meta'),
|
||||
];
|
||||
const args = [count, timestamp, state];
|
||||
return keys.concat(args);
|
||||
}
|
||||
async retryJobs(state = 'failed', count = 1000, timestamp = new Date().getTime()) {
|
||||
const client = await this.queue.client;
|
||||
const args = this.moveJobsToWaitArgs(state, count, timestamp);
|
||||
return this.execCommand(client, 'moveJobsToWait', args);
|
||||
}
|
||||
async promoteJobs(count = 1000) {
|
||||
const client = await this.queue.client;
|
||||
const args = this.moveJobsToWaitArgs('delayed', count, Number.MAX_VALUE);
|
||||
return this.execCommand(client, 'moveJobsToWait', args);
|
||||
}
|
||||
/**
|
||||
* Attempts to reprocess a job
|
||||
*
|
||||
* @param job -
|
||||
* @param state - The expected job state. If the job is not found
|
||||
* on the provided state, then it's not reprocessed. Supported states: 'failed', 'completed'
|
||||
*
|
||||
* @returns Returns a promise that evaluates to a return code:
|
||||
* 1 means the operation was a success
|
||||
* 0 means the job does not exist
|
||||
* -1 means the job is currently locked and can't be retried.
|
||||
* -2 means the job was not found in the expected set
|
||||
*/
|
||||
async reprocessJob(job, state) {
|
||||
const client = await this.queue.client;
|
||||
const keys = [
|
||||
this.queue.toKey(job.id),
|
||||
this.queue.keys.events,
|
||||
this.queue.toKey(state),
|
||||
this.queue.keys.wait,
|
||||
this.queue.keys.meta,
|
||||
this.queue.keys.paused,
|
||||
];
|
||||
const args = [
|
||||
job.id,
|
||||
(job.opts.lifo ? 'R' : 'L') + 'PUSH',
|
||||
state === 'failed' ? 'failedReason' : 'returnvalue',
|
||||
state,
|
||||
];
|
||||
const result = await this.execCommand(client, 'reprocessJob', keys.concat(args));
|
||||
switch (result) {
|
||||
case 1:
|
||||
return;
|
||||
default:
|
||||
throw this.finishedErrors(result, job.id, 'reprocessJob', state);
|
||||
}
|
||||
}
|
||||
async moveToActive(client, token, jobId) {
|
||||
const opts = this.queue.opts;
|
||||
const queueKeys = this.queue.keys;
|
||||
const keys = [
|
||||
queueKeys.wait,
|
||||
queueKeys.active,
|
||||
queueKeys.prioritized,
|
||||
queueKeys.events,
|
||||
queueKeys.stalled,
|
||||
queueKeys.limiter,
|
||||
queueKeys.delayed,
|
||||
queueKeys.paused,
|
||||
queueKeys.meta,
|
||||
queueKeys.pc,
|
||||
];
|
||||
const args = [
|
||||
queueKeys[''],
|
||||
Date.now(),
|
||||
jobId || '',
|
||||
pack({
|
||||
token,
|
||||
lockDuration: opts.lockDuration,
|
||||
limiter: opts.limiter,
|
||||
}),
|
||||
];
|
||||
const result = await this.execCommand(client, 'moveToActive', keys.concat(args));
|
||||
return raw2NextJobData(result);
|
||||
}
|
||||
async promote(jobId) {
|
||||
const client = await this.queue.client;
|
||||
const keys = [
|
||||
this.queue.keys.delayed,
|
||||
this.queue.keys.wait,
|
||||
this.queue.keys.paused,
|
||||
this.queue.keys.meta,
|
||||
this.queue.keys.prioritized,
|
||||
this.queue.keys.pc,
|
||||
this.queue.keys.events,
|
||||
];
|
||||
const args = [this.queue.toKey(''), jobId];
|
||||
const code = await this.execCommand(client, 'promote', keys.concat(args));
|
||||
if (code < 0) {
|
||||
throw this.finishedErrors(code, jobId, 'promote', 'delayed');
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Looks for unlocked jobs in the active queue.
|
||||
*
|
||||
* The job was being worked on, but the worker process died and it failed to renew the lock.
|
||||
* We call these jobs 'stalled'. This is the most common case. We resolve these by moving them
|
||||
* back to wait to be re-processed. To prevent jobs from cycling endlessly between active and wait,
|
||||
* (e.g. if the job handler keeps crashing),
|
||||
* we limit the number stalled job recoveries to settings.maxStalledCount.
|
||||
*/
|
||||
async moveStalledJobsToWait() {
|
||||
const client = await this.queue.client;
|
||||
const opts = this.queue.opts;
|
||||
const keys = [
|
||||
this.queue.keys.stalled,
|
||||
this.queue.keys.wait,
|
||||
this.queue.keys.active,
|
||||
this.queue.keys.failed,
|
||||
this.queue.keys['stalled-check'],
|
||||
this.queue.keys.meta,
|
||||
this.queue.keys.paused,
|
||||
this.queue.keys.events,
|
||||
];
|
||||
const args = [
|
||||
opts.maxStalledCount,
|
||||
this.queue.toKey(''),
|
||||
Date.now(),
|
||||
opts.stalledInterval,
|
||||
];
|
||||
return this.execCommand(client, 'moveStalledJobsToWait', keys.concat(args));
|
||||
}
|
||||
/**
|
||||
* Moves a job back from Active to Wait.
|
||||
* This script is used when a job has been manually rate limited and needs
|
||||
* to be moved back to wait from active status.
|
||||
*
|
||||
* @param client - Redis client
|
||||
* @param jobId - Job id
|
||||
* @returns
|
||||
*/
|
||||
async moveJobFromActiveToWait(jobId, token) {
|
||||
const client = await this.queue.client;
|
||||
const lockKey = `${this.queue.toKey(jobId)}:lock`;
|
||||
const keys = [
|
||||
this.queue.keys.active,
|
||||
this.queue.keys.wait,
|
||||
this.queue.keys.stalled,
|
||||
lockKey,
|
||||
this.queue.keys.paused,
|
||||
this.queue.keys.meta,
|
||||
this.queue.keys.limiter,
|
||||
this.queue.keys.prioritized,
|
||||
this.queue.keys.events,
|
||||
];
|
||||
const args = [jobId, token, this.queue.toKey(jobId)];
|
||||
const pttl = await this.execCommand(client, 'moveJobFromActiveToWait', keys.concat(args));
|
||||
return pttl < 0 ? 0 : pttl;
|
||||
}
|
||||
async obliterate(opts) {
|
||||
const client = await this.queue.client;
|
||||
const keys = [
|
||||
this.queue.keys.meta,
|
||||
this.queue.toKey(''),
|
||||
];
|
||||
const args = [opts.count, opts.force ? 'force' : null];
|
||||
const result = await this.execCommand(client, 'obliterate', keys.concat(args));
|
||||
if (result < 0) {
|
||||
switch (result) {
|
||||
case -1:
|
||||
throw new Error('Cannot obliterate non-paused queue');
|
||||
case -2:
|
||||
throw new Error('Cannot obliterate queue with active jobs');
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
/**
|
||||
* Paginate a set or hash keys.
|
||||
* @param opts
|
||||
*
|
||||
*/
|
||||
async paginate(key, opts) {
|
||||
const client = await this.queue.client;
|
||||
const keys = [key];
|
||||
const maxIterations = 5;
|
||||
const pageSize = opts.end >= 0 ? opts.end - opts.start + 1 : Infinity;
|
||||
let cursor = '0', offset = 0, items, total, rawJobs, page = [], jobs = [];
|
||||
do {
|
||||
const args = [
|
||||
opts.start + page.length,
|
||||
opts.end,
|
||||
cursor,
|
||||
offset,
|
||||
maxIterations,
|
||||
];
|
||||
if (opts.fetchJobs) {
|
||||
args.push(1);
|
||||
}
|
||||
[cursor, offset, items, total, rawJobs] = await this.execCommand(client, 'paginate', keys.concat(args));
|
||||
page = page.concat(items);
|
||||
if (rawJobs && rawJobs.length) {
|
||||
jobs = jobs.concat(rawJobs.map(utils_1.array2obj));
|
||||
}
|
||||
// Important to keep this coercive inequality (!=) instead of strict inequality (!==)
|
||||
} while (cursor != '0' && page.length < pageSize);
|
||||
// If we get an array of arrays, it means we are paginating a hash
|
||||
if (page.length && Array.isArray(page[0])) {
|
||||
const result = [];
|
||||
for (let index = 0; index < page.length; index++) {
|
||||
const [id, value] = page[index];
|
||||
try {
|
||||
result.push({ id, v: JSON.parse(value) });
|
||||
}
|
||||
catch (err) {
|
||||
result.push({ id, err: err.message });
|
||||
}
|
||||
}
|
||||
return {
|
||||
cursor,
|
||||
items: result,
|
||||
total,
|
||||
jobs,
|
||||
};
|
||||
}
|
||||
else {
|
||||
return {
|
||||
cursor,
|
||||
items: page.map(item => ({ id: item })),
|
||||
total,
|
||||
jobs,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
exports.Scripts = Scripts;
|
||||
function raw2NextJobData(raw) {
|
||||
if (raw) {
|
||||
const result = [null, raw[1], raw[2], raw[3]];
|
||||
if (raw[0]) {
|
||||
result[0] = (0, utils_1.array2obj)(raw[0]);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
return [];
|
||||
}
|
||||
exports.raw2NextJobData = raw2NextJobData;
|
||||
//# sourceMappingURL=scripts.js.map
|
||||
+1
File diff suppressed because one or more lines are too long
+634
@@ -0,0 +1,634 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.Worker = void 0;
|
||||
const fs = require("fs");
|
||||
const url_1 = require("url");
|
||||
const path = require("path");
|
||||
const uuid_1 = require("uuid");
|
||||
// Note: this Polyfill is only needed for Node versions < 15.4.0
|
||||
const node_abort_controller_1 = require("node-abort-controller");
|
||||
const utils_1 = require("../utils");
|
||||
const queue_base_1 = require("./queue-base");
|
||||
const repeat_1 = require("./repeat");
|
||||
const child_pool_1 = require("./child-pool");
|
||||
const job_1 = require("./job");
|
||||
const redis_connection_1 = require("./redis-connection");
|
||||
const sandbox_1 = require("./sandbox");
|
||||
const async_fifo_queue_1 = require("./async-fifo-queue");
|
||||
const errors_1 = require("./errors");
|
||||
// 10 seconds is the maximum time a BRPOPLPUSH can block.
|
||||
const maximumBlockTimeout = 10;
|
||||
/**
|
||||
*
|
||||
* This class represents a worker that is able to process jobs from the queue.
|
||||
* As soon as the class is instantiated and a connection to Redis is established
|
||||
* it will start processing jobs.
|
||||
*
|
||||
*/
|
||||
class Worker extends queue_base_1.QueueBase {
|
||||
static RateLimitError() {
|
||||
return new errors_1.RateLimitError();
|
||||
}
|
||||
constructor(name, processor, opts = {}, Connection) {
|
||||
super(name, Object.assign(Object.assign({}, opts), { blockingConnection: true }), Connection);
|
||||
this.abortDelayController = null;
|
||||
this.blockUntil = 0;
|
||||
this.drained = false;
|
||||
this.extendLocksTimer = null;
|
||||
this.limitUntil = 0;
|
||||
this.waiting = null;
|
||||
this.running = false;
|
||||
this.opts = Object.assign({ drainDelay: 5, concurrency: 1, lockDuration: 30000, maxStalledCount: 1, stalledInterval: 30000, autorun: true, runRetryDelay: 15000 }, this.opts);
|
||||
if (this.opts.stalledInterval <= 0) {
|
||||
throw new Error('stalledInterval must be greater than 0');
|
||||
}
|
||||
this.concurrency = this.opts.concurrency;
|
||||
this.opts.lockRenewTime =
|
||||
this.opts.lockRenewTime || this.opts.lockDuration / 2;
|
||||
this.id = (0, uuid_1.v4)();
|
||||
if (processor) {
|
||||
if (typeof processor === 'function') {
|
||||
this.processFn = processor;
|
||||
}
|
||||
else {
|
||||
// SANDBOXED
|
||||
if (processor instanceof url_1.URL) {
|
||||
if (!fs.existsSync(processor)) {
|
||||
throw new Error(`URL ${processor} does not exist in the local file system`);
|
||||
}
|
||||
processor = processor.href;
|
||||
}
|
||||
else {
|
||||
const supportedFileTypes = ['.js', '.ts', '.flow', '.cjs'];
|
||||
const processorFile = processor +
|
||||
(supportedFileTypes.includes(path.extname(processor)) ? '' : '.js');
|
||||
if (!fs.existsSync(processorFile)) {
|
||||
throw new Error(`File ${processorFile} does not exist`);
|
||||
}
|
||||
}
|
||||
const mainFile = this.opts.useWorkerThreads
|
||||
? 'main-worker.js'
|
||||
: 'main.js';
|
||||
let mainFilePath = path.join(path.dirname(module.filename), `${mainFile}`);
|
||||
try {
|
||||
fs.statSync(mainFilePath); // would throw if file not exists
|
||||
}
|
||||
catch (_) {
|
||||
mainFilePath = path.join(process.cwd(), `dist/cjs/classes/${mainFile}`);
|
||||
fs.statSync(mainFilePath);
|
||||
}
|
||||
this.childPool = new child_pool_1.ChildPool({
|
||||
mainFile: mainFilePath,
|
||||
useWorkerThreads: this.opts.useWorkerThreads,
|
||||
});
|
||||
this.processFn = (0, sandbox_1.default)(processor, this.childPool).bind(this);
|
||||
}
|
||||
if (this.opts.autorun) {
|
||||
this.run().catch(error => this.emit('error', error));
|
||||
}
|
||||
}
|
||||
const connectionName = this.clientName(utils_1.WORKER_SUFFIX);
|
||||
this.blockingConnection = new redis_connection_1.RedisConnection((0, utils_1.isRedisInstance)(opts.connection)
|
||||
? opts.connection.duplicate({ connectionName })
|
||||
: Object.assign(Object.assign({}, opts.connection), { connectionName }), false, true, opts.skipVersionCheck);
|
||||
this.blockingConnection.on('error', error => this.emit('error', error));
|
||||
this.blockingConnection.on('ready', () => setTimeout(() => this.emit('ready'), 0));
|
||||
}
|
||||
emit(event, ...args) {
|
||||
return super.emit(event, ...args);
|
||||
}
|
||||
off(eventName, listener) {
|
||||
super.off(eventName, listener);
|
||||
return this;
|
||||
}
|
||||
on(event, listener) {
|
||||
super.on(event, listener);
|
||||
return this;
|
||||
}
|
||||
once(event, listener) {
|
||||
super.once(event, listener);
|
||||
return this;
|
||||
}
|
||||
callProcessJob(job, token) {
|
||||
return this.processFn(job, token);
|
||||
}
|
||||
createJob(data, jobId) {
|
||||
return this.Job.fromJSON(this, data, jobId);
|
||||
}
|
||||
/**
|
||||
*
|
||||
* Waits until the worker is ready to start processing jobs.
|
||||
* In general only useful when writing tests.
|
||||
*
|
||||
*/
|
||||
async waitUntilReady() {
|
||||
await super.waitUntilReady();
|
||||
return this.blockingConnection.client;
|
||||
}
|
||||
set concurrency(concurrency) {
|
||||
if (typeof concurrency !== 'number' ||
|
||||
concurrency < 1 ||
|
||||
!isFinite(concurrency)) {
|
||||
throw new Error('concurrency must be a finite number greater than 0');
|
||||
}
|
||||
this.opts.concurrency = concurrency;
|
||||
}
|
||||
get repeat() {
|
||||
return new Promise(async (resolve) => {
|
||||
if (!this._repeat) {
|
||||
const connection = await this.client;
|
||||
this._repeat = new repeat_1.Repeat(this.name, Object.assign(Object.assign({}, this.opts), { connection }));
|
||||
this._repeat.on('error', e => this.emit.bind(this, e));
|
||||
}
|
||||
resolve(this._repeat);
|
||||
});
|
||||
}
|
||||
async run() {
|
||||
if (!this.processFn) {
|
||||
throw new Error('No process function is defined.');
|
||||
}
|
||||
if (this.running) {
|
||||
throw new Error('Worker is already running.');
|
||||
}
|
||||
try {
|
||||
this.running = true;
|
||||
if (this.closing) {
|
||||
return;
|
||||
}
|
||||
await this.startStalledCheckTimer();
|
||||
const jobsInProgress = new Set();
|
||||
this.startLockExtenderTimer(jobsInProgress);
|
||||
const asyncFifoQueue = (this.asyncFifoQueue =
|
||||
new async_fifo_queue_1.AsyncFifoQueue());
|
||||
let tokenPostfix = 0;
|
||||
const client = await this.client;
|
||||
const bclient = await this.blockingConnection.client;
|
||||
while (!this.closing) {
|
||||
let numTotal = asyncFifoQueue.numTotal();
|
||||
while (!this.waiting &&
|
||||
numTotal < this.opts.concurrency &&
|
||||
(!this.limitUntil || numTotal == 0)) {
|
||||
const token = `${this.id}:${tokenPostfix++}`;
|
||||
const fetchedJob = this.retryIfFailed(() => this._getNextJob(client, bclient, token, { block: true }), this.opts.runRetryDelay);
|
||||
asyncFifoQueue.add(fetchedJob);
|
||||
numTotal = asyncFifoQueue.numTotal();
|
||||
if (this.waiting && numTotal > 1) {
|
||||
// We have a job waiting but we have others that we could start processing already
|
||||
break;
|
||||
}
|
||||
// We await here so that we fetch jobs in sequence, this is important to avoid unnecessary calls
|
||||
// to Redis in high concurrency scenarios.
|
||||
const job = await fetchedJob;
|
||||
// No more jobs waiting but we have others that could start processing already
|
||||
if (!job && numTotal > 1) {
|
||||
break;
|
||||
}
|
||||
// If there are potential jobs to be processed and blockUntil is set, we should exit to avoid waiting
|
||||
// for processing this job.
|
||||
if (this.blockUntil) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Since there can be undefined jobs in the queue (when a job fails or queue is empty)
|
||||
// we iterate until we find a job.
|
||||
let job;
|
||||
do {
|
||||
job = await asyncFifoQueue.fetch();
|
||||
} while (!job &&
|
||||
asyncFifoQueue.numTotal() > 0 &&
|
||||
asyncFifoQueue.numQueued() > 0);
|
||||
if (job) {
|
||||
const token = job.token;
|
||||
asyncFifoQueue.add(this.retryIfFailed(() => this.processJob(job, token, () => asyncFifoQueue.numTotal() <= this.opts.concurrency, jobsInProgress), this.opts.runRetryDelay));
|
||||
}
|
||||
}
|
||||
this.running = false;
|
||||
return asyncFifoQueue.waitAll();
|
||||
}
|
||||
catch (error) {
|
||||
this.running = false;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Returns a promise that resolves to the next job in queue.
|
||||
* @param token - worker token to be assigned to retrieved job
|
||||
* @returns a Job or undefined if no job was available in the queue.
|
||||
*/
|
||||
async getNextJob(token, { block = true } = {}) {
|
||||
return this._getNextJob(await this.client, await this.blockingConnection.client, token, { block });
|
||||
}
|
||||
async _getNextJob(client, bclient, token, { block = true } = {}) {
|
||||
var _a;
|
||||
if (this.paused) {
|
||||
if (block) {
|
||||
await this.paused;
|
||||
}
|
||||
else {
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (this.closing) {
|
||||
return;
|
||||
}
|
||||
if (this.drained && block && !this.limitUntil && !this.waiting) {
|
||||
this.waiting = this.waitForJob(bclient);
|
||||
try {
|
||||
const jobId = await this.waiting;
|
||||
return this.moveToActive(client, token, jobId);
|
||||
}
|
||||
catch (err) {
|
||||
// Swallow error if locally paused or closing since we did force a disconnection
|
||||
if (!(this.paused || this.closing) &&
|
||||
(0, utils_1.isNotConnectionError)(err)) {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
finally {
|
||||
this.waiting = null;
|
||||
}
|
||||
}
|
||||
else {
|
||||
if (this.limitUntil) {
|
||||
(_a = this.abortDelayController) === null || _a === void 0 ? void 0 : _a.abort();
|
||||
this.abortDelayController = new node_abort_controller_1.AbortController();
|
||||
await this.delay(this.limitUntil, this.abortDelayController);
|
||||
}
|
||||
return this.moveToActive(client, token);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Overrides the rate limit to be active for the next jobs.
|
||||
*
|
||||
* @param expireTimeMs - expire time in ms of this rate limit.
|
||||
*/
|
||||
async rateLimit(expireTimeMs) {
|
||||
await this.client.then(client => client.set(this.keys.limiter, Number.MAX_SAFE_INTEGER, 'PX', expireTimeMs));
|
||||
}
|
||||
async moveToActive(client, token, jobId) {
|
||||
// If we get the special delayed job ID, we pick the delay as the next
|
||||
// block timeout.
|
||||
if (jobId && jobId.startsWith('0:')) {
|
||||
this.blockUntil = parseInt(jobId.split(':')[1]) || 0;
|
||||
// Remove marker from active list.
|
||||
await client.lrem(this.keys.active, 1, jobId);
|
||||
if (this.blockUntil > 0) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
const [jobData, id, limitUntil, delayUntil] = await this.scripts.moveToActive(client, token, jobId);
|
||||
this.updateDelays(limitUntil, delayUntil);
|
||||
return this.nextJobFromJobData(jobData, id, token);
|
||||
}
|
||||
async waitForJob(bclient) {
|
||||
if (this.paused) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const opts = this.opts;
|
||||
if (!this.closing) {
|
||||
let blockTimeout = Math.max(this.blockUntil
|
||||
? (this.blockUntil - Date.now()) / 1000
|
||||
: opts.drainDelay, 0);
|
||||
let jobId;
|
||||
// Blocking for less than 50ms is useless.
|
||||
if (blockTimeout > 0.05) {
|
||||
blockTimeout = this.blockingConnection.capabilities.canDoubleTimeout
|
||||
? blockTimeout
|
||||
: Math.ceil(blockTimeout);
|
||||
// We restrict the maximum block timeout to 10 second to avoid
|
||||
// blocking the connection for too long in the case of reconnections
|
||||
// reference: https://github.com/taskforcesh/bullmq/issues/1658
|
||||
blockTimeout = Math.min(blockTimeout, maximumBlockTimeout);
|
||||
jobId = await bclient.brpoplpush(this.keys.wait, this.keys.active, blockTimeout);
|
||||
}
|
||||
else {
|
||||
jobId = await bclient.rpoplpush(this.keys.wait, this.keys.active);
|
||||
}
|
||||
this.blockUntil = 0;
|
||||
return jobId;
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
if ((0, utils_1.isNotConnectionError)(error)) {
|
||||
this.emit('error', error);
|
||||
}
|
||||
if (!this.closing) {
|
||||
await this.delay();
|
||||
}
|
||||
}
|
||||
finally {
|
||||
this.waiting = null;
|
||||
}
|
||||
}
|
||||
/**
|
||||
*
|
||||
* This function is exposed only for testing purposes.
|
||||
*/
|
||||
async delay(milliseconds, abortController) {
|
||||
await (0, utils_1.delay)(milliseconds || utils_1.DELAY_TIME_1, abortController);
|
||||
}
|
||||
updateDelays(limitUntil = 0, delayUntil = 0) {
|
||||
this.limitUntil = Math.max(limitUntil, 0) || 0;
|
||||
this.blockUntil = Math.max(delayUntil, 0) || 0;
|
||||
}
|
||||
async nextJobFromJobData(jobData, jobId, token) {
|
||||
if (!jobData) {
|
||||
if (!this.drained) {
|
||||
this.emit('drained');
|
||||
this.drained = true;
|
||||
}
|
||||
}
|
||||
else {
|
||||
this.drained = false;
|
||||
const job = this.createJob(jobData, jobId);
|
||||
job.token = token;
|
||||
if (job.opts.repeat) {
|
||||
const repeat = await this.repeat;
|
||||
await repeat.addNextRepeatableJob(job.name, job.data, job.opts);
|
||||
}
|
||||
return job;
|
||||
}
|
||||
}
|
||||
async processJob(job, token, fetchNextCallback = () => true, jobsInProgress) {
|
||||
if (!job || this.closing || this.paused) {
|
||||
return;
|
||||
}
|
||||
const handleCompleted = async (result) => {
|
||||
if (!this.connection.closing) {
|
||||
const completed = await job.moveToCompleted(result, token, fetchNextCallback() && !(this.closing || this.paused));
|
||||
this.emit('completed', job, result, 'active');
|
||||
const [jobData, jobId, limitUntil, delayUntil] = completed || [];
|
||||
this.updateDelays(limitUntil, delayUntil);
|
||||
return this.nextJobFromJobData(jobData, jobId, token);
|
||||
}
|
||||
};
|
||||
const handleFailed = async (err) => {
|
||||
if (!this.connection.closing) {
|
||||
try {
|
||||
if (err.message == errors_1.RATE_LIMIT_ERROR) {
|
||||
this.limitUntil = await this.moveLimitedBackToWait(job, token);
|
||||
return;
|
||||
}
|
||||
if (err instanceof errors_1.DelayedError ||
|
||||
err.message == 'DelayedError' ||
|
||||
err instanceof errors_1.WaitingChildrenError ||
|
||||
err.name == 'WaitingChildrenError') {
|
||||
return;
|
||||
}
|
||||
await job.moveToFailed(err, token);
|
||||
this.emit('failed', job, err, 'active');
|
||||
}
|
||||
catch (err) {
|
||||
this.emit('error', err);
|
||||
// It probably means that the job has lost the lock before completion
|
||||
// A worker will (or already has) moved the job back
|
||||
// to the waiting list (as stalled)
|
||||
}
|
||||
}
|
||||
};
|
||||
this.emit('active', job, 'waiting');
|
||||
const inProgressItem = { job, ts: Date.now() };
|
||||
try {
|
||||
jobsInProgress.add(inProgressItem);
|
||||
const result = await this.callProcessJob(job, token);
|
||||
return await handleCompleted(result);
|
||||
}
|
||||
catch (err) {
|
||||
return handleFailed(err);
|
||||
}
|
||||
finally {
|
||||
jobsInProgress.delete(inProgressItem);
|
||||
}
|
||||
}
|
||||
/**
|
||||
*
|
||||
* Pauses the processing of this queue only for this worker.
|
||||
*/
|
||||
async pause(doNotWaitActive) {
|
||||
if (!this.paused) {
|
||||
this.paused = new Promise(resolve => {
|
||||
this.resumeWorker = function () {
|
||||
resolve();
|
||||
this.paused = null; // Allow pause to be checked externally for paused state.
|
||||
this.resumeWorker = null;
|
||||
};
|
||||
});
|
||||
await (!doNotWaitActive && this.whenCurrentJobsFinished());
|
||||
this.emit('paused');
|
||||
}
|
||||
}
|
||||
/**
|
||||
*
|
||||
* Resumes processing of this worker (if paused).
|
||||
*/
|
||||
resume() {
|
||||
if (this.resumeWorker) {
|
||||
this.resumeWorker();
|
||||
this.emit('resumed');
|
||||
}
|
||||
}
|
||||
/**
|
||||
*
|
||||
* Checks if worker is paused.
|
||||
*
|
||||
* @returns true if worker is paused, false otherwise.
|
||||
*/
|
||||
isPaused() {
|
||||
return !!this.paused;
|
||||
}
|
||||
/**
|
||||
*
|
||||
* Checks if worker is currently running.
|
||||
*
|
||||
* @returns true if worker is running, false otherwise.
|
||||
*/
|
||||
isRunning() {
|
||||
return this.running;
|
||||
}
|
||||
/**
|
||||
*
|
||||
* Closes the worker and related redis connections.
|
||||
*
|
||||
* This method waits for current jobs to finalize before returning.
|
||||
*
|
||||
* @param force - Use force boolean parameter if you do not want to wait for
|
||||
* current jobs to be processed.
|
||||
*
|
||||
* @returns Promise that resolves when the worker has been closed.
|
||||
*/
|
||||
close(force = false) {
|
||||
if (this.closing) {
|
||||
return this.closing;
|
||||
}
|
||||
this.closing = (async () => {
|
||||
var _a;
|
||||
this.emit('closing', 'closing queue');
|
||||
(_a = this.abortDelayController) === null || _a === void 0 ? void 0 : _a.abort();
|
||||
const client = await this.blockingConnection.client;
|
||||
this.resume();
|
||||
await Promise.resolve()
|
||||
.finally(() => {
|
||||
return force || this.whenCurrentJobsFinished(false);
|
||||
})
|
||||
.finally(() => {
|
||||
var _a;
|
||||
const closePoolPromise = (_a = this.childPool) === null || _a === void 0 ? void 0 : _a.clean();
|
||||
if (force) {
|
||||
// since we're not waiting for the job to end attach
|
||||
// an error handler to avoid crashing the whole process
|
||||
closePoolPromise === null || closePoolPromise === void 0 ? void 0 : closePoolPromise.catch(err => {
|
||||
console.error(err); // TODO: emit error in next breaking change version
|
||||
});
|
||||
return;
|
||||
}
|
||||
return closePoolPromise;
|
||||
})
|
||||
.finally(() => clearTimeout(this.extendLocksTimer))
|
||||
.finally(() => clearTimeout(this.stalledCheckTimer))
|
||||
.finally(() => client.disconnect())
|
||||
.finally(() => this.connection.close())
|
||||
.finally(() => this.emit('closed'));
|
||||
this.closed = true;
|
||||
})();
|
||||
return this.closing;
|
||||
}
|
||||
/**
|
||||
*
|
||||
* Manually starts the stalled checker.
|
||||
* The check will run once as soon as this method is called, and
|
||||
* then every opts.stalledInterval milliseconds until the worker is closed.
|
||||
* Note: Normally you do not need to call this method, since the stalled checker
|
||||
* is automatically started when the worker starts processing jobs after
|
||||
* calling run. However if you want to process the jobs manually you need
|
||||
* to call this method to start the stalled checker.
|
||||
*
|
||||
* @see {@link https://docs.bullmq.io/patterns/manually-fetching-jobs}
|
||||
*/
|
||||
async startStalledCheckTimer() {
|
||||
if (!this.opts.skipStalledCheck) {
|
||||
clearTimeout(this.stalledCheckTimer);
|
||||
if (!this.closing) {
|
||||
try {
|
||||
await this.checkConnectionError(() => this.moveStalledJobsToWait());
|
||||
this.stalledCheckTimer = setTimeout(async () => {
|
||||
await this.startStalledCheckTimer();
|
||||
}, this.opts.stalledInterval);
|
||||
}
|
||||
catch (err) {
|
||||
this.emit('error', err);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
startLockExtenderTimer(jobsInProgress) {
|
||||
if (!this.opts.skipLockRenewal) {
|
||||
clearTimeout(this.extendLocksTimer);
|
||||
if (!this.closed) {
|
||||
this.extendLocksTimer = setTimeout(async () => {
|
||||
// Get all the jobs whose locks expire in less than 1/2 of the lockRenewTime
|
||||
const now = Date.now();
|
||||
const jobsToExtend = [];
|
||||
for (const item of jobsInProgress) {
|
||||
const { job, ts } = item;
|
||||
if (!ts) {
|
||||
item.ts = now;
|
||||
continue;
|
||||
}
|
||||
if (ts + this.opts.lockRenewTime / 2 < now) {
|
||||
item.ts = now;
|
||||
jobsToExtend.push(job);
|
||||
}
|
||||
}
|
||||
try {
|
||||
if (jobsToExtend.length) {
|
||||
await this.extendLocks(jobsToExtend);
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
this.emit('error', err);
|
||||
}
|
||||
this.startLockExtenderTimer(jobsInProgress);
|
||||
}, this.opts.lockRenewTime / 2);
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Returns a promise that resolves when active jobs are cleared
|
||||
*
|
||||
* @returns
|
||||
*/
|
||||
async whenCurrentJobsFinished(reconnect = true) {
|
||||
//
|
||||
// Force reconnection of blocking connection to abort blocking redis call immediately.
|
||||
//
|
||||
if (this.waiting) {
|
||||
// If we are not going to reconnect, we will not wait for the disconnection.
|
||||
await this.blockingConnection.disconnect(reconnect);
|
||||
}
|
||||
else {
|
||||
reconnect = false;
|
||||
}
|
||||
if (this.asyncFifoQueue) {
|
||||
await this.asyncFifoQueue.waitAll();
|
||||
}
|
||||
reconnect && (await this.blockingConnection.reconnect());
|
||||
}
|
||||
async retryIfFailed(fn, delayInMs) {
|
||||
const retry = 1;
|
||||
do {
|
||||
try {
|
||||
return await fn();
|
||||
}
|
||||
catch (err) {
|
||||
this.emit('error', err);
|
||||
if (delayInMs) {
|
||||
await this.delay(delayInMs);
|
||||
}
|
||||
else {
|
||||
return;
|
||||
}
|
||||
}
|
||||
} while (retry);
|
||||
}
|
||||
async extendLocks(jobs) {
|
||||
try {
|
||||
const multi = (await this.client).multi();
|
||||
for (const job of jobs) {
|
||||
await this.scripts.extendLock(job.id, job.token, this.opts.lockDuration, multi);
|
||||
}
|
||||
const result = (await multi.exec());
|
||||
for (const [err, jobId] of result) {
|
||||
if (err) {
|
||||
// TODO: signal process function that the job has been lost.
|
||||
this.emit('error', new Error(`could not renew lock for job ${jobId}`));
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
this.emit('error', err);
|
||||
}
|
||||
}
|
||||
async moveStalledJobsToWait() {
|
||||
const chunkSize = 50;
|
||||
const [failed, stalled] = await this.scripts.moveStalledJobsToWait();
|
||||
stalled.forEach((jobId) => this.emit('stalled', jobId, 'active'));
|
||||
const jobPromises = [];
|
||||
for (let i = 0; i < failed.length; i++) {
|
||||
jobPromises.push(job_1.Job.fromId(this, failed[i]));
|
||||
if ((i + 1) % chunkSize === 0) {
|
||||
this.notifyFailedJobs(await Promise.all(jobPromises));
|
||||
jobPromises.length = 0;
|
||||
}
|
||||
}
|
||||
this.notifyFailedJobs(await Promise.all(jobPromises));
|
||||
}
|
||||
notifyFailedJobs(failedJobs) {
|
||||
failedJobs.forEach((job) => this.emit('failed', job, new Error('job stalled more than allowable limit'), 'active'));
|
||||
}
|
||||
moveLimitedBackToWait(job, token) {
|
||||
return this.scripts.moveJobFromActiveToWait(job.id, token);
|
||||
}
|
||||
}
|
||||
exports.Worker = Worker;
|
||||
//# sourceMappingURL=worker.js.map
|
||||
+1
File diff suppressed because one or more lines are too long
+124
@@ -0,0 +1,124 @@
|
||||
--[[
|
||||
Adds a delayed job to the queue by doing the following:
|
||||
- Increases the job counter if needed.
|
||||
- Creates a new job key with the job data.
|
||||
|
||||
- computes timestamp.
|
||||
- adds to delayed zset.
|
||||
- Emits a global event 'delayed' if the job is delayed.
|
||||
|
||||
Input:
|
||||
KEYS[1] 'wait',
|
||||
KEYS[2] 'paused'
|
||||
KEYS[3] 'meta'
|
||||
KEYS[4] 'id'
|
||||
KEYS[5] 'delayed'
|
||||
KEYS[6] 'completed'
|
||||
KEYS[7] events stream key
|
||||
|
||||
ARGV[1] msgpacked arguments array
|
||||
[1] key prefix,
|
||||
[2] custom id (use custom instead of one generated automatically)
|
||||
[3] name
|
||||
[4] timestamp
|
||||
[5] parentKey?
|
||||
x [6] waitChildrenKey key.
|
||||
[7] parent dependencies key.
|
||||
[8] parent? {id, queueKey}
|
||||
[9] repeat job key
|
||||
|
||||
ARGV[2] Json stringified job data
|
||||
ARGV[3] msgpacked options
|
||||
|
||||
Output:
|
||||
jobId - OK
|
||||
-5 - Missing parent key
|
||||
]]
|
||||
local waitKey = KEYS[1]
|
||||
local pausedKey = KEYS[2]
|
||||
|
||||
local metaKey = KEYS[3]
|
||||
local idKey = KEYS[4]
|
||||
local delayedKey = KEYS[5]
|
||||
|
||||
local completedKey = KEYS[6]
|
||||
local eventsKey = KEYS[7]
|
||||
|
||||
local jobId
|
||||
local jobIdKey
|
||||
local rcall = redis.call
|
||||
|
||||
local args = cmsgpack.unpack(ARGV[1])
|
||||
|
||||
local data = ARGV[2]
|
||||
local opts = cmsgpack.unpack(ARGV[3])
|
||||
|
||||
local parentKey = args[5]
|
||||
local repeatJobKey = args[9]
|
||||
local parent = args[8]
|
||||
local parentData
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/storeJob"
|
||||
--- @include "includes/addDelayMarkerIfNeeded"
|
||||
--- @include "includes/getTargetQueueList"
|
||||
--- @include "includes/getNextDelayedTimestamp"
|
||||
--- @include "includes/updateExistingJobsParent"
|
||||
--- @include "includes/getOrSetMaxEvents"
|
||||
|
||||
if parentKey ~= nil then
|
||||
if rcall("EXISTS", parentKey) ~= 1 then return -5 end
|
||||
|
||||
parentData = cjson.encode(parent)
|
||||
end
|
||||
|
||||
local jobCounter = rcall("INCR", idKey)
|
||||
|
||||
local maxEvents = getOrSetMaxEvents(metaKey)
|
||||
|
||||
local parentDependenciesKey = args[7]
|
||||
local timestamp = args[4]
|
||||
if args[2] == "" then
|
||||
jobId = jobCounter
|
||||
jobIdKey = args[1] .. jobId
|
||||
else
|
||||
-- Refactor to: handleDuplicateJob.lua
|
||||
jobId = args[2]
|
||||
jobIdKey = args[1] .. jobId
|
||||
if rcall("EXISTS", jobIdKey) == 1 then
|
||||
updateExistingJobsParent(parentKey, parent, parentData,
|
||||
parentDependenciesKey, completedKey, jobIdKey,
|
||||
jobId, timestamp)
|
||||
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
|
||||
"duplicated", "jobId", jobId)
|
||||
|
||||
return jobId .. "" -- convert to string
|
||||
end
|
||||
end
|
||||
|
||||
-- Store the job.
|
||||
local delay, priority = storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2],
|
||||
opts, timestamp, parentKey, parentData,
|
||||
repeatJobKey)
|
||||
|
||||
-- Compute delayed timestamp and the score.
|
||||
local delayedTimestamp = (delay > 0 and (timestamp + delay)) or 0
|
||||
local score = delayedTimestamp * 0x1000 + bit.band(jobCounter, 0xfff)
|
||||
|
||||
rcall("ZADD", delayedKey, score, jobId)
|
||||
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
|
||||
"jobId", jobId, "delay", delayedTimestamp)
|
||||
|
||||
-- If wait list is empty, and this delayed job is the next one to be processed,
|
||||
-- then we need to signal the workers by adding a dummy job (jobId 0:delay) to the wait list.
|
||||
local target = getTargetQueueList(metaKey, KEYS[1], KEYS[2])
|
||||
addDelayMarkerIfNeeded(target, delayedKey)
|
||||
|
||||
-- Check if this job is a child of another job, if so add it to the parents dependencies
|
||||
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
|
||||
-- fail in this case.
|
||||
if parentDependenciesKey ~= nil then
|
||||
rcall("SADD", parentDependenciesKey, jobIdKey)
|
||||
end
|
||||
|
||||
return jobId .. "" -- convert to string
|
||||
+101
@@ -0,0 +1,101 @@
|
||||
--[[
|
||||
Adds a parent job to the queue by doing the following:
|
||||
- Increases the job counter if needed.
|
||||
- Creates a new job key with the job data.
|
||||
- adds the job to the waiting-children zset
|
||||
|
||||
Input:
|
||||
KEYS[1] 'meta'
|
||||
KEYS[2] 'id'
|
||||
KEYS[3] 'completed'
|
||||
KEYS[4] events stream key
|
||||
|
||||
ARGV[1] msgpacked arguments array
|
||||
[1] key prefix,
|
||||
[2] custom id (will not generate one automatically)
|
||||
[3] name
|
||||
[4] timestamp
|
||||
[5] parentKey?
|
||||
[6] waitChildrenKey key.
|
||||
[7] parent dependencies key.
|
||||
[8] parent? {id, queueKey}
|
||||
[9] repeat job key
|
||||
|
||||
ARGV[2] Json stringified job data
|
||||
ARGV[3] msgpacked options
|
||||
|
||||
Output:
|
||||
jobId - OK
|
||||
-5 - Missing parent key
|
||||
]]
|
||||
local metaKey = KEYS[1]
|
||||
local idKey = KEYS[2]
|
||||
|
||||
local completedKey = KEYS[3]
|
||||
local eventsKey = KEYS[4]
|
||||
|
||||
local jobId
|
||||
local jobIdKey
|
||||
local rcall = redis.call
|
||||
|
||||
local args = cmsgpack.unpack(ARGV[1])
|
||||
|
||||
local data = ARGV[2]
|
||||
local opts = cmsgpack.unpack(ARGV[3])
|
||||
|
||||
local parentKey = args[5]
|
||||
local repeatJobKey = args[9]
|
||||
local parent = args[8]
|
||||
local parentData
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/storeJob"
|
||||
--- @include "includes/updateExistingJobsParent"
|
||||
--- @include "includes/getOrSetMaxEvents"
|
||||
|
||||
if parentKey ~= nil then
|
||||
if rcall("EXISTS", parentKey) ~= 1 then return -5 end
|
||||
|
||||
parentData = cjson.encode(parent)
|
||||
end
|
||||
|
||||
local jobCounter = rcall("INCR", idKey)
|
||||
|
||||
local maxEvents = getOrSetMaxEvents(metaKey)
|
||||
|
||||
local parentDependenciesKey = args[7]
|
||||
local timestamp = args[4]
|
||||
if args[2] == "" then
|
||||
jobId = jobCounter
|
||||
jobIdKey = args[1] .. jobId
|
||||
else
|
||||
jobId = args[2]
|
||||
jobIdKey = args[1] .. jobId
|
||||
if rcall("EXISTS", jobIdKey) == 1 then
|
||||
updateExistingJobsParent(parentKey, parent, parentData,
|
||||
parentDependenciesKey, completedKey, jobIdKey,
|
||||
jobId, timestamp)
|
||||
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
|
||||
"duplicated", "jobId", jobId)
|
||||
|
||||
return jobId .. "" -- convert to string
|
||||
end
|
||||
end
|
||||
|
||||
-- Store the job.
|
||||
storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2], opts, timestamp,
|
||||
parentKey, parentData, repeatJobKey)
|
||||
|
||||
local waitChildrenKey = args[6]
|
||||
rcall("ZADD", waitChildrenKey, timestamp, jobId)
|
||||
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
|
||||
"waiting-children", "jobId", jobId)
|
||||
|
||||
-- Check if this job is a child of another job, if so add it to the parents dependencies
|
||||
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
|
||||
-- fail in this case.
|
||||
if parentDependenciesKey ~= nil then
|
||||
rcall("SADD", parentDependenciesKey, jobIdKey)
|
||||
end
|
||||
|
||||
return jobId .. "" -- convert to string
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
--[[
|
||||
Adds a priotitized job to the queue by doing the following:
|
||||
- Increases the job counter if needed.
|
||||
- Creates a new job key with the job data.
|
||||
- Adds the job to the "added" list so that workers gets notified.
|
||||
|
||||
Input:
|
||||
KEYS[1] 'wait',
|
||||
KEYS[2] 'paused'
|
||||
KEYS[3] 'meta'
|
||||
KEYS[4] 'id'
|
||||
KEYS[5] 'prioritized'
|
||||
KEYS[6] 'completed'
|
||||
KEYS[7] events stream key
|
||||
KEYS[8] 'pc' priority counter
|
||||
|
||||
ARGV[1] msgpacked arguments array
|
||||
[1] key prefix,
|
||||
[2] custom id (will not generate one automatically)
|
||||
[3] name
|
||||
[4] timestamp
|
||||
[5] parentKey?
|
||||
[6] waitChildrenKey key.
|
||||
[7] parent dependencies key.
|
||||
[8] parent? {id, queueKey}
|
||||
[9] repeat job key
|
||||
|
||||
ARGV[2] Json stringified job data
|
||||
ARGV[3] msgpacked options
|
||||
|
||||
Output:
|
||||
jobId - OK
|
||||
-5 - Missing parent key
|
||||
]]
|
||||
local waitKey = KEYS[1]
|
||||
local pausedKey = KEYS[2]
|
||||
|
||||
local metaKey = KEYS[3]
|
||||
local idKey = KEYS[4]
|
||||
local priorityKey = KEYS[5]
|
||||
|
||||
local completedKey = KEYS[6]
|
||||
local eventsKey = KEYS[7]
|
||||
local priorityCounterKey = KEYS[8]
|
||||
|
||||
local jobId
|
||||
local jobIdKey
|
||||
local rcall = redis.call
|
||||
|
||||
local args = cmsgpack.unpack(ARGV[1])
|
||||
|
||||
local data = ARGV[2]
|
||||
local opts = cmsgpack.unpack(ARGV[3])
|
||||
|
||||
local parentKey = args[5]
|
||||
local repeatJobKey = args[9]
|
||||
local parent = args[8]
|
||||
local parentData
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/storeJob"
|
||||
--- @include "includes/addJobWithPriority"
|
||||
--- @include "includes/getTargetQueueList"
|
||||
--- @include "includes/updateExistingJobsParent"
|
||||
--- @include "includes/getOrSetMaxEvents"
|
||||
|
||||
if parentKey ~= nil then
|
||||
if rcall("EXISTS", parentKey) ~= 1 then return -5 end
|
||||
|
||||
parentData = cjson.encode(parent)
|
||||
end
|
||||
|
||||
local jobCounter = rcall("INCR", idKey)
|
||||
|
||||
local maxEvents = getOrSetMaxEvents(metaKey)
|
||||
|
||||
local parentDependenciesKey = args[7]
|
||||
local timestamp = args[4]
|
||||
if args[2] == "" then
|
||||
jobId = jobCounter
|
||||
jobIdKey = args[1] .. jobId
|
||||
else
|
||||
jobId = args[2]
|
||||
jobIdKey = args[1] .. jobId
|
||||
if rcall("EXISTS", jobIdKey) == 1 then
|
||||
updateExistingJobsParent(parentKey, parent, parentData,
|
||||
parentDependenciesKey, completedKey, jobIdKey,
|
||||
jobId, timestamp)
|
||||
|
||||
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
|
||||
"duplicated", "jobId", jobId)
|
||||
|
||||
return jobId .. "" -- convert to string
|
||||
end
|
||||
end
|
||||
|
||||
-- Store the job.
|
||||
local delay, priority = storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2],
|
||||
opts, timestamp, parentKey, parentData,
|
||||
repeatJobKey)
|
||||
|
||||
local target, paused = getTargetQueueList(metaKey, waitKey, pausedKey)
|
||||
|
||||
addJobWithPriority(waitKey, priorityKey, priority, paused, jobId,
|
||||
priorityCounterKey)
|
||||
-- Emit waiting event
|
||||
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting",
|
||||
"jobId", jobId)
|
||||
|
||||
-- Check if this job is a child of another job, if so add it to the parents dependencies
|
||||
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
|
||||
-- fail in this case.
|
||||
if parentDependenciesKey ~= nil then
|
||||
rcall("SADD", parentDependenciesKey, jobIdKey)
|
||||
end
|
||||
|
||||
return jobId .. "" -- convert to string
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
--[[
|
||||
Adds a job to the queue by doing the following:
|
||||
- Increases the job counter if needed.
|
||||
- Creates a new job key with the job data.
|
||||
|
||||
- if delayed:
|
||||
- computes timestamp.
|
||||
- adds to delayed zset.
|
||||
- Emits a global event 'delayed' if the job is delayed.
|
||||
- if not delayed
|
||||
- Adds the jobId to the wait/paused list in one of three ways:
|
||||
- LIFO
|
||||
- FIFO
|
||||
- prioritized.
|
||||
- Adds the job to the "added" list so that workers gets notified.
|
||||
|
||||
Input:
|
||||
KEYS[1] 'wait',
|
||||
KEYS[2] 'paused'
|
||||
KEYS[3] 'meta'
|
||||
KEYS[4] 'id'
|
||||
KEYS[5] 'completed'
|
||||
KEYS[6] events stream key
|
||||
|
||||
ARGV[1] msgpacked arguments array
|
||||
[1] key prefix,
|
||||
[2] custom id (will not generate one automatically)
|
||||
[3] name
|
||||
[4] timestamp
|
||||
[5] parentKey?
|
||||
[6] waitChildrenKey key.
|
||||
[7] parent dependencies key.
|
||||
[8] parent? {id, queueKey}
|
||||
[9] repeat job key
|
||||
|
||||
ARGV[2] Json stringified job data
|
||||
ARGV[3] msgpacked options
|
||||
|
||||
Output:
|
||||
jobId - OK
|
||||
-5 - Missing parent key
|
||||
]]
|
||||
local eventsKey = KEYS[6]
|
||||
|
||||
local jobId
|
||||
local jobIdKey
|
||||
local rcall = redis.call
|
||||
|
||||
local args = cmsgpack.unpack(ARGV[1])
|
||||
|
||||
local data = ARGV[2]
|
||||
local opts = cmsgpack.unpack(ARGV[3])
|
||||
|
||||
local parentKey = args[5]
|
||||
local repeatJobKey = args[9]
|
||||
local parent = args[8]
|
||||
local parentData
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/storeJob"
|
||||
--- @include "includes/updateExistingJobsParent"
|
||||
--- @include "includes/getTargetQueueList"
|
||||
--- @include "includes/getOrSetMaxEvents"
|
||||
|
||||
if parentKey ~= nil then
|
||||
if rcall("EXISTS", parentKey) ~= 1 then return -5 end
|
||||
|
||||
parentData = cjson.encode(parent)
|
||||
end
|
||||
|
||||
local jobCounter = rcall("INCR", KEYS[4])
|
||||
|
||||
local metaKey = KEYS[3]
|
||||
local maxEvents = getOrSetMaxEvents(metaKey)
|
||||
|
||||
local parentDependenciesKey = args[7]
|
||||
local timestamp = args[4]
|
||||
if args[2] == "" then
|
||||
jobId = jobCounter
|
||||
jobIdKey = args[1] .. jobId
|
||||
else
|
||||
jobId = args[2]
|
||||
jobIdKey = args[1] .. jobId
|
||||
if rcall("EXISTS", jobIdKey) == 1 then
|
||||
updateExistingJobsParent(parentKey, parent, parentData,
|
||||
parentDependenciesKey, KEYS[5], jobIdKey,
|
||||
jobId, timestamp)
|
||||
|
||||
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
|
||||
"duplicated", "jobId", jobId)
|
||||
|
||||
return jobId .. "" -- convert to string
|
||||
end
|
||||
end
|
||||
|
||||
-- Store the job.
|
||||
storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2], opts, timestamp,
|
||||
parentKey, parentData, repeatJobKey)
|
||||
|
||||
local target, paused = getTargetQueueList(metaKey, KEYS[1], KEYS[2])
|
||||
|
||||
-- LIFO or FIFO
|
||||
local pushCmd = opts['lifo'] and 'RPUSH' or 'LPUSH'
|
||||
rcall(pushCmd, target, jobId)
|
||||
|
||||
-- Emit waiting event
|
||||
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting",
|
||||
"jobId", jobId)
|
||||
|
||||
-- Check if this job is a child of another job, if so add it to the parents dependencies
|
||||
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
|
||||
-- fail in this case.
|
||||
if parentDependenciesKey ~= nil then
|
||||
rcall("SADD", parentDependenciesKey, jobIdKey)
|
||||
end
|
||||
|
||||
return jobId .. "" -- convert to string
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
--[[
|
||||
Change job delay when it is in delayed set.
|
||||
Input:
|
||||
KEYS[1] delayed key
|
||||
KEYS[2] job key
|
||||
KEYS[3] events stream
|
||||
|
||||
ARGV[1] delay
|
||||
ARGV[2] delayedTimestamp
|
||||
ARGV[3] the id of the job
|
||||
Output:
|
||||
0 - OK
|
||||
-1 - Missing job.
|
||||
-3 - Job not in delayed set.
|
||||
|
||||
Events:
|
||||
- delayed key.
|
||||
]]
|
||||
local rcall = redis.call
|
||||
|
||||
if rcall("EXISTS", KEYS[2]) == 1 then
|
||||
local jobId = ARGV[3]
|
||||
local score = tonumber(ARGV[2])
|
||||
local delayedTimestamp = (score / 0x1000)
|
||||
|
||||
local numRemovedElements = rcall("ZREM", KEYS[1], jobId)
|
||||
|
||||
if numRemovedElements < 1 then
|
||||
return -3
|
||||
end
|
||||
|
||||
rcall("HSET", KEYS[2], "delay", tonumber(ARGV[1]))
|
||||
rcall("ZADD", KEYS[1], score, jobId)
|
||||
|
||||
rcall("XADD", KEYS[3], "*", "event", "delayed", "jobId", jobId, "delay", delayedTimestamp)
|
||||
|
||||
return 0
|
||||
else
|
||||
return -1
|
||||
end
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
--[[
|
||||
Change job priority
|
||||
Input:
|
||||
KEYS[1] 'wait',
|
||||
KEYS[2] 'paused'
|
||||
KEYS[3] 'meta'
|
||||
KEYS[4] 'prioritized'
|
||||
KEYS[5] 'pc' priority counter
|
||||
|
||||
ARGV[1] priority value
|
||||
ARGV[2] job key
|
||||
ARGV[3] job id
|
||||
ARGV[4] lifo
|
||||
|
||||
Output:
|
||||
0 - OK
|
||||
-1 - Missing job
|
||||
]]
|
||||
local jobKey = ARGV[2]
|
||||
local jobId = ARGV[3]
|
||||
local priority = tonumber(ARGV[1])
|
||||
local rcall = redis.call
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/addJobWithPriority"
|
||||
--- @include "includes/getTargetQueueList"
|
||||
|
||||
if rcall("EXISTS", jobKey) == 1 then
|
||||
local target, paused = getTargetQueueList(KEYS[3], KEYS[1], KEYS[2])
|
||||
|
||||
if rcall("ZREM", KEYS[4], jobId) > 0 then
|
||||
addJobWithPriority(KEYS[1], KEYS[4], priority, paused, jobId, KEYS[5])
|
||||
else
|
||||
local numRemovedElements = rcall("LREM", target, -1, jobId)
|
||||
if numRemovedElements > 0 then
|
||||
-- Standard or priority add
|
||||
if priority == 0 then
|
||||
-- LIFO or FIFO
|
||||
local pushCmd = ARGV[4] == '1' and 'RPUSH' or 'LPUSH';
|
||||
rcall(pushCmd, target, jobId)
|
||||
else
|
||||
addJobWithPriority(KEYS[1], KEYS[4], priority, paused, jobId, KEYS[5])
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
rcall("HSET", jobKey, "priority", priority)
|
||||
|
||||
return 0
|
||||
else
|
||||
return -1
|
||||
end
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
--[[
|
||||
Remove jobs from the specific set.
|
||||
|
||||
Input:
|
||||
KEYS[1] set key,
|
||||
KEYS[2] events stream key
|
||||
|
||||
ARGV[1] jobKey prefix
|
||||
ARGV[2] timestamp
|
||||
ARGV[3] limit the number of jobs to be removed. 0 is unlimited
|
||||
ARGV[4] set name, can be any of 'wait', 'active', 'paused', 'delayed', 'completed', or 'failed'
|
||||
]]
|
||||
local rcall = redis.call
|
||||
local rangeStart = 0
|
||||
local rangeEnd = -1
|
||||
|
||||
local limit = tonumber(ARGV[3])
|
||||
|
||||
-- If we're only deleting _n_ items, avoid retrieving all items
|
||||
-- for faster performance
|
||||
--
|
||||
-- Start from the tail of the list, since that's where oldest elements
|
||||
-- are generally added for FIFO lists
|
||||
if limit > 0 then
|
||||
rangeStart = -1 - limit + 1
|
||||
rangeEnd = -1
|
||||
end
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/cleanList"
|
||||
--- @include "includes/cleanSet"
|
||||
|
||||
local result
|
||||
if ARGV[4] == "active" then
|
||||
result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], false)
|
||||
elseif ARGV[4] == "delayed" then
|
||||
rangeEnd = "+inf"
|
||||
result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
|
||||
{"processedOn", "timestamp"}, false)
|
||||
elseif ARGV[4] == "prioritized" then
|
||||
rangeEnd = "+inf"
|
||||
result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
|
||||
{"timestamp"}, false)
|
||||
elseif ARGV[4] == "wait" or ARGV[4] == "paused" then
|
||||
result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], true)
|
||||
else
|
||||
rangeEnd = ARGV[2]
|
||||
result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
|
||||
{"finishedOn"}, true)
|
||||
end
|
||||
|
||||
rcall("XADD", KEYS[2], "*", "event", "cleaned", "count", result[2])
|
||||
|
||||
return result[1]
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
--[[
|
||||
Drains the queue, removes all jobs that are waiting
|
||||
or delayed, but not active, completed or failed
|
||||
|
||||
Input:
|
||||
KEYS[1] 'wait',
|
||||
KEYS[2] 'paused'
|
||||
KEYS[3] 'delayed'
|
||||
KEYS[4] 'prioritized'
|
||||
|
||||
ARGV[1] queue key prefix
|
||||
]]
|
||||
local rcall = redis.call
|
||||
local queueBaseKey = ARGV[1]
|
||||
|
||||
--- @include "includes/removeListJobs"
|
||||
--- @include "includes/removeZSetJobs"
|
||||
|
||||
removeListJobs(KEYS[1], true, queueBaseKey, 0) --wait
|
||||
removeListJobs(KEYS[2], true, queueBaseKey, 0) --paused
|
||||
|
||||
if KEYS[3] ~= "" then
|
||||
removeZSetJobs(KEYS[3], true, queueBaseKey, 0) --delayed
|
||||
end
|
||||
|
||||
removeZSetJobs(KEYS[4], true, queueBaseKey, 0) --prioritized
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
--[[
|
||||
Extend lock and removes the job from the stalled set.
|
||||
|
||||
Input:
|
||||
KEYS[1] 'lock',
|
||||
KEYS[2] 'stalled'
|
||||
|
||||
ARGV[1] token
|
||||
ARGV[2] lock duration in milliseconds
|
||||
ARGV[3] jobid
|
||||
|
||||
Output:
|
||||
"1" if lock extented succesfully.
|
||||
]]
|
||||
local rcall = redis.call
|
||||
if rcall("GET", KEYS[1]) == ARGV[1] then
|
||||
-- if rcall("SET", KEYS[1], ARGV[1], "PX", ARGV[2], "XX") then
|
||||
if rcall("SET", KEYS[1], ARGV[1], "PX", ARGV[2]) then
|
||||
rcall("SREM", KEYS[2], ARGV[3])
|
||||
return 1
|
||||
end
|
||||
end
|
||||
return 0
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
--[[
|
||||
Get counts per provided states
|
||||
|
||||
Input:
|
||||
KEYS[1] 'prefix'
|
||||
|
||||
ARGV[1...] types
|
||||
]]
|
||||
local rcall = redis.call;
|
||||
local prefix = KEYS[1]
|
||||
local results = {}
|
||||
|
||||
for i = 1, #ARGV do
|
||||
local stateKey = prefix .. ARGV[i]
|
||||
if ARGV[i] == "wait" or ARGV[i] == "paused" then
|
||||
local marker = rcall("LINDEX", stateKey, -1)
|
||||
if marker and string.sub(marker, 1, 2) == "0:" then
|
||||
local count = rcall("LLEN", stateKey)
|
||||
if count > 1 then
|
||||
rcall("RPOP", stateKey)
|
||||
results[#results+1] = count-1
|
||||
else
|
||||
results[#results+1] = 0
|
||||
end
|
||||
else
|
||||
results[#results+1] = rcall("LLEN", stateKey)
|
||||
end
|
||||
elseif ARGV[i] == "active" then
|
||||
results[#results+1] = rcall("LLEN", stateKey)
|
||||
else
|
||||
results[#results+1] = rcall("ZCARD", stateKey)
|
||||
end
|
||||
end
|
||||
|
||||
return results
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
--[[
|
||||
Get job ids per provided states
|
||||
|
||||
Input:
|
||||
KEYS[1] 'prefix'
|
||||
|
||||
ARGV[1] start
|
||||
ARGV[2] end
|
||||
ARGV[3] asc
|
||||
ARGV[4...] types
|
||||
]]
|
||||
local rcall = redis.call
|
||||
local prefix = KEYS[1]
|
||||
local rangeStart = tonumber(ARGV[1])
|
||||
local rangeEnd = tonumber(ARGV[2])
|
||||
local asc = ARGV[3]
|
||||
local results = {}
|
||||
|
||||
local function getRangeInList(listKey, asc, rangeStart, rangeEnd, results)
|
||||
if asc == "1" then
|
||||
local modifiedRangeStart
|
||||
local modifiedRangeEnd
|
||||
if rangeStart == -1 then
|
||||
modifiedRangeStart = 0
|
||||
else
|
||||
modifiedRangeStart = -(rangeStart + 1)
|
||||
end
|
||||
|
||||
if rangeEnd == -1 then
|
||||
modifiedRangeEnd = 0
|
||||
else
|
||||
modifiedRangeEnd = -(rangeEnd + 1)
|
||||
end
|
||||
|
||||
results[#results+1] = rcall("LRANGE", listKey,
|
||||
modifiedRangeEnd,
|
||||
modifiedRangeStart)
|
||||
else
|
||||
results[#results+1] = rcall("LRANGE", listKey, rangeStart, rangeEnd)
|
||||
end
|
||||
end
|
||||
|
||||
for i = 4, #ARGV do
|
||||
local stateKey = prefix .. ARGV[i]
|
||||
if ARGV[i] == "wait" or ARGV[i] == "paused" then
|
||||
local marker = rcall("LINDEX", stateKey, -1)
|
||||
if marker and string.sub(marker, 1, 2) == "0:" then
|
||||
local count = rcall("LLEN", stateKey)
|
||||
if count > 1 then
|
||||
rcall("RPOP", stateKey)
|
||||
getRangeInList(stateKey, asc, rangeStart, rangeEnd, results)
|
||||
else
|
||||
results[#results+1] = {}
|
||||
end
|
||||
else
|
||||
getRangeInList(stateKey, asc, rangeStart, rangeEnd, results)
|
||||
end
|
||||
elseif ARGV[i] == "active" then
|
||||
getRangeInList(stateKey, asc, rangeStart, rangeEnd, results)
|
||||
else
|
||||
if asc == "1" then
|
||||
results[#results+1] = rcall("ZRANGE", stateKey, rangeStart, rangeEnd)
|
||||
else
|
||||
results[#results+1] = rcall("ZREVRANGE", stateKey, rangeStart, rangeEnd)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
return results
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
--[[
|
||||
Get a job state
|
||||
|
||||
Input:
|
||||
KEYS[1] 'completed' key,
|
||||
KEYS[2] 'failed' key
|
||||
KEYS[3] 'delayed' key
|
||||
KEYS[4] 'active' key
|
||||
KEYS[5] 'wait' key
|
||||
KEYS[6] 'paused' key
|
||||
KEYS[7] 'waiting-children' key
|
||||
KEYS[8] 'prioritized' key
|
||||
|
||||
ARGV[1] job id
|
||||
Output:
|
||||
'completed'
|
||||
'failed'
|
||||
'delayed'
|
||||
'active'
|
||||
'prioritized'
|
||||
'waiting'
|
||||
'waiting-children'
|
||||
'unknown'
|
||||
]]
|
||||
local rcall = redis.call
|
||||
|
||||
if rcall("ZSCORE", KEYS[1], ARGV[1]) ~= false then
|
||||
return "completed"
|
||||
end
|
||||
|
||||
if rcall("ZSCORE", KEYS[2], ARGV[1]) ~= false then
|
||||
return "failed"
|
||||
end
|
||||
|
||||
if rcall("ZSCORE", KEYS[3], ARGV[1]) ~= false then
|
||||
return "delayed"
|
||||
end
|
||||
|
||||
if rcall("ZSCORE", KEYS[8], ARGV[1]) ~= false then
|
||||
return "prioritized"
|
||||
end
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/checkItemInList"
|
||||
|
||||
local active_items = rcall("LRANGE", KEYS[4] , 0, -1)
|
||||
if checkItemInList(active_items, ARGV[1]) ~= nil then
|
||||
return "active"
|
||||
end
|
||||
|
||||
local wait_items = rcall("LRANGE", KEYS[5] , 0, -1)
|
||||
if checkItemInList(wait_items, ARGV[1]) ~= nil then
|
||||
return "waiting"
|
||||
end
|
||||
|
||||
local paused_items = rcall("LRANGE", KEYS[6] , 0, -1)
|
||||
if checkItemInList(paused_items, ARGV[1]) ~= nil then
|
||||
return "waiting"
|
||||
end
|
||||
|
||||
if rcall("ZSCORE", KEYS[7], ARGV[1]) ~= false then
|
||||
return "waiting-children"
|
||||
end
|
||||
|
||||
return "unknown"
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
--[[
|
||||
Get a job state
|
||||
|
||||
Input:
|
||||
KEYS[1] 'completed' key,
|
||||
KEYS[2] 'failed' key
|
||||
KEYS[3] 'delayed' key
|
||||
KEYS[4] 'active' key
|
||||
KEYS[5] 'wait' key
|
||||
KEYS[6] 'paused' key
|
||||
KEYS[7] 'waiting-children' key
|
||||
KEYS[8] 'prioritized' key
|
||||
|
||||
ARGV[1] job id
|
||||
Output:
|
||||
'completed'
|
||||
'failed'
|
||||
'delayed'
|
||||
'active'
|
||||
'waiting'
|
||||
'waiting-children'
|
||||
'unknown'
|
||||
]]
|
||||
local rcall = redis.call
|
||||
|
||||
if rcall("ZSCORE", KEYS[1], ARGV[1]) ~= false then
|
||||
return "completed"
|
||||
end
|
||||
|
||||
if rcall("ZSCORE", KEYS[2], ARGV[1]) ~= false then
|
||||
return "failed"
|
||||
end
|
||||
|
||||
if rcall("ZSCORE", KEYS[3], ARGV[1]) ~= false then
|
||||
return "delayed"
|
||||
end
|
||||
|
||||
if rcall("ZSCORE", KEYS[8], ARGV[1]) ~= false then
|
||||
return "prioritized"
|
||||
end
|
||||
|
||||
if rcall("LPOS", KEYS[4] , ARGV[1]) ~= false then
|
||||
return "active"
|
||||
end
|
||||
|
||||
if rcall("LPOS", KEYS[5] , ARGV[1]) ~= false then
|
||||
return "waiting"
|
||||
end
|
||||
|
||||
if rcall("LPOS", KEYS[6] , ARGV[1]) ~= false then
|
||||
return "waiting"
|
||||
end
|
||||
|
||||
if rcall("ZSCORE", KEYS[7] , ARGV[1]) ~= false then
|
||||
return "waiting-children"
|
||||
end
|
||||
|
||||
return "unknown"
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
--[[
|
||||
Add delay marker if needed.
|
||||
]]
|
||||
|
||||
-- Includes
|
||||
--- @include "getNextDelayedTimestamp"
|
||||
|
||||
local function addDelayMarkerIfNeeded(targetKey, delayedKey)
|
||||
local waitLen = rcall("LLEN", targetKey)
|
||||
if waitLen <= 1 then
|
||||
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
|
||||
if nextTimestamp ~= nil then
|
||||
-- Check if there is already a marker with older timestamp
|
||||
-- if there is, we need to replace it.
|
||||
if waitLen == 1 then
|
||||
local marker = rcall("LINDEX", targetKey, 0)
|
||||
local oldTimestamp = tonumber(marker:sub(3))
|
||||
if oldTimestamp and oldTimestamp > nextTimestamp then
|
||||
rcall("LSET", targetKey, 0, "0:" .. nextTimestamp)
|
||||
end
|
||||
else
|
||||
-- if there is no marker, then we need to add one
|
||||
rcall("LPUSH", targetKey, "0:" .. nextTimestamp)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
--[[
|
||||
Function to add job considering priority.
|
||||
]]
|
||||
|
||||
-- Includes
|
||||
--- @include "addPriorityMarkerIfNeeded"
|
||||
--- @include "getPriorityScore"
|
||||
|
||||
local function addJobWithPriority(waitKey, prioritizedKey, priority, paused, jobId, priorityCounterKey)
|
||||
local score = getPriorityScore(priority, priorityCounterKey)
|
||||
rcall("ZADD", prioritizedKey, score, jobId)
|
||||
if not paused then
|
||||
addPriorityMarkerIfNeeded(waitKey)
|
||||
end
|
||||
end
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
--[[
|
||||
Function priority marker to wait if needed
|
||||
in order to wake up our workers and to respect priority
|
||||
order as much as possible
|
||||
]]
|
||||
local function addPriorityMarkerIfNeeded(waitKey)
|
||||
local waitLen = rcall("LLEN", waitKey)
|
||||
|
||||
if waitLen == 0 then
|
||||
rcall("LPUSH", waitKey, "0:0")
|
||||
end
|
||||
end
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
--[[
|
||||
Function to loop in batches.
|
||||
Just a bit of warning, some commands as ZREM
|
||||
could receive a maximum of 7000 parameters per call.
|
||||
]]
|
||||
|
||||
local function batches(n, batchSize)
|
||||
local i = 0
|
||||
|
||||
return function()
|
||||
local from = i * batchSize + 1
|
||||
i = i + 1
|
||||
if (from <= n) then
|
||||
local to = math.min(from + batchSize - 1, n)
|
||||
return from, to
|
||||
end
|
||||
end
|
||||
end
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
--[[
|
||||
Functions to check if a item belongs to a list.
|
||||
]]
|
||||
|
||||
local function checkItemInList(list, item)
|
||||
for _, v in pairs(list) do
|
||||
if v == item then
|
||||
return 1
|
||||
end
|
||||
end
|
||||
return nil
|
||||
end
|
||||
+139
@@ -0,0 +1,139 @@
|
||||
--[[
|
||||
Move stalled jobs to wait.
|
||||
|
||||
Input:
|
||||
stalledKey 'stalled' (SET)
|
||||
waitKey 'wait', (LIST)
|
||||
activeKey 'active', (LIST)
|
||||
failedKey 'failed', (ZSET)
|
||||
stalledCheckKey 'stalled-check', (KEY)
|
||||
metaKey 'meta', (KEY)
|
||||
pausedKey 'paused', (LIST)
|
||||
eventStreamKey 'event stream' (STREAM)
|
||||
|
||||
maxStalledJobCount Max stalled job count
|
||||
queueKeyPrefix queue.toKey('')
|
||||
timestamp timestamp
|
||||
maxCheckTime max check time
|
||||
|
||||
Events:
|
||||
'stalled' with stalled job id.
|
||||
]]
|
||||
local rcall = redis.call
|
||||
|
||||
-- Includes
|
||||
--- @include "batches"
|
||||
--- @include "getTargetQueueList"
|
||||
--- @include "removeJob"
|
||||
--- @include "removeJobsByMaxAge"
|
||||
--- @include "removeJobsByMaxCount"
|
||||
--- @include "trimEvents"
|
||||
|
||||
-- Check if we need to check for stalled jobs now.
|
||||
|
||||
local function checkStalledJobs(stalledKey, waitKey, activeKey, failedKey,
|
||||
stalledCheckKey, metaKey, pausedKey,
|
||||
eventStreamKey, maxStalledJobCount,
|
||||
queueKeyPrefix, timestamp, maxCheckTime)
|
||||
if rcall("EXISTS", stalledCheckKey) == 1 then return {{}, {}} end
|
||||
|
||||
rcall("SET", stalledCheckKey, timestamp, "PX", maxCheckTime)
|
||||
|
||||
-- Trim events before emiting them to avoid trimming events emitted in this script
|
||||
trimEvents(metaKey, eventStreamKey)
|
||||
|
||||
-- Move all stalled jobs to wait
|
||||
local stalling = rcall('SMEMBERS', stalledKey)
|
||||
local stalled = {}
|
||||
local failed = {}
|
||||
if (#stalling > 0) then
|
||||
rcall('DEL', stalledKey)
|
||||
|
||||
local MAX_STALLED_JOB_COUNT = tonumber(maxStalledJobCount)
|
||||
|
||||
-- Remove from active list
|
||||
for i, jobId in ipairs(stalling) do
|
||||
|
||||
if string.sub(jobId, 1, 2) == "0:" then
|
||||
-- If the jobId is a delay marker ID we just remove it.
|
||||
rcall("LREM", activeKey, 1, jobId)
|
||||
else
|
||||
local jobKey = queueKeyPrefix .. jobId
|
||||
|
||||
-- Check that the lock is also missing, then we can handle this job as really stalled.
|
||||
if (rcall("EXISTS", jobKey .. ":lock") == 0) then
|
||||
-- Remove from the active queue.
|
||||
local removed = rcall("LREM", activeKey, 1, jobId)
|
||||
|
||||
if (removed > 0) then
|
||||
-- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
|
||||
local stalledCount =
|
||||
rcall("HINCRBY", jobKey, "stalledCounter", 1)
|
||||
if (stalledCount > MAX_STALLED_JOB_COUNT) then
|
||||
local rawOpts = rcall("HGET", jobKey, "opts")
|
||||
local opts = cjson.decode(rawOpts)
|
||||
local removeOnFailType = type(opts["removeOnFail"])
|
||||
rcall("ZADD", failedKey, timestamp, jobId)
|
||||
local failedReason =
|
||||
"job stalled more than allowable limit"
|
||||
rcall("HMSET", jobKey, "failedReason", failedReason,
|
||||
"finishedOn", timestamp)
|
||||
rcall("XADD", eventStreamKey, "*", "event",
|
||||
"failed", "jobId", jobId, 'prev', 'active',
|
||||
'failedReason', failedReason)
|
||||
|
||||
if removeOnFailType == "number" then
|
||||
removeJobsByMaxCount(opts["removeOnFail"],
|
||||
failedKey, queueKeyPrefix)
|
||||
elseif removeOnFailType == "boolean" then
|
||||
if opts["removeOnFail"] then
|
||||
removeJob(jobId, false, queueKeyPrefix)
|
||||
rcall("ZREM", failedKey, jobId)
|
||||
end
|
||||
elseif removeOnFailType ~= "nil" then
|
||||
local maxAge = opts["removeOnFail"]["age"]
|
||||
local maxCount = opts["removeOnFail"]["count"]
|
||||
|
||||
if maxAge ~= nil then
|
||||
removeJobsByMaxAge(timestamp, maxAge,
|
||||
failedKey, queueKeyPrefix)
|
||||
end
|
||||
|
||||
if maxCount ~= nil and maxCount > 0 then
|
||||
removeJobsByMaxCount(maxCount, failedKey,
|
||||
queueKeyPrefix)
|
||||
end
|
||||
end
|
||||
|
||||
table.insert(failed, jobId)
|
||||
else
|
||||
local target =
|
||||
getTargetQueueList(metaKey, waitKey, pausedKey)
|
||||
|
||||
-- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
|
||||
rcall("RPUSH", target, jobId)
|
||||
rcall("XADD", eventStreamKey, "*", "event",
|
||||
"waiting", "jobId", jobId, 'prev', 'active')
|
||||
|
||||
-- Emit the stalled event
|
||||
rcall("XADD", eventStreamKey, "*", "event",
|
||||
"stalled", "jobId", jobId)
|
||||
table.insert(stalled, jobId)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
-- Mark potentially stalled jobs
|
||||
local active = rcall('LRANGE', activeKey, 0, -1)
|
||||
|
||||
if (#active > 0) then
|
||||
for from, to in batches(#active, 7000) do
|
||||
rcall('SADD', stalledKey, unpack(active, from, to))
|
||||
end
|
||||
end
|
||||
|
||||
return {failed, stalled}
|
||||
end
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
--[[
|
||||
Function to clean job list.
|
||||
Returns jobIds and deleted count number.
|
||||
]]
|
||||
|
||||
-- Includes
|
||||
--- @include "getTimestamp"
|
||||
--- @include "removeJob"
|
||||
|
||||
local function cleanList(listKey, jobKeyPrefix, rangeStart, rangeEnd,
|
||||
timestamp, isWaiting)
|
||||
local jobs = rcall("LRANGE", listKey, rangeStart, rangeEnd)
|
||||
local deleted = {}
|
||||
local deletedCount = 0
|
||||
local jobTS
|
||||
local deletionMarker = ''
|
||||
local jobIdsLen = #jobs
|
||||
for i, job in ipairs(jobs) do
|
||||
if limit > 0 and deletedCount >= limit then
|
||||
break
|
||||
end
|
||||
|
||||
local jobKey = jobKeyPrefix .. job
|
||||
if (isWaiting or rcall("EXISTS", jobKey .. ":lock") == 0) then
|
||||
-- Find the right timestamp of the job to compare to maxTimestamp:
|
||||
-- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed
|
||||
-- * processedOn represents when the job was last attempted, but it doesn't get populated until
|
||||
-- the job is first tried
|
||||
-- * timestamp is the original job submission time
|
||||
-- Fetch all three of these (in that order) and use the first one that is set so that we'll leave jobs
|
||||
-- that have been active within the grace period:
|
||||
jobTS = getTimestamp(jobKey, {"finishedOn", "processedOn", "timestamp"})
|
||||
if (not jobTS or jobTS <= timestamp) then
|
||||
-- replace the entry with a deletion marker; the actual deletion will
|
||||
-- occur at the end of the script
|
||||
rcall("LSET", listKey, rangeEnd - jobIdsLen + i, deletionMarker)
|
||||
removeJob(job, true, jobKeyPrefix)
|
||||
deletedCount = deletedCount + 1
|
||||
table.insert(deleted, job)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
rcall("LREM", listKey, 0, deletionMarker)
|
||||
|
||||
return {deleted, deletedCount}
|
||||
end
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
--[[
|
||||
Function to clean job set.
|
||||
Returns jobIds and deleted count number.
|
||||
]]
|
||||
|
||||
-- Includes
|
||||
--- @include "batches"
|
||||
--- @include "getJobsInZset"
|
||||
--- @include "getTimestamp"
|
||||
--- @include "removeJob"
|
||||
|
||||
local function cleanSet(setKey, jobKeyPrefix, rangeEnd, timestamp, limit, attributes, isFinished)
|
||||
local jobs = getJobsInZset(setKey, rangeEnd, limit)
|
||||
local deleted = {}
|
||||
local deletedCount = 0
|
||||
local jobTS
|
||||
for i, job in ipairs(jobs) do
|
||||
if limit > 0 and deletedCount >= limit then
|
||||
break
|
||||
end
|
||||
|
||||
local jobKey = jobKeyPrefix .. job
|
||||
if isFinished then
|
||||
removeJob(job, true, jobKeyPrefix)
|
||||
deletedCount = deletedCount + 1
|
||||
table.insert(deleted, job)
|
||||
else
|
||||
-- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed
|
||||
jobTS = getTimestamp(jobKey, attributes)
|
||||
if (not jobTS or jobTS <= timestamp) then
|
||||
removeJob(job, true, jobKeyPrefix)
|
||||
deletedCount = deletedCount + 1
|
||||
table.insert(deleted, job)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
if(#deleted > 0) then
|
||||
for from, to in batches(#deleted, 7000) do
|
||||
rcall("ZREM", setKey, unpack(deleted, from, to))
|
||||
end
|
||||
end
|
||||
|
||||
return {deleted, deletedCount}
|
||||
end
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
--[[
|
||||
Functions to collect metrics based on a current and previous count of jobs.
|
||||
Granualarity is fixed at 1 minute.
|
||||
]]
|
||||
--- @include "batches"
|
||||
local function collectMetrics(metaKey, dataPointsList, maxDataPoints,
|
||||
timestamp)
|
||||
-- Increment current count
|
||||
local count = rcall("HINCRBY", metaKey, "count", 1) - 1
|
||||
|
||||
-- Compute how many data points we need to add to the list, N.
|
||||
local prevTS = rcall("HGET", metaKey, "prevTS")
|
||||
|
||||
if not prevTS then
|
||||
-- If prevTS is nil, set it to the current timestamp
|
||||
rcall("HSET", metaKey, "prevTS", timestamp, "prevCount", 0)
|
||||
return
|
||||
end
|
||||
|
||||
local N = math.floor((timestamp - prevTS) / 60000)
|
||||
|
||||
if N > 0 then
|
||||
local delta = count - rcall("HGET", metaKey, "prevCount")
|
||||
-- If N > 1, add N-1 zeros to the list
|
||||
if N > 1 then
|
||||
local points = {}
|
||||
points[1] = delta
|
||||
for i = 2, N do
|
||||
points[i] = 0
|
||||
end
|
||||
|
||||
for from, to in batches(#points, 7000) do
|
||||
rcall("LPUSH", dataPointsList, unpack(points, from, to))
|
||||
end
|
||||
else
|
||||
-- LPUSH delta to the list
|
||||
rcall("LPUSH", dataPointsList, delta)
|
||||
end
|
||||
|
||||
-- LTRIM to keep list to its max size
|
||||
rcall("LTRIM", dataPointsList, 0, maxDataPoints - 1)
|
||||
|
||||
-- update prev count with current count
|
||||
rcall("HSET", metaKey, "prevCount", count, "prevTS", timestamp)
|
||||
end
|
||||
end
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
--[[
|
||||
Functions to destructure job key.
|
||||
Just a bit of warning, these functions may be a bit slow and affect performance significantly.
|
||||
]]
|
||||
|
||||
local getJobIdFromKey = function (jobKey)
|
||||
return string.match(jobKey, ".*:(.*)")
|
||||
end
|
||||
|
||||
local getJobKeyPrefix = function (jobKey, jobId)
|
||||
return string.sub(jobKey, 0, #jobKey - #jobId)
|
||||
end
|
||||
+70
@@ -0,0 +1,70 @@
|
||||
--[[
|
||||
Function to achieve pagination for a set or hash.
|
||||
This function simulates pagination in the most efficient way possible
|
||||
for a set using sscan or hscan.
|
||||
|
||||
The main limitation is that sets are not order preserving, so the
|
||||
pagination is not stable. This means that if the set is modified
|
||||
between pages, the same element may appear in different pages.
|
||||
]] -- Maximum number of elements to be returned by sscan per iteration.
|
||||
local maxCount = 100
|
||||
|
||||
-- Finds the cursor, and returns the first elements available for the requested page.
|
||||
local function findPage(key, command, pageStart, pageSize, cursor, offset,
|
||||
maxIterations, fetchJobs)
|
||||
local items = {}
|
||||
local jobs = {}
|
||||
local iterations = 0
|
||||
|
||||
repeat
|
||||
-- Iterate over the set using sscan/hscan.
|
||||
local result = rcall(command, key, cursor, "COUNT", maxCount)
|
||||
|
||||
cursor = result[1]
|
||||
local members = result[2]
|
||||
local step = 1
|
||||
if command == "HSCAN" then
|
||||
step = 2
|
||||
end
|
||||
|
||||
if #members == 0 then
|
||||
-- If the result is empty, we can return the result.
|
||||
return cursor, offset, items, jobs
|
||||
end
|
||||
|
||||
local chunkStart = offset
|
||||
local chunkEnd = offset + #members / step
|
||||
|
||||
local pageEnd = pageStart + pageSize
|
||||
|
||||
if chunkEnd < pageStart then
|
||||
-- If the chunk is before the page, we can skip it.
|
||||
offset = chunkEnd
|
||||
elseif chunkStart > pageEnd then
|
||||
-- If the chunk is after the page, we can return the result.
|
||||
return cursor, offset, items, jobs
|
||||
else
|
||||
-- If the chunk is overlapping the page, we need to add the elements to the result.
|
||||
for i = 1, #members, step do
|
||||
if offset >= pageEnd then
|
||||
return cursor, offset, items, jobs
|
||||
end
|
||||
if offset >= pageStart then
|
||||
local index = #items + 1
|
||||
if fetchJobs ~= nil then
|
||||
jobs[#jobs+1] = rcall("HGETALL", members[i])
|
||||
end
|
||||
if step == 2 then
|
||||
items[index] = {members[i], members[i + 1]}
|
||||
else
|
||||
items[index] = members[i]
|
||||
end
|
||||
end
|
||||
offset = offset + 1
|
||||
end
|
||||
end
|
||||
iterations = iterations + 1
|
||||
until cursor == "0" or iterations >= maxIterations
|
||||
|
||||
return cursor, offset, items, jobs
|
||||
end
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
-- We use ZRANGEBYSCORE to make the case where we're deleting a limited number
|
||||
-- of items in a sorted set only run a single iteration. If we simply used
|
||||
-- ZRANGE, we may take a long time traversing through jobs that are within the
|
||||
-- grace period.
|
||||
local function getJobsInZset(zsetKey, rangeEnd, limit)
|
||||
if limit > 0 then
|
||||
return rcall("ZRANGEBYSCORE", zsetKey, 0, rangeEnd, "LIMIT", 0, limit)
|
||||
else
|
||||
return rcall("ZRANGEBYSCORE", zsetKey, 0, rangeEnd)
|
||||
end
|
||||
end
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
--[[
|
||||
Function to return the next delayed job timestamp.
|
||||
]]
|
||||
local function getNextDelayedTimestamp(delayedKey)
|
||||
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
|
||||
if #result then
|
||||
local nextTimestamp = tonumber(result[2])
|
||||
if (nextTimestamp ~= nil) then
|
||||
nextTimestamp = nextTimestamp / 0x1000
|
||||
end
|
||||
return nextTimestamp
|
||||
end
|
||||
end
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
|
||||
local function getOrSetMaxEvents(metaKey)
|
||||
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
||||
if not maxEvents then
|
||||
maxEvents = 10000
|
||||
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
|
||||
end
|
||||
return maxEvents
|
||||
end
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
--[[
|
||||
Function to get priority score.
|
||||
]]
|
||||
local function getPriorityScore(priority, priorityCounterKey)
|
||||
local prioCounter = rcall("INCR", priorityCounterKey)
|
||||
return priority * 0x100000000 + prioCounter % 0x100000000
|
||||
end
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
local function getRateLimitTTL(maxJobs, rateLimiterKey)
|
||||
if maxJobs and maxJobs <= tonumber(rcall("GET", rateLimiterKey) or 0) then
|
||||
local pttl = rcall("PTTL", rateLimiterKey)
|
||||
|
||||
if pttl == 0 then
|
||||
rcall("DEL", rateLimiterKey)
|
||||
end
|
||||
|
||||
if pttl > 0 then
|
||||
return pttl
|
||||
end
|
||||
end
|
||||
return 0
|
||||
end
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
--[[
|
||||
Function to check for the meta.paused key to decide if we are paused or not
|
||||
(since an empty list and !EXISTS are not really the same).
|
||||
]]
|
||||
|
||||
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
|
||||
if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
|
||||
return waitKey, false
|
||||
else
|
||||
return pausedKey, true
|
||||
end
|
||||
end
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
--[[
|
||||
Function to get the latest saved timestamp.
|
||||
]]
|
||||
|
||||
local function getTimestamp(jobKey, attributes)
|
||||
if #attributes == 1 then
|
||||
return rcall("HGET", jobKey, attributes[1])
|
||||
end
|
||||
|
||||
local jobTs
|
||||
for _, ts in ipairs(rcall("HMGET", jobKey, unpack(attributes))) do
|
||||
if (ts) then
|
||||
jobTs = ts
|
||||
break
|
||||
end
|
||||
end
|
||||
|
||||
return jobTs
|
||||
end
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
--[[
|
||||
Function to get ZSet items.
|
||||
]]
|
||||
|
||||
local function getZSetItems(keyName, max)
|
||||
return rcall('ZRANGE', keyName, 0, max - 1)
|
||||
end
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
--[[
|
||||
Function to recursively check if there are no locks
|
||||
on the jobs to be removed.
|
||||
|
||||
returns:
|
||||
boolean
|
||||
]]
|
||||
|
||||
local function isLocked( prefix, jobId, removeChildren)
|
||||
local jobKey = prefix .. jobId;
|
||||
|
||||
-- Check if this job is locked
|
||||
local lockKey = jobKey .. ':lock'
|
||||
local lock = rcall("GET", lockKey)
|
||||
if not lock then
|
||||
if removeChildren == "1" then
|
||||
local dependencies = rcall("SMEMBERS", jobKey .. ":dependencies")
|
||||
if (#dependencies > 0) then
|
||||
for i, childJobKey in ipairs(dependencies) do
|
||||
-- We need to get the jobId for this job.
|
||||
local childJobId = getJobIdFromKey(childJobKey)
|
||||
local childJobPrefix = getJobKeyPrefix(childJobKey, childJobId)
|
||||
local result = isLocked( childJobPrefix, childJobId, removeChildren )
|
||||
if result then
|
||||
return true
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
return false
|
||||
end
|
||||
return true
|
||||
end
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
--[[
|
||||
Function to move job from prioritized state to active.
|
||||
]]
|
||||
|
||||
local function moveJobFromPriorityToActive(priorityKey, activeKey, priorityCounterKey)
|
||||
local prioritizedJob = rcall("ZPOPMIN", priorityKey)
|
||||
if #prioritizedJob > 0 then
|
||||
rcall("LPUSH", activeKey, prioritizedJob[1])
|
||||
return prioritizedJob[1]
|
||||
else
|
||||
rcall("DEL", priorityCounterKey)
|
||||
end
|
||||
end
|
||||
Generated
Vendored
+38
@@ -0,0 +1,38 @@
|
||||
--[[
|
||||
Function to recursively move from waitingChildren to failed.
|
||||
]]
|
||||
|
||||
-- Includes
|
||||
--- @include "moveParentToWaitIfNeeded"
|
||||
|
||||
local function moveParentFromWaitingChildrenToFailed( parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
|
||||
if rcall("ZREM", parentQueueKey .. ":waiting-children", parentId) == 1 then
|
||||
rcall("ZADD", parentQueueKey .. ":failed", timestamp, parentId)
|
||||
local failedReason = "child " .. jobIdKey .. " failed"
|
||||
rcall("HMSET", parentKey, "failedReason", failedReason, "finishedOn", timestamp)
|
||||
rcall("XADD", parentQueueKey .. ":events", "*", "event", "failed", "jobId", parentId, "failedReason",
|
||||
failedReason, "prev", "waiting-children")
|
||||
|
||||
local rawParentData = rcall("HGET", parentKey, "parent")
|
||||
|
||||
if rawParentData ~= false then
|
||||
local parentData = cjson.decode(rawParentData)
|
||||
if parentData['fpof'] then
|
||||
moveParentFromWaitingChildrenToFailed(
|
||||
parentData['queueKey'],
|
||||
parentData['queueKey'] .. ':' .. parentData['id'],
|
||||
parentData['id'],
|
||||
parentKey,
|
||||
timestamp
|
||||
)
|
||||
elseif parentData['rdof'] then
|
||||
local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
||||
local grandParentDependenciesSet = grandParentKey .. ":dependencies"
|
||||
if rcall("SREM", grandParentDependenciesSet, parentKey) == 1 then
|
||||
moveParentToWaitIfNeeded(parentData['queueKey'], grandParentDependenciesSet,
|
||||
grandParentKey, parentData['id'], timestamp)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
--[[
|
||||
Validate and move parent to active if needed.
|
||||
]]
|
||||
|
||||
-- Includes
|
||||
--- @include "addDelayMarkerIfNeeded"
|
||||
--- @include "addJobWithPriority"
|
||||
--- @include "getTargetQueueList"
|
||||
|
||||
local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
|
||||
local isParentActive = rcall("ZSCORE", parentQueueKey .. ":waiting-children", parentId)
|
||||
if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then
|
||||
rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
|
||||
local parentWaitKey = parentQueueKey .. ":wait"
|
||||
local parentTarget, paused = getTargetQueueList(parentQueueKey .. ":meta", parentWaitKey,
|
||||
parentQueueKey .. ":paused")
|
||||
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
||||
local priority = tonumber(jobAttributes[1]) or 0
|
||||
local delay = tonumber(jobAttributes[2]) or 0
|
||||
|
||||
if delay > 0 then
|
||||
local delayedTimestamp = tonumber(timestamp) + delay
|
||||
local score = delayedTimestamp * 0x1000
|
||||
local parentDelayedKey = parentQueueKey .. ":delayed"
|
||||
rcall("ZADD", parentDelayedKey, score, parentId)
|
||||
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId,
|
||||
"delay", delayedTimestamp)
|
||||
|
||||
addDelayMarkerIfNeeded(parentTarget, parentDelayedKey)
|
||||
else
|
||||
if priority == 0 then
|
||||
rcall("RPUSH", parentTarget, parentId)
|
||||
else
|
||||
addJobWithPriority(parentWaitKey, parentQueueKey .. ":prioritized", priority, paused,
|
||||
parentId, parentQueueKey .. ":pc")
|
||||
end
|
||||
|
||||
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId,
|
||||
"prev", "waiting-children")
|
||||
end
|
||||
end
|
||||
end
|
||||
+73
@@ -0,0 +1,73 @@
|
||||
|
||||
--[[
|
||||
Function to move job from wait state to active.
|
||||
Input:
|
||||
keys[1] wait key
|
||||
keys[2] active key
|
||||
keys[3] prioritized key
|
||||
keys[4] stream events key
|
||||
keys[5] stalled key
|
||||
|
||||
-- Rate limiting
|
||||
keys[6] rate limiter key
|
||||
keys[7] delayed key
|
||||
|
||||
keys[8] paused key
|
||||
keys[9] meta key
|
||||
keys[10] pc priority counter
|
||||
|
||||
opts - token - lock token
|
||||
opts - lockDuration
|
||||
opts - limiter
|
||||
]]
|
||||
|
||||
-- Includes
|
||||
--- @include "pushBackJobWithPriority"
|
||||
|
||||
local function prepareJobForProcessing(keys, keyPrefix, targetKey, jobId, processedOn,
|
||||
maxJobs, expireTime, opts)
|
||||
local jobKey = keyPrefix .. jobId
|
||||
|
||||
-- Check if we need to perform rate limiting.
|
||||
if maxJobs then
|
||||
local rateLimiterKey = keys[6];
|
||||
|
||||
-- check if we exceeded rate limit, we need to remove the job and return expireTime
|
||||
if expireTime > 0 then
|
||||
-- remove from active queue and add back to the wait list
|
||||
rcall("LREM", keys[2], 1, jobId)
|
||||
|
||||
local priority = tonumber(rcall("HGET", jobKey, "priority")) or 0
|
||||
|
||||
if priority == 0 then
|
||||
rcall("RPUSH", targetKey, jobId)
|
||||
else
|
||||
pushBackJobWithPriority(keys[3], priority, jobId)
|
||||
end
|
||||
|
||||
-- Return when we can process more jobs
|
||||
return {0, 0, expireTime, 0}
|
||||
end
|
||||
|
||||
local jobCounter = tonumber(rcall("INCR", rateLimiterKey))
|
||||
|
||||
if jobCounter == 1 then
|
||||
local limiterDuration = opts['limiter'] and opts['limiter']['duration']
|
||||
local integerDuration = math.floor(math.abs(limiterDuration))
|
||||
rcall("PEXPIRE", rateLimiterKey, integerDuration)
|
||||
end
|
||||
end
|
||||
|
||||
local lockKey = jobKey .. ':lock'
|
||||
|
||||
-- get a lock
|
||||
if opts['token'] ~= "0" then
|
||||
rcall("SET", lockKey, opts['token'], "PX", opts['lockDuration'])
|
||||
end
|
||||
|
||||
rcall("XADD", keys[4], "*", "event", "active", "jobId", jobId, "prev", "waiting")
|
||||
rcall("HSET", jobKey, "processedOn", processedOn)
|
||||
rcall("HINCRBY", jobKey, "attemptsMade", 1)
|
||||
|
||||
return {rcall("HGETALL", jobKey), jobId, 0, 0} -- get job data
|
||||
end
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
--[[
|
||||
Updates the delay set, by moving delayed jobs that should
|
||||
be processed now to "wait".
|
||||
|
||||
Events:
|
||||
'waiting'
|
||||
]]
|
||||
|
||||
-- Includes
|
||||
--- @include "addPriorityMarkerIfNeeded"
|
||||
--- @include "getPriorityScore"
|
||||
|
||||
-- Try to get as much as 1000 jobs at once
|
||||
local function promoteDelayedJobs(delayedKey, waitKey, targetKey, prioritizedKey,
|
||||
eventStreamKey, prefix, timestamp, paused, priorityCounterKey)
|
||||
local jobs = rcall("ZRANGEBYSCORE", delayedKey, 0, (timestamp + 1) * 0x1000, "LIMIT", 0, 1000)
|
||||
|
||||
if (#jobs > 0) then
|
||||
rcall("ZREM", delayedKey, unpack(jobs))
|
||||
|
||||
for _, jobId in ipairs(jobs) do
|
||||
local jobKey = prefix .. jobId
|
||||
local priority =
|
||||
tonumber(rcall("HGET", jobKey, "priority")) or 0
|
||||
|
||||
if priority == 0 then
|
||||
-- LIFO or FIFO
|
||||
rcall("LPUSH", targetKey, jobId)
|
||||
else
|
||||
local score = getPriorityScore(priority, priorityCounterKey)
|
||||
rcall("ZADD", prioritizedKey, score, jobId)
|
||||
end
|
||||
|
||||
-- Emit waiting event
|
||||
rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId",
|
||||
jobId, "prev", "delayed")
|
||||
rcall("HSET", jobKey, "delay", 0)
|
||||
end
|
||||
|
||||
if not paused then
|
||||
addPriorityMarkerIfNeeded(targetKey)
|
||||
end
|
||||
end
|
||||
end
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
--[[
|
||||
Function to push back job considering priority in front of same prioritized jobs.
|
||||
]]
|
||||
local function pushBackJobWithPriority(prioritizedKey, priority, jobId)
|
||||
-- in order to put it at front of same prioritized jobs
|
||||
-- we consider prioritized counter as 0
|
||||
local score = priority * 0x100000000
|
||||
rcall("ZADD", prioritizedKey, score, jobId)
|
||||
end
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
--[[
|
||||
Function to remove job.
|
||||
]]
|
||||
|
||||
-- Includes
|
||||
--- @include "removeParentDependencyKey"
|
||||
|
||||
local function removeJob(jobId, hard, baseKey)
|
||||
local jobKey = baseKey .. jobId
|
||||
removeParentDependencyKey(jobKey, hard, nil, baseKey)
|
||||
rcall("DEL", jobKey, jobKey .. ':logs',
|
||||
jobKey .. ':dependencies', jobKey .. ':processed')
|
||||
end
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
--[[
|
||||
Function to remove from any state.
|
||||
|
||||
returns:
|
||||
prev state
|
||||
]]
|
||||
|
||||
local function removeJobFromAnyState( prefix, jobId)
|
||||
-- We start with the ZSCORE checks, since they have O(1) complexity
|
||||
if rcall("ZSCORE", prefix .. "completed", jobId) then
|
||||
rcall("ZREM", prefix .. "completed", jobId)
|
||||
return "completed"
|
||||
elseif rcall("ZSCORE", prefix .. "waiting-children", jobId) then
|
||||
rcall("ZREM", prefix .. "waiting-children", jobId)
|
||||
return "waiting-children"
|
||||
elseif rcall("ZSCORE", prefix .. "delayed", jobId) then
|
||||
rcall("ZREM", prefix .. "delayed", jobId)
|
||||
return "delayed"
|
||||
elseif rcall("ZSCORE", prefix .. "failed", jobId) then
|
||||
rcall("ZREM", prefix .. "failed", jobId)
|
||||
return "failed"
|
||||
elseif rcall("ZSCORE", prefix .. "prioritized", jobId) then
|
||||
rcall("ZREM", prefix .. "prioritized", jobId)
|
||||
return "prioritized"
|
||||
-- We remove only 1 element from the list, since we assume they are not added multiple times
|
||||
elseif rcall("LREM", prefix .. "wait", 1, jobId) == 1 then
|
||||
return "wait"
|
||||
elseif rcall("LREM", prefix .. "paused", 1, jobId) == 1 then
|
||||
return "paused"
|
||||
elseif rcall("LREM", prefix .. "active", 1, jobId) == 1 then
|
||||
return "active"
|
||||
end
|
||||
|
||||
return "unknown"
|
||||
end
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
--[[
|
||||
Functions to remove jobs.
|
||||
]]
|
||||
|
||||
-- Includes
|
||||
--- @include "removeJob"
|
||||
|
||||
local function removeJobs(keys, hard, baseKey, max)
|
||||
for i, key in ipairs(keys) do
|
||||
removeJob(key, hard, baseKey)
|
||||
end
|
||||
return max - #keys
|
||||
end
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
--[[
|
||||
Functions to remove jobs by max age.
|
||||
]]
|
||||
|
||||
-- Includes
|
||||
--- @include "removeJob"
|
||||
|
||||
local function removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
|
||||
local start = timestamp - maxAge * 1000
|
||||
local jobIds = rcall("ZREVRANGEBYSCORE", targetSet, start, "-inf")
|
||||
for i, jobId in ipairs(jobIds) do
|
||||
removeJob(jobId, false, prefix)
|
||||
end
|
||||
rcall("ZREMRANGEBYSCORE", targetSet, "-inf", start)
|
||||
end
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
--[[
|
||||
Functions to remove jobs by max count.
|
||||
]]
|
||||
|
||||
-- Includes
|
||||
--- @include "removeJob"
|
||||
|
||||
local function removeJobsByMaxCount(maxCount, targetSet, prefix)
|
||||
local start = maxCount
|
||||
local jobIds = rcall("ZREVRANGE", targetSet, start, -1)
|
||||
for i, jobId in ipairs(jobIds) do
|
||||
removeJob(jobId, false, prefix)
|
||||
end
|
||||
rcall("ZREMRANGEBYRANK", targetSet, 0, -(maxCount + 1))
|
||||
end
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user