Initial commit of working RSS Aggregator build

This commit is contained in:
2026-05-12 17:04:02 -03:00
parent ea3a2ca53e
commit 7ac2f6e384
4962 changed files with 1032666 additions and 0 deletions
+97
View File
@@ -0,0 +1,97 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.AsyncFifoQueue = void 0;
/**
* AsyncFifoQueue
*
* A minimal FIFO queue for asynchronous operations. Allows adding asynchronous operations
* and consume them in the order they are resolved.
*
* TODO: Optimize using a linked list for the queue instead of an array.
* Current implementation requires memory copies when shifting the queue.
* For a linked linked implementation, we can exploit the fact that the
* maximum number of elements in the list will never exceen the concurrency factor
* of the worker, so the nodes of the list could be pre-allocated.
*/
class AsyncFifoQueue {
constructor(ignoreErrors = false) {
this.ignoreErrors = ignoreErrors;
/**
* A queue of completed promises. As the pending
* promises are resolved, they are added to this queue.
*/
this.queue = [];
/**
* A set of pending promises.
*/
this.pending = new Set();
this.newPromise();
}
add(promise) {
this.pending.add(promise);
promise
.then(data => {
this.pending.delete(promise);
if (this.queue.length === 0) {
this.resolvePromise(data);
}
this.queue.push(data);
})
.catch(err => {
// Ignore errors
if (this.ignoreErrors) {
this.queue.push(undefined);
}
this.pending.delete(promise);
this.rejectPromise(err);
});
}
async waitAll() {
await Promise.all(this.pending);
}
numTotal() {
return this.pending.size + this.queue.length;
}
numPending() {
return this.pending.size;
}
numQueued() {
return this.queue.length;
}
resolvePromise(data) {
this.resolve(data);
this.newPromise();
}
rejectPromise(err) {
this.reject(err);
this.newPromise();
}
newPromise() {
this.nextPromise = new Promise((resolve, reject) => {
this.resolve = resolve;
this.reject = reject;
});
}
async wait() {
return this.nextPromise;
}
async fetch() {
if (this.pending.size === 0 && this.queue.length === 0) {
return;
}
while (this.queue.length === 0) {
try {
await this.wait();
}
catch (err) {
// Ignore errors
if (!this.ignoreErrors) {
console.error('Unexpected Error in AsyncFifoQueue', err);
}
}
}
return this.queue.shift();
}
}
exports.AsyncFifoQueue = AsyncFifoQueue;
//# sourceMappingURL=async-fifo-queue.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"async-fifo-queue.js","sourceRoot":"","sources":["../../../src/classes/async-fifo-queue.ts"],"names":[],"mappings":";;;AAAA;;;;;;;;;;;GAWG;AACH,MAAa,cAAc;IAqBzB,YAAoB,eAAe,KAAK;QAApB,iBAAY,GAAZ,YAAY,CAAQ;QApBxC;;;WAGG;QACK,UAAK,GAAsB,EAAE,CAAC;QAEtC;;WAEG;QACK,YAAO,GAAG,IAAI,GAAG,EAAc,CAAC;QAYtC,IAAI,CAAC,UAAU,EAAE,CAAC;IACpB,CAAC;IAEM,GAAG,CAAC,OAAmB;QAC5B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAE1B,OAAO;aACJ,IAAI,CAAC,IAAI,CAAC,EAAE;YACX,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAE7B,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;gBAC3B,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;aAC3B;YACD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,CAAC,CAAC;aACD,KAAK,CAAC,GAAG,CAAC,EAAE;YACX,gBAAgB;YAChB,IAAI,IAAI,CAAC,YAAY,EAAE;gBACrB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;aAC5B;YACD,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC7B,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;IACP,CAAC;IAEM,KAAK,CAAC,OAAO;QAClB,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAClC,CAAC;IAEM,QAAQ;QACb,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAC/C,CAAC;IAEM,UAAU;QACf,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;IAC3B,CAAC;IAEM,SAAS;QACd,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAC3B,CAAC;IAEO,cAAc,CAAC,IAAO;QAC5B,IAAI,CAAC,OAAQ,CAAC,IAAI,CAAC,CAAC;QACpB,IAAI,CAAC,UAAU,EAAE,CAAC;IACpB,CAAC;IAEO,aAAa,CAAC,GAAQ;QAC5B,IAAI,CAAC,MAAO,CAAC,GAAG,CAAC,CAAC;QAClB,IAAI,CAAC,UAAU,EAAE,CAAC;IACpB,CAAC;IAEO,UAAU;QAChB,IAAI,CAAC,WAAW,GAAG,IAAI,OAAO,CAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAChE,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;YACvB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACvB,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,IAAI;QAChB,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAEM,KAAK,CAAC,KAAK;QAChB,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;YACtD,OAAO;SACR;QACD,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;YAC9B,IAAI;gBACF,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;aACnB;YAAC,OAAO,GAAG,EAAE;gBACZ,gBAAgB;gBAChB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;oBACtB,OAAO,CAAC,KAAK,CAAC,oCAAoC,EAAE,GAAG,CAAC,CAAC;iBAC1D;aACF;SACF;QACD,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IAC5B,CAAC;CACF;AApGD,wCAoGC"}
+48
View File
@@ -0,0 +1,48 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.Backoffs = void 0;
class Backoffs {
static normalize(backoff) {
if (Number.isFinite(backoff)) {
return {
type: 'fixed',
delay: backoff,
};
}
else if (backoff) {
return backoff;
}
}
static calculate(backoff, attemptsMade, err, job, customStrategy) {
if (backoff) {
const strategy = lookupStrategy(backoff, customStrategy);
return strategy(attemptsMade, backoff.type, err, job);
}
}
}
exports.Backoffs = Backoffs;
Backoffs.builtinStrategies = {
fixed: function (delay) {
return function () {
return delay;
};
},
exponential: function (delay) {
return function (attemptsMade) {
return Math.round(Math.pow(2, attemptsMade - 1) * delay);
};
},
};
function lookupStrategy(backoff, customStrategy) {
if (backoff.type in Backoffs.builtinStrategies) {
return Backoffs.builtinStrategies[backoff.type](backoff.delay);
}
else if (customStrategy) {
return customStrategy;
}
else {
throw new Error(`Unknown backoff strategy ${backoff.type}.
If a custom backoff strategy is used, specify it when the queue is created.`);
}
}
//# sourceMappingURL=backoffs.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"backoffs.js","sourceRoot":"","sources":["../../../src/classes/backoffs.ts"],"names":[],"mappings":";;;AAOA,MAAa,QAAQ;IAenB,MAAM,CAAC,SAAS,CACd,OAAgC;QAEhC,IAAI,MAAM,CAAC,QAAQ,CAAS,OAAO,CAAC,EAAE;YACpC,OAAO;gBACL,IAAI,EAAE,OAAO;gBACb,KAAK,EAAU,OAAO;aACvB,CAAC;SACH;aAAM,IAAI,OAAO,EAAE;YAClB,OAAuB,OAAO,CAAC;SAChC;IACH,CAAC;IAED,MAAM,CAAC,SAAS,CACd,OAAuB,EACvB,YAAoB,EACpB,GAAU,EACV,GAAe,EACf,cAAgC;QAEhC,IAAI,OAAO,EAAE;YACX,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;YAEzD,OAAO,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;SACvD;IACH,CAAC;;AAxCH,4BAyCC;AAxCQ,0BAAiB,GAAsB;IAC5C,KAAK,EAAE,UAAU,KAAa;QAC5B,OAAO;YACL,OAAO,KAAK,CAAC;QACf,CAAC,CAAC;IACJ,CAAC;IAED,WAAW,EAAE,UAAU,KAAa;QAClC,OAAO,UAAU,YAAoB;YACnC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,YAAY,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC;QAC3D,CAAC,CAAC;IACJ,CAAC;CACF,CAAC;AA8BJ,SAAS,cAAc,CACrB,OAAuB,EACvB,cAAgC;IAEhC,IAAI,OAAO,CAAC,IAAI,IAAI,QAAQ,CAAC,iBAAiB,EAAE;QAC9C,OAAO,QAAQ,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAM,CAAC,CAAC;KACjE;SAAM,IAAI,cAAc,EAAE;QACzB,OAAO,cAAc,CAAC;KACvB;SAAM;QACL,MAAM,IAAI,KAAK,CACb,4BAA4B,OAAO,CAAC,IAAI;kFACoC,CAC7E,CAAC;KACH;AACH,CAAC"}
+64
View File
@@ -0,0 +1,64 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.ChildPool = void 0;
const path = require("path");
const child_1 = require("./child");
const CHILD_KILL_TIMEOUT = 30000;
class ChildPool {
constructor({ mainFile = path.join(process.cwd(), 'dist/cjs/classes/main.js'), useWorkerThreads, }) {
this.retained = {};
this.free = {};
this.opts = { mainFile, useWorkerThreads };
}
async retain(processFile) {
let child = this.getFree(processFile).pop();
if (child) {
this.retained[child.pid] = child;
return child;
}
child = new child_1.Child(this.opts.mainFile, processFile, {
useWorkerThreads: this.opts.useWorkerThreads,
});
child.on('exit', this.remove.bind(this, child));
try {
await child.init();
this.retained[child.pid] = child;
return child;
}
catch (err) {
console.error(err);
this.release(child);
throw err;
}
}
release(child) {
delete this.retained[child.pid];
this.getFree(child.processFile).push(child);
}
remove(child) {
delete this.retained[child.pid];
const free = this.getFree(child.processFile);
const childIndex = free.indexOf(child);
if (childIndex > -1) {
free.splice(childIndex, 1);
}
}
async kill(child, signal = 'SIGKILL') {
this.remove(child);
return child.kill(signal, CHILD_KILL_TIMEOUT);
}
async clean() {
const children = Object.values(this.retained).concat(this.getAllFree());
this.retained = {};
this.free = {};
await Promise.all(children.map(c => this.kill(c, 'SIGTERM')));
}
getFree(id) {
return (this.free[id] = this.free[id] || []);
}
getAllFree() {
return Object.values(this.free).reduce((first, second) => first.concat(second), []);
}
}
exports.ChildPool = ChildPool;
//# sourceMappingURL=child-pool.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"child-pool.js","sourceRoot":"","sources":["../../../src/classes/child-pool.ts"],"names":[],"mappings":";;;AAAA,6BAA6B;AAC7B,mCAAgC;AAEhC,MAAM,kBAAkB,GAAG,KAAM,CAAC;AAOlC,MAAa,SAAS;IAKpB,YAAY,EACV,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,0BAA0B,CAAC,EAC/D,gBAAgB,GACF;QAPhB,aAAQ,GAA6B,EAAE,CAAC;QACxC,SAAI,GAA+B,EAAE,CAAC;QAOpC,IAAI,CAAC,IAAI,GAAG,EAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,WAAmB;QAC9B,IAAI,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,GAAG,EAAE,CAAC;QAE5C,IAAI,KAAK,EAAE;YACT,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;YACjC,OAAO,KAAK,CAAC;SACd;QAED,KAAK,GAAG,IAAI,aAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE;YACjD,gBAAgB,EAAE,IAAI,CAAC,IAAI,CAAC,gBAAgB;SAC7C,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;QAEhD,IAAI;YACF,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;YAEjC,OAAO,KAAK,CAAC;SACd;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACnB,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,GAAG,CAAC;SACX;IACH,CAAC;IAED,OAAO,CAAC,KAAY;QAClB,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,CAAC,KAAY;QACjB,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAEhC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QAE7C,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACvC,IAAI,UAAU,GAAG,CAAC,CAAC,EAAE;YACnB,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;SAC5B;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CACR,KAAY,EACZ,SAAgC,SAAS;QAEzC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QACxE,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;QACnB,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC;QAEf,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC;IAChE,CAAC;IAED,OAAO,CAAC,EAAU;QAChB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;IAC/C,CAAC;IAED,UAAU;QACR,OAAO,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,CACpC,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,EACvC,EAAE,CACH,CAAC;IACJ,CAAC;CACF;AA/ED,8BA+EC"}
+152
View File
@@ -0,0 +1,152 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.ChildProcessor = void 0;
const enums_1 = require("../enums");
const utils_1 = require("../utils");
var ChildStatus;
(function (ChildStatus) {
ChildStatus[ChildStatus["Idle"] = 0] = "Idle";
ChildStatus[ChildStatus["Started"] = 1] = "Started";
ChildStatus[ChildStatus["Terminating"] = 2] = "Terminating";
ChildStatus[ChildStatus["Errored"] = 3] = "Errored";
})(ChildStatus || (ChildStatus = {}));
/**
* ChildProcessor
*
* This class acts as the interface between a child process and it parent process
* so that jobs can be processed in different processes.
*
*/
class ChildProcessor {
constructor(send) {
this.send = send;
}
async init(processorFile) {
let processor;
try {
const { default: processorFn } = await import(processorFile);
processor = processorFn;
if (processor.default) {
// support es2015 module.
processor = processor.default;
}
if (typeof processor !== 'function') {
throw new Error('No function is exported in processor file');
}
}
catch (err) {
this.status = ChildStatus.Errored;
return this.send({
cmd: enums_1.ParentCommand.InitFailed,
err: (0, utils_1.errorToJSON)(err),
});
}
const origProcessor = processor;
processor = function (job, token) {
try {
return Promise.resolve(origProcessor(job, token));
}
catch (err) {
return Promise.reject(err);
}
};
this.processor = processor;
this.status = ChildStatus.Idle;
await this.send({
cmd: enums_1.ParentCommand.InitCompleted,
});
}
async start(jobJson, token) {
if (this.status !== ChildStatus.Idle) {
return this.send({
cmd: enums_1.ParentCommand.Error,
err: (0, utils_1.errorToJSON)(new Error('cannot start a not idling child process')),
});
}
this.status = ChildStatus.Started;
this.currentJobPromise = (async () => {
try {
const job = this.wrapJob(jobJson, this.send);
const result = await this.processor(job, token);
await this.send({
cmd: enums_1.ParentCommand.Completed,
value: typeof result === 'undefined' ? null : result,
});
}
catch (err) {
await this.send({
cmd: enums_1.ParentCommand.Failed,
value: (0, utils_1.errorToJSON)(!err.message ? new Error(err) : err),
});
}
finally {
this.status = ChildStatus.Idle;
this.currentJobPromise = undefined;
}
})();
}
async stop() { }
async waitForCurrentJobAndExit() {
this.status = ChildStatus.Terminating;
try {
await this.currentJobPromise;
}
finally {
process.exit(process.exitCode || 0);
}
}
/**
* Enhance the given job argument with some functions
* that can be called from the sandboxed job processor.
*
* Note, the `job` argument is a JSON deserialized message
* from the main node process to this forked child process,
* the functions on the original job object are not in tact.
* The wrapped job adds back some of those original functions.
*/
wrapJob(job, send) {
return Object.assign(Object.assign({}, job), { data: JSON.parse(job.data || '{}'), opts: job.opts, returnValue: JSON.parse(job.returnvalue || '{}'),
/*
* Emulate the real job `updateProgress` function, should works as `progress` function.
*/
async updateProgress(progress) {
// Locally store reference to new progress value
// so that we can return it from this process synchronously.
this.progress = progress;
// Send message to update job progress.
await send({
cmd: enums_1.ParentCommand.Progress,
value: progress,
});
},
/*
* Emulate the real job `log` function.
*/
log: async (row) => {
send({
cmd: enums_1.ParentCommand.Log,
value: row,
});
},
/*
* Emulate the real job `moveToDelayed` function.
*/
moveToDelayed: async (timestamp, token) => {
send({
cmd: enums_1.ParentCommand.MoveToDelayed,
value: { timestamp, token },
});
},
/*
* Emulate the real job `updateData` function.
*/
updateData: async (data) => {
send({
cmd: enums_1.ParentCommand.Update,
value: data,
});
} });
}
}
exports.ChildProcessor = ChildProcessor;
//# sourceMappingURL=child-processor.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"child-processor.js","sourceRoot":"","sources":["../../../src/classes/child-processor.ts"],"names":[],"mappings":";;;AAAA,oCAAyC;AAEzC,oCAAuC;AAEvC,IAAK,WAKJ;AALD,WAAK,WAAW;IACd,6CAAI,CAAA;IACJ,mDAAO,CAAA;IACP,2DAAW,CAAA;IACX,mDAAO,CAAA;AACT,CAAC,EALI,WAAW,KAAX,WAAW,QAKf;AAED;;;;;;GAMG;AACH,MAAa,cAAc;IAKzB,YAAoB,IAAiC;QAAjC,SAAI,GAAJ,IAAI,CAA6B;IAAG,CAAC;IAElD,KAAK,CAAC,IAAI,CAAC,aAAqB;QACrC,IAAI,SAAS,CAAC;QACd,IAAI;YACF,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;YAC7D,SAAS,GAAG,WAAW,CAAC;YAExB,IAAI,SAAS,CAAC,OAAO,EAAE;gBACrB,yBAAyB;gBACzB,SAAS,GAAG,SAAS,CAAC,OAAO,CAAC;aAC/B;YAED,IAAI,OAAO,SAAS,KAAK,UAAU,EAAE;gBACnC,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;aAC9D;SACF;QAAC,OAAO,GAAG,EAAE;YACZ,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC;YAClC,OAAO,IAAI,CAAC,IAAI,CAAC;gBACf,GAAG,EAAE,qBAAa,CAAC,UAAU;gBAC7B,GAAG,EAAE,IAAA,mBAAW,EAAC,GAAG,CAAC;aACtB,CAAC,CAAC;SACJ;QAED,MAAM,aAAa,GAAG,SAAS,CAAC;QAChC,SAAS,GAAG,UAAU,GAAiB,EAAE,KAAc;YACrD,IAAI;gBACF,OAAO,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;aACnD;YAAC,OAAO,GAAG,EAAE;gBACZ,OAAO,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;aAC5B;QACH,CAAC,CAAC;QAEF,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC;QAC/B,MAAM,IAAI,CAAC,IAAI,CAAC;YACd,GAAG,EAAE,qBAAa,CAAC,aAAa;SACjC,CAAC,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,KAAK,CAAC,OAAgB,EAAE,KAAc;QACjD,IAAI,IAAI,CAAC,MAAM,KAAK,WAAW,CAAC,IAAI,EAAE;YACpC,OAAO,IAAI,CAAC,IAAI,CAAC;gBACf,GAAG,EAAE,qBAAa,CAAC,KAAK;gBACxB,GAAG,EAAE,IAAA,mBAAW,EAAC,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;aACvE,CAAC,CAAC;SACJ;QACD,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC;QAClC,IAAI,CAAC,iBAAiB,GAAG,CAAC,KAAK,IAAI,EAAE;YACnC,IAAI;gBACF,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC7C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;gBAChD,MAAM,IAAI,CAAC,IAAI,CAAC;oBACd,GAAG,EAAE,qBAAa,CAAC,SAAS;oBAC5B,KAAK,EAAE,OAAO,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM;iBACrD,CAAC,CAAC;aACJ;YAAC,OAAO,GAAG,EAAE;gBACZ,MAAM,IAAI,CAAC,IAAI,CAAC;oBACd,GAAG,EAAE,qBAAa,CAAC,MAAM;oBACzB,KAAK,EAAE,IAAA,mBAAW,EAAC,CAAS,GAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,KAAK,CAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;iBACtE,CAAC,CAAC;aACJ;oBAAS;gBACR,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC;gBAC/B,IAAI,CAAC,iBAAiB,GAAG,SAAS,CAAC;aACpC;QACH,CAAC,CAAC,EAAE,CAAC;IACP,CAAC;IAEM,KAAK,CAAC,IAAI,KAAmB,CAAC;IAErC,KAAK,CAAC,wBAAwB;QAC5B,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,WAAW,CAAC;QACtC,IAAI;YACF,MAAM,IAAI,CAAC,iBAAiB,CAAC;SAC9B;gBAAS;YACR,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;SACrC;IACH,CAAC;IAED;;;;;;;;OAQG;IACO,OAAO,CACf,GAAY,EACZ,IAAiC;QAEjC,uCACK,GAAG,KACN,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,IAAI,IAAI,CAAC,EAClC,IAAI,EAAE,GAAG,CAAC,IAAI,EACd,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,IAAI,IAAI,CAAC;YAChD;;eAEG;YACH,KAAK,CAAC,cAAc,CAAC,QAAyB;gBAC5C,gDAAgD;gBAChD,4DAA4D;gBAC5D,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;gBACzB,uCAAuC;gBACvC,MAAM,IAAI,CAAC;oBACT,GAAG,EAAE,qBAAa,CAAC,QAAQ;oBAC3B,KAAK,EAAE,QAAQ;iBAChB,CAAC,CAAC;YACL,CAAC;YACD;;eAEG;YACH,GAAG,EAAE,KAAK,EAAE,GAAQ,EAAE,EAAE;gBACtB,IAAI,CAAC;oBACH,GAAG,EAAE,qBAAa,CAAC,GAAG;oBACtB,KAAK,EAAE,GAAG;iBACX,CAAC,CAAC;YACL,CAAC;YACD;;eAEG;YACH,aAAa,EAAE,KAAK,EAAE,SAAiB,EAAE,KAAc,EAAE,EAAE;gBACzD,IAAI,CAAC;oBACH,GAAG,EAAE,qBAAa,CAAC,aAAa;oBAChC,KAAK,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE;iBAC5B,CAAC,CAAC;YACL,CAAC;YACD;;eAEG;YACH,UAAU,EAAE,KAAK,EAAE,IAAS,EAAE,EAAE;gBAC9B,IAAI,CAAC;oBACH,GAAG,EAAE,qBAAa,CAAC,MAAM;oBACzB,KAAK,EAAE,IAAI;iBACZ,CAAC,CAAC;YACL,CAAC,IACD;IACJ,CAAC;CACF;AAhJD,wCAgJC"}
+216
View File
@@ -0,0 +1,216 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.Child = void 0;
const child_process_1 = require("child_process");
const worker_threads_1 = require("worker_threads");
const net_1 = require("net");
const enums_1 = require("../enums");
const events_1 = require("events");
/**
* @see https://nodejs.org/api/process.html#process_exit_codes
*/
const exitCodesErrors = {
1: 'Uncaught Fatal Exception',
2: 'Unused',
3: 'Internal JavaScript Parse Error',
4: 'Internal JavaScript Evaluation Failure',
5: 'Fatal Error',
6: 'Non-function Internal Exception Handler',
7: 'Internal Exception Handler Run-Time Failure',
8: 'Unused',
9: 'Invalid Argument',
10: 'Internal JavaScript Run-Time Failure',
12: 'Invalid Debug Argument',
13: 'Unfinished Top-Level Await',
};
/**
* Child class
*
* This class is used to create a child process or worker thread, and allows using
* isolated processes or threads for processing jobs.
*
*/
class Child extends events_1.EventEmitter {
constructor(mainFile, processFile, opts = {
useWorkerThreads: false,
}) {
super();
this.mainFile = mainFile;
this.processFile = processFile;
this.opts = opts;
this._exitCode = null;
this._signalCode = null;
this._killed = false;
}
get pid() {
if (this.childProcess) {
return this.childProcess.pid;
}
else if (this.worker) {
return this.worker.threadId;
}
else {
throw new Error('No child process or worker thread');
}
}
get exitCode() {
return this._exitCode;
}
get signalCode() {
return this._signalCode;
}
get killed() {
if (this.childProcess) {
return this.childProcess.killed;
}
return this._killed;
}
async init() {
const execArgv = await convertExecArgv(process.execArgv);
let parent;
if (this.opts.useWorkerThreads) {
this.worker = parent = new worker_threads_1.Worker(this.mainFile, {
execArgv,
stdin: true,
stdout: true,
stderr: true,
});
}
else {
this.childProcess = parent = (0, child_process_1.fork)(this.mainFile, [], {
execArgv,
stdio: 'pipe',
});
}
parent.on('exit', (exitCode, signalCode) => {
this._exitCode = exitCode;
// Coerce to null if undefined for backwards compatibility
signalCode = typeof signalCode === 'undefined' ? null : signalCode;
this._signalCode = signalCode;
this._killed = true;
this.emit('exit', exitCode, signalCode);
// Clean all listeners, we do not expect any more events after "exit"
parent.removeAllListeners();
this.removeAllListeners();
});
parent.on('error', (...args) => this.emit('error', ...args));
parent.on('message', (...args) => this.emit('message', ...args));
parent.on('close', (...args) => this.emit('close', ...args));
parent.stdout.pipe(process.stdout);
parent.stderr.pipe(process.stderr);
await this.initChild();
}
async send(msg) {
return new Promise((resolve, reject) => {
if (this.childProcess) {
this.childProcess.send(msg, (err) => {
if (err) {
reject(err);
}
else {
resolve();
}
});
}
else if (this.worker) {
resolve(this.worker.postMessage(msg));
}
else {
resolve();
}
});
}
killProcess(signal = 'SIGKILL') {
if (this.childProcess) {
this.childProcess.kill(signal);
}
else if (this.worker) {
this.worker.terminate();
}
}
async kill(signal = 'SIGKILL', timeoutMs) {
if (this.hasProcessExited()) {
return;
}
const onExit = onExitOnce(this.childProcess || this.worker);
this.killProcess(signal);
if (timeoutMs !== undefined && (timeoutMs === 0 || isFinite(timeoutMs))) {
const timeoutHandle = setTimeout(() => {
if (!this.hasProcessExited()) {
this.killProcess('SIGKILL');
}
}, timeoutMs);
await onExit;
clearTimeout(timeoutHandle);
}
await onExit;
}
async initChild() {
const onComplete = new Promise((resolve, reject) => {
const onMessageHandler = (msg) => {
if (msg.cmd === enums_1.ParentCommand.InitCompleted) {
resolve();
}
else if (msg.cmd === enums_1.ParentCommand.InitFailed) {
const err = new Error();
err.stack = msg.err.stack;
err.message = msg.err.message;
reject(err);
}
this.off('message', onMessageHandler);
this.off('close', onCloseHandler);
};
const onCloseHandler = (code, signal) => {
if (code > 128) {
code -= 128;
}
const msg = exitCodesErrors[code] || `Unknown exit code ${code}`;
reject(new Error(`Error initializing child: ${msg} and signal ${signal}`));
this.off('message', onMessageHandler);
this.off('close', onCloseHandler);
};
this.on('message', onMessageHandler);
this.on('close', onCloseHandler);
});
await this.send({
cmd: enums_1.ChildCommand.Init,
value: this.processFile,
});
await onComplete;
}
hasProcessExited() {
return !!(this.exitCode !== null || this.signalCode);
}
}
exports.Child = Child;
function onExitOnce(child) {
return new Promise(resolve => {
child.once('exit', () => resolve());
});
}
const getFreePort = async () => {
return new Promise(resolve => {
const server = (0, net_1.createServer)();
server.listen(0, () => {
const { port } = server.address();
server.close(() => resolve(port));
});
});
};
const convertExecArgv = async (execArgv) => {
const standard = [];
const convertedArgs = [];
for (let i = 0; i < execArgv.length; i++) {
const arg = execArgv[i];
if (arg.indexOf('--inspect') === -1) {
standard.push(arg);
}
else {
const argName = arg.split('=')[0];
const port = await getFreePort();
convertedArgs.push(`${argName}=${port}`);
}
}
return standard.concat(convertedArgs);
};
//# sourceMappingURL=child.js.map
File diff suppressed because one or more lines are too long
+19
View File
@@ -0,0 +1,19 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.DelayedError = void 0;
/**
* DelayedError
*
* Error to be thrown when job is moved to delayed state
* from job in active state.
*
*/
class DelayedError extends Error {
constructor(message) {
super(message);
this.name = this.constructor.name;
Object.setPrototypeOf(this, new.target.prototype);
}
}
exports.DelayedError = DelayedError;
//# sourceMappingURL=delayed-error.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"delayed-error.js","sourceRoot":"","sources":["../../../../src/classes/errors/delayed-error.ts"],"names":[],"mappings":";;;AAAA;;;;;;GAMG;AACH,MAAa,YAAa,SAAQ,KAAK;IACrC,YAAY,OAAgB;QAC1B,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;QAClC,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACpD,CAAC;CACF;AAND,oCAMC"}
+8
View File
@@ -0,0 +1,8 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const tslib_1 = require("tslib");
tslib_1.__exportStar(require("./delayed-error"), exports);
tslib_1.__exportStar(require("./unrecoverable-error"), exports);
tslib_1.__exportStar(require("./rate-limit-error"), exports);
tslib_1.__exportStar(require("./waiting-children-error"), exports);
//# sourceMappingURL=index.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/classes/errors/index.ts"],"names":[],"mappings":";;;AAAA,0DAAgC;AAChC,gEAAsC;AACtC,6DAAmC;AACnC,mEAAyC"}
+19
View File
@@ -0,0 +1,19 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.RateLimitError = exports.RATE_LIMIT_ERROR = void 0;
exports.RATE_LIMIT_ERROR = 'bullmq:rateLimitExceeded';
/**
* RateLimitError
*
* Error to be thrown when queue reaches a rate limit.
*
*/
class RateLimitError extends Error {
constructor(message = exports.RATE_LIMIT_ERROR) {
super(message);
this.name = this.constructor.name;
Object.setPrototypeOf(this, new.target.prototype);
}
}
exports.RateLimitError = RateLimitError;
//# sourceMappingURL=rate-limit-error.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"rate-limit-error.js","sourceRoot":"","sources":["../../../../src/classes/errors/rate-limit-error.ts"],"names":[],"mappings":";;;AAAa,QAAA,gBAAgB,GAAG,0BAA0B,CAAC;AAE3D;;;;;GAKG;AACH,MAAa,cAAe,SAAQ,KAAK;IACvC,YAAY,UAAkB,wBAAgB;QAC5C,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;QAClC,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACpD,CAAC;CACF;AAND,wCAMC"}
+19
View File
@@ -0,0 +1,19 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.UnrecoverableError = void 0;
/**
* UnrecoverableError
*
* Error to move a job to failed even if the attemptsMade
* are lower than the expected limit.
*
*/
class UnrecoverableError extends Error {
constructor(message) {
super(message);
this.name = this.constructor.name;
Object.setPrototypeOf(this, new.target.prototype);
}
}
exports.UnrecoverableError = UnrecoverableError;
//# sourceMappingURL=unrecoverable-error.js.map
@@ -0,0 +1 @@
{"version":3,"file":"unrecoverable-error.js","sourceRoot":"","sources":["../../../../src/classes/errors/unrecoverable-error.ts"],"names":[],"mappings":";;;AAAA;;;;;;GAMG;AACH,MAAa,kBAAmB,SAAQ,KAAK;IAC3C,YAAY,OAAgB;QAC1B,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;QAClC,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACpD,CAAC;CACF;AAND,gDAMC"}
+19
View File
@@ -0,0 +1,19 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.WaitingChildrenError = void 0;
/**
* WaitingChildrenError
*
* Error to be thrown when job is moved to waiting-children state
* from job in active state.
*
*/
class WaitingChildrenError extends Error {
constructor(message) {
super(message);
this.name = this.constructor.name;
Object.setPrototypeOf(this, new.target.prototype);
}
}
exports.WaitingChildrenError = WaitingChildrenError;
//# sourceMappingURL=waiting-children-error.js.map
@@ -0,0 +1 @@
{"version":3,"file":"waiting-children-error.js","sourceRoot":"","sources":["../../../../src/classes/errors/waiting-children-error.ts"],"names":[],"mappings":";;;AAAA;;;;;;GAMG;AACH,MAAa,oBAAqB,SAAQ,KAAK;IAC7C,YAAY,OAAgB;QAC1B,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;QAClC,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACpD,CAAC;CACF;AAND,oDAMC"}
+309
View File
@@ -0,0 +1,309 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.FlowProducer = void 0;
const events_1 = require("events");
const lodash_1 = require("lodash");
const uuid_1 = require("uuid");
const utils_1 = require("../utils");
const job_1 = require("./job");
const queue_keys_1 = require("./queue-keys");
const redis_connection_1 = require("./redis-connection");
/**
* This class allows to add jobs with dependencies between them in such
* a way that it is possible to build complex flows.
* Note: A flow is a tree-like structure of jobs that depend on each other.
* Whenever the children of a given parent are completed, the parent
* will be processed, being able to access the children's result data.
* All Jobs can be in different queues, either children or parents,
*/
class FlowProducer extends events_1.EventEmitter {
constructor(opts = {}, Connection = redis_connection_1.RedisConnection) {
super();
this.opts = opts;
this.opts = Object.assign({ prefix: 'bull' }, opts);
if (!opts.connection) {
console.warn([
'BullMQ: DEPRECATION WARNING! Optional instantiation of Queue, Worker, QueueEvents and FlowProducer',
'without providing explicitly a connection or connection options is deprecated. This behaviour will',
'be removed in the next major release',
].join(' '));
}
this.connection = new Connection(opts.connection, (0, utils_1.isRedisInstance)(opts === null || opts === void 0 ? void 0 : opts.connection), false, opts.skipVersionCheck);
this.connection.on('error', (error) => this.emit('error', error));
this.connection.on('close', () => {
if (!this.closing) {
this.emit('ioredis:close');
}
});
this.queueKeys = new queue_keys_1.QueueKeys(opts.prefix);
}
emit(event, ...args) {
return super.emit(event, ...args);
}
off(eventName, listener) {
super.off(eventName, listener);
return this;
}
on(event, listener) {
super.on(event, listener);
return this;
}
once(event, listener) {
super.once(event, listener);
return this;
}
/**
* Returns a promise that resolves to a redis client. Normally used only by subclasses.
*/
get client() {
return this.connection.client;
}
/**
* Helper to easily extend Job class calls.
*/
get Job() {
return job_1.Job;
}
waitUntilReady() {
return this.client;
}
/**
* Adds a flow.
*
* This call would be atomic, either it fails and no jobs will
* be added to the queues, or it succeeds and all jobs will be added.
*
* @param flow - an object with a tree-like structure where children jobs
* will be processed before their parents.
* @param opts - options that will be applied to the flow object.
*/
async add(flow, opts) {
var _a;
if (this.closing) {
return;
}
const client = await this.connection.client;
const multi = client.multi();
const parentOpts = (_a = flow === null || flow === void 0 ? void 0 : flow.opts) === null || _a === void 0 ? void 0 : _a.parent;
const parentKey = (0, utils_1.getParentKey)(parentOpts);
const parentDependenciesKey = parentKey
? `${parentKey}:dependencies`
: undefined;
const jobsTree = this.addNode({
multi,
node: flow,
queuesOpts: opts === null || opts === void 0 ? void 0 : opts.queuesOptions,
parent: {
parentOpts,
parentDependenciesKey,
},
});
await multi.exec();
return jobsTree;
}
/**
* Get a flow.
*
* @param opts - an object with options for getting a JobNode.
*/
async getFlow(opts) {
if (this.closing) {
return;
}
const client = await this.connection.client;
const updatedOpts = Object.assign({
depth: 10,
maxChildren: 20,
}, opts);
const jobsTree = this.getNode(client, updatedOpts);
return jobsTree;
}
/**
* Adds multiple flows.
*
* A flow is a tree-like structure of jobs that depend on each other.
* Whenever the children of a given parent are completed, the parent
* will be processed, being able to access the children's result data.
*
* All Jobs can be in different queues, either children or parents,
* however this call would be atomic, either it fails and no jobs will
* be added to the queues, or it succeeds and all jobs will be added.
*
* @param flows - an array of objects with a tree-like structure where children jobs
* will be processed before their parents.
*/
async addBulk(flows) {
if (this.closing) {
return;
}
const client = await this.connection.client;
const multi = client.multi();
const jobsTrees = this.addNodes(multi, flows);
await multi.exec();
return jobsTrees;
}
/**
* Add a node (job) of a flow to the queue. This method will recursively
* add all its children as well. Note that a given job can potentially be
* a parent and a child job at the same time depending on where it is located
* in the tree hierarchy.
*
* @param multi - ioredis ChainableCommander
* @param node - the node representing a job to be added to some queue
* @param parent - parent data sent to children to create the "links" to their parent
* @returns
*/
addNode({ multi, node, parent, queuesOpts }) {
var _a;
const prefix = node.prefix || this.opts.prefix;
const queue = this.queueFromNode(node, new queue_keys_1.QueueKeys(prefix), prefix);
const queueOpts = queuesOpts && queuesOpts[node.queueName];
const jobsOpts = (0, lodash_1.get)(queueOpts, 'defaultJobOptions');
const jobId = ((_a = node.opts) === null || _a === void 0 ? void 0 : _a.jobId) || (0, uuid_1.v4)();
const job = new this.Job(queue, node.name, node.data, Object.assign(Object.assign(Object.assign({}, (jobsOpts ? jobsOpts : {})), node.opts), { parent: parent === null || parent === void 0 ? void 0 : parent.parentOpts }), jobId);
const parentKey = (0, utils_1.getParentKey)(parent === null || parent === void 0 ? void 0 : parent.parentOpts);
if (node.children && node.children.length > 0) {
// Create parent job, will be a job in status "waiting-children".
const parentId = jobId;
const queueKeysParent = new queue_keys_1.QueueKeys(node.prefix || this.opts.prefix);
const waitChildrenKey = queueKeysParent.toKey(node.queueName, 'waiting-children');
job.addJob(multi, {
parentDependenciesKey: parent === null || parent === void 0 ? void 0 : parent.parentDependenciesKey,
waitChildrenKey,
parentKey,
});
const parentDependenciesKey = `${queueKeysParent.toKey(node.queueName, parentId)}:dependencies`;
const children = this.addChildren({
multi,
nodes: node.children,
parent: {
parentOpts: {
id: parentId,
queue: queueKeysParent.getQueueQualifiedName(node.queueName),
},
parentDependenciesKey,
},
queuesOpts,
});
return { job, children };
}
else {
job.addJob(multi, {
parentDependenciesKey: parent === null || parent === void 0 ? void 0 : parent.parentDependenciesKey,
parentKey,
});
return { job };
}
}
/**
* Adds nodes (jobs) of multiple flows to the queue. This method will recursively
* add all its children as well. Note that a given job can potentially be
* a parent and a child job at the same time depending on where it is located
* in the tree hierarchy.
*
* @param multi - ioredis ChainableCommander
* @param nodes - the nodes representing jobs to be added to some queue
* @returns
*/
addNodes(multi, nodes) {
return nodes.map(node => {
var _a;
const parentOpts = (_a = node === null || node === void 0 ? void 0 : node.opts) === null || _a === void 0 ? void 0 : _a.parent;
const parentKey = (0, utils_1.getParentKey)(parentOpts);
const parentDependenciesKey = parentKey
? `${parentKey}:dependencies`
: undefined;
return this.addNode({
multi,
node,
parent: {
parentOpts,
parentDependenciesKey,
},
});
});
}
async getNode(client, node) {
const queue = this.queueFromNode(node, new queue_keys_1.QueueKeys(node.prefix), node.prefix);
const job = await this.Job.fromId(queue, node.id);
if (job) {
const { processed = {}, unprocessed = [] } = await job.getDependencies({
processed: {
count: node.maxChildren,
},
unprocessed: {
count: node.maxChildren,
},
});
const processedKeys = Object.keys(processed);
const childrenCount = processedKeys.length + unprocessed.length;
const newDepth = node.depth - 1;
if (childrenCount > 0 && newDepth) {
const children = await this.getChildren(client, [...processedKeys, ...unprocessed], newDepth, node.maxChildren);
return { job, children };
}
else {
return { job };
}
}
}
addChildren({ multi, nodes, parent, queuesOpts }) {
return nodes.map(node => this.addNode({ multi, node, parent, queuesOpts }));
}
getChildren(client, childrenKeys, depth, maxChildren) {
const getChild = (key) => {
const [prefix, queueName, id] = key.split(':');
return this.getNode(client, {
id,
queueName,
prefix,
depth,
maxChildren,
});
};
return Promise.all([...childrenKeys.map(getChild)]);
}
/**
* Helper factory method that creates a queue-like object
* required to create jobs in any queue.
*
* @param node -
* @param queueKeys -
* @returns
*/
queueFromNode(node, queueKeys, prefix) {
return {
client: this.connection.client,
name: node.queueName,
keys: queueKeys.getKeys(node.queueName),
toKey: (type) => queueKeys.toKey(node.queueName, type),
opts: { prefix },
qualifiedName: queueKeys.getQueueQualifiedName(node.queueName),
closing: this.closing,
waitUntilReady: async () => this.connection.client,
removeListener: this.removeListener.bind(this),
emit: this.emit.bind(this),
on: this.on.bind(this),
redisVersion: this.connection.redisVersion,
};
}
/**
*
* Closes the connection and returns a promise that resolves when the connection is closed.
*/
async close() {
if (!this.closing) {
this.closing = this.connection.close();
}
await this.closing;
}
/**
*
* Force disconnects a connection.
*/
disconnect() {
return this.connection.disconnect();
}
}
exports.FlowProducer = FlowProducer;
//# sourceMappingURL=flow-producer.js.map
File diff suppressed because one or more lines are too long
+23
View File
@@ -0,0 +1,23 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const tslib_1 = require("tslib");
tslib_1.__exportStar(require("./async-fifo-queue"), exports);
tslib_1.__exportStar(require("./backoffs"), exports);
tslib_1.__exportStar(require("./child-pool"), exports);
tslib_1.__exportStar(require("./child-processor"), exports);
tslib_1.__exportStar(require("./errors"), exports);
tslib_1.__exportStar(require("./flow-producer"), exports);
tslib_1.__exportStar(require("./job"), exports);
// export * from './main'; this file must not be exported
// export * from './main-worker'; this file must not be exported
tslib_1.__exportStar(require("./queue-base"), exports);
tslib_1.__exportStar(require("./queue-events"), exports);
tslib_1.__exportStar(require("./queue-getters"), exports);
tslib_1.__exportStar(require("./queue-keys"), exports);
tslib_1.__exportStar(require("./queue"), exports);
tslib_1.__exportStar(require("./redis-connection"), exports);
tslib_1.__exportStar(require("./repeat"), exports);
tslib_1.__exportStar(require("./sandbox"), exports);
tslib_1.__exportStar(require("./scripts"), exports);
tslib_1.__exportStar(require("./worker"), exports);
//# sourceMappingURL=index.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/classes/index.ts"],"names":[],"mappings":";;;AAAA,6DAAmC;AACnC,qDAA2B;AAC3B,uDAA6B;AAC7B,4DAAkC;AAClC,mDAAyB;AACzB,0DAAgC;AAChC,gDAAsB;AACtB,yDAAyD;AACzD,gEAAgE;AAChE,uDAA6B;AAC7B,yDAA+B;AAC/B,0DAAgC;AAChC,uDAA6B;AAC7B,kDAAwB;AACxB,6DAAmC;AACnC,mDAAyB;AACzB,oDAA0B;AAC1B,oDAA0B;AAC1B,mDAAyB"}
+791
View File
@@ -0,0 +1,791 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.Job = exports.PRIORITY_LIMIT = void 0;
const tslib_1 = require("tslib");
const lodash_1 = require("lodash");
const util_1 = require("util");
const utils_1 = require("../utils");
const backoffs_1 = require("./backoffs");
const scripts_1 = require("./scripts");
const unrecoverable_error_1 = require("./errors/unrecoverable-error");
const logger = (0, util_1.debuglog)('bull');
const optsDecodeMap = {
fpof: 'failParentOnFailure',
kl: 'keepLogs',
rdof: 'removeDependencyOnFailure',
};
const optsEncodeMap = (0, lodash_1.invert)(optsDecodeMap);
exports.PRIORITY_LIMIT = 2 ** 21;
/**
* Job
*
* This class represents a Job in the queue. Normally job are implicitly created when
* you add a job to the queue with methods such as Queue.addJob( ... )
*
* A Job instance is also passed to the Worker's process function.
*
* @class Job
*/
class Job {
constructor(queue,
/**
* The name of the Job
*/
name,
/**
* The payload for this job.
*/
data,
/**
* The options object for this job.
*/
opts = {}, id) {
this.queue = queue;
this.name = name;
this.data = data;
this.opts = opts;
this.id = id;
/**
* The progress a job has performed so far.
* @defaultValue 0
*/
this.progress = 0;
/**
* The value returned by the processor when processing this job.
* @defaultValue null
*/
this.returnvalue = null;
/**
* Stacktrace for the error (for failed jobs).
* @defaultValue null
*/
this.stacktrace = null;
/**
* Number of attempts after the job has failed.
* @defaultValue 0
*/
this.attemptsMade = 0;
const _a = this.opts, { repeatJobKey } = _a, restOpts = tslib_1.__rest(_a, ["repeatJobKey"]);
this.opts = Object.assign({
attempts: 0,
delay: 0,
}, restOpts);
this.delay = this.opts.delay;
this.repeatJobKey = repeatJobKey;
this.timestamp = opts.timestamp ? opts.timestamp : Date.now();
this.opts.backoff = backoffs_1.Backoffs.normalize(opts.backoff);
this.parentKey = (0, utils_1.getParentKey)(opts.parent);
this.parent = opts.parent
? { id: opts.parent.id, queueKey: opts.parent.queue }
: undefined;
this.toKey = queue.toKey.bind(queue);
this.scripts = new scripts_1.Scripts(queue);
this.queueQualifiedName = queue.qualifiedName;
}
/**
* Creates a new job and adds it to the queue.
*
* @param queue - the queue where to add the job.
* @param name - the name of the job.
* @param data - the payload of the job.
* @param opts - the options bag for this job.
* @returns
*/
static async create(queue, name, data, opts) {
const client = await queue.client;
const job = new this(queue, name, data, opts, opts && opts.jobId);
job.id = await job.addJob(client, {
parentKey: job.parentKey,
parentDependenciesKey: job.parentKey
? `${job.parentKey}:dependencies`
: '',
});
return job;
}
/**
* Creates a bulk of jobs and adds them atomically to the given queue.
*
* @param queue -the queue were to add the jobs.
* @param jobs - an array of jobs to be added to the queue.
* @returns
*/
static async createBulk(queue, jobs) {
const client = await queue.client;
const jobInstances = jobs.map(job => { var _a; return new this(queue, job.name, job.data, job.opts, (_a = job.opts) === null || _a === void 0 ? void 0 : _a.jobId); });
const multi = client.multi();
for (const job of jobInstances) {
job.addJob(multi, {
parentKey: job.parentKey,
parentDependenciesKey: job.parentKey
? `${job.parentKey}:dependencies`
: '',
});
}
const results = (await multi.exec());
for (let index = 0; index < results.length; ++index) {
const [err, id] = results[index];
if (err) {
throw err;
}
jobInstances[index].id = id;
}
return jobInstances;
}
/**
* Instantiates a Job from a JobJsonRaw object (coming from a deserialized JSON object)
*
* @param queue - the queue where the job belongs to.
* @param json - the plain object containing the job.
* @param jobId - an optional job id (overrides the id coming from the JSON object)
* @returns
*/
static fromJSON(queue, json, jobId) {
const data = JSON.parse(json.data || '{}');
const opts = Job.optsFromJSON(json.opts);
const job = new this(queue, json.name, data, opts, json.id || jobId);
job.progress = JSON.parse(json.progress || '0');
job.delay = parseInt(json.delay);
job.timestamp = parseInt(json.timestamp);
if (json.finishedOn) {
job.finishedOn = parseInt(json.finishedOn);
}
if (json.processedOn) {
job.processedOn = parseInt(json.processedOn);
}
if (json.rjk) {
job.repeatJobKey = json.rjk;
}
job.failedReason = json.failedReason;
job.attemptsMade = parseInt(json.attemptsMade || '0');
job.stacktrace = getTraces(json.stacktrace);
if (typeof json.returnvalue === 'string') {
job.returnvalue = getReturnValue(json.returnvalue);
}
if (json.parentKey) {
job.parentKey = json.parentKey;
}
if (json.parent) {
job.parent = JSON.parse(json.parent);
}
return job;
}
static optsFromJSON(rawOpts) {
const opts = JSON.parse(rawOpts || '{}');
const optionEntries = Object.entries(opts);
const options = {};
for (const item of optionEntries) {
const [attributeName, value] = item;
if (optsDecodeMap[attributeName]) {
options[optsDecodeMap[attributeName]] =
value;
}
else {
options[attributeName] = value;
}
}
return options;
}
/**
* Fetches a Job from the queue given the passed job id.
*
* @param queue - the queue where the job belongs to.
* @param jobId - the job id.
* @returns
*/
static async fromId(queue, jobId) {
// jobId can be undefined if moveJob returns undefined
if (jobId) {
const client = await queue.client;
const jobData = await client.hgetall(queue.toKey(jobId));
return (0, utils_1.isEmpty)(jobData)
? undefined
: this.fromJSON(queue, jobData, jobId);
}
}
/**
* addJobLog
*
* @param queue Queue instance
* @param jobId Job id
* @param logRow Log row
* @param keepLogs optional maximum number of logs to keep
*
* @returns The total number of log entries for this job so far.
*/
static async addJobLog(queue, jobId, logRow, keepLogs) {
const client = await queue.client;
const logsKey = queue.toKey(jobId) + ':logs';
const multi = client.multi();
multi.rpush(logsKey, logRow);
if (keepLogs) {
multi.ltrim(logsKey, -keepLogs, -1);
}
const result = (await multi.exec());
return keepLogs ? Math.min(keepLogs, result[0][1]) : result[0][1];
}
toJSON() {
const _a = this, { queue, scripts } = _a, withoutQueueAndScripts = tslib_1.__rest(_a, ["queue", "scripts"]);
return withoutQueueAndScripts;
}
/**
* Prepares a job to be serialized for storage in Redis.
* @returns
*/
asJSON() {
return {
id: this.id,
name: this.name,
data: JSON.stringify(typeof this.data === 'undefined' ? {} : this.data),
opts: this.optsAsJSON(this.opts),
parent: this.parent ? Object.assign({}, this.parent) : undefined,
parentKey: this.parentKey,
progress: this.progress,
attemptsMade: this.attemptsMade,
finishedOn: this.finishedOn,
processedOn: this.processedOn,
timestamp: this.timestamp,
failedReason: JSON.stringify(this.failedReason),
stacktrace: JSON.stringify(this.stacktrace),
repeatJobKey: this.repeatJobKey,
returnvalue: JSON.stringify(this.returnvalue),
};
}
optsAsJSON(opts = {}) {
const optionEntries = Object.entries(opts);
const options = {};
for (const item of optionEntries) {
const [attributeName, value] = item;
if (optsEncodeMap[attributeName]) {
options[optsEncodeMap[attributeName]] =
value;
}
else {
options[attributeName] = value;
}
}
return options;
}
/**
* Prepares a job to be passed to Sandbox.
* @returns
*/
asJSONSandbox() {
return Object.assign(Object.assign({}, this.asJSON()), { queueName: this.queueName, prefix: this.prefix });
}
/**
* Updates a job's data
*
* @param data - the data that will replace the current jobs data.
*/
updateData(data) {
this.data = data;
return this.scripts.updateData(this, data);
}
/**
* Updates a job's progress
*
* @param progress - number or object to be saved as progress.
*/
async updateProgress(progress) {
this.progress = progress;
await this.scripts.updateProgress(this.id, progress);
this.queue.emit('progress', this, progress);
}
/**
* Logs one row of log data.
*
* @param logRow - string with log data to be logged.
* @returns The total number of log entries for this job so far.
*/
async log(logRow) {
return Job.addJobLog(this.queue, this.id, logRow, this.opts.keepLogs);
}
/**
* Clears job's logs
*
* @param keepLogs - the amount of log entries to preserve
*/
async clearLogs(keepLogs) {
const client = await this.queue.client;
const logsKey = this.toKey(this.id) + ':logs';
if (keepLogs) {
await client.ltrim(logsKey, -keepLogs, -1);
}
else {
await client.del(logsKey);
}
}
/**
* Completely remove the job from the queue.
* Note, this call will throw an exception if the job
* is being processed when the call is performed.
*
* @param opts - Options to remove a job
*/
async remove({ removeChildren = true } = {}) {
await this.queue.waitUntilReady();
const queue = this.queue;
const job = this;
const removed = await this.scripts.remove(job.id, removeChildren);
if (removed) {
queue.emit('removed', job);
}
else {
throw new Error(`Job ${this.id} could not be removed because it is locked by another worker`);
}
}
/**
* Extend the lock for this job.
*
* @param token - unique token for the lock
* @param duration - lock duration in milliseconds
*/
extendLock(token, duration) {
return this.scripts.extendLock(this.id, token, duration);
}
/**
* Moves a job to the completed queue.
* Returned job to be used with Queue.prototype.nextJobFromJobData.
*
* @param returnValue - The jobs success message.
* @param token - Worker token used to acquire completed job.
* @param fetchNext - True when wanting to fetch the next job.
* @returns Returns the jobData of the next job in the waiting queue.
*/
async moveToCompleted(returnValue, token, fetchNext = true) {
await this.queue.waitUntilReady();
this.returnvalue = returnValue || void 0;
const stringifiedReturnValue = (0, utils_1.tryCatch)(JSON.stringify, JSON, [
returnValue,
]);
if (stringifiedReturnValue === utils_1.errorObject) {
throw utils_1.errorObject.value;
}
const args = this.scripts.moveToCompletedArgs(this, stringifiedReturnValue, this.opts.removeOnComplete, token, fetchNext);
const result = await this.scripts.moveToFinished(this.id, args);
this.finishedOn = args[14];
return result;
}
/**
* Moves a job to the failed queue.
*
* @param err - the jobs error message.
* @param token - token to check job is locked by current worker
* @param fetchNext - true when wanting to fetch the next job
* @returns void
*/
async moveToFailed(err, token, fetchNext = false) {
const client = await this.queue.client;
const message = err === null || err === void 0 ? void 0 : err.message;
const queue = this.queue;
this.failedReason = message;
let command;
const multi = client.multi();
this.saveStacktrace(multi, err);
//
// Check if an automatic retry should be performed
//
let moveToFailed = false;
let finishedOn, delay;
if (this.attemptsMade < this.opts.attempts &&
!this.discarded &&
!(err instanceof unrecoverable_error_1.UnrecoverableError || err.name == 'UnrecoverableError')) {
const opts = queue.opts;
// Check if backoff is needed
delay = await backoffs_1.Backoffs.calculate(this.opts.backoff, this.attemptsMade, err, this, opts.settings && opts.settings.backoffStrategy);
if (delay === -1) {
moveToFailed = true;
}
else if (delay) {
const args = this.scripts.moveToDelayedArgs(this.id, Date.now() + delay, token, delay);
this.scripts.execCommand(multi, 'moveToDelayed', args);
command = 'delayed';
}
else {
// Retry immediately
this.scripts.execCommand(multi, 'retryJob', this.scripts.retryJobArgs(this.id, this.opts.lifo, token));
command = 'retryJob';
}
}
else {
// If not, move to failed
moveToFailed = true;
}
if (moveToFailed) {
const args = this.scripts.moveToFailedArgs(this, message, this.opts.removeOnFail, token, fetchNext);
this.scripts.execCommand(multi, 'moveToFinished', args);
finishedOn = args[14];
command = 'failed';
}
const results = await multi.exec();
const anyError = results.find(result => result[0]);
if (anyError) {
throw new Error(`Error "moveToFailed" with command ${command}: ${anyError}`);
}
const code = results[results.length - 1][1];
if (code < 0) {
throw this.scripts.finishedErrors(code, this.id, command, 'active');
}
if (finishedOn && typeof finishedOn === 'number') {
this.finishedOn = finishedOn;
}
if (delay && typeof delay === 'number') {
this.delay = delay;
}
}
/**
* @returns true if the job has completed.
*/
isCompleted() {
return this.isInZSet('completed');
}
/**
* @returns true if the job has failed.
*/
isFailed() {
return this.isInZSet('failed');
}
/**
* @returns true if the job is delayed.
*/
isDelayed() {
return this.isInZSet('delayed');
}
/**
* @returns true if the job is waiting for children.
*/
isWaitingChildren() {
return this.isInZSet('waiting-children');
}
/**
* @returns true of the job is active.
*/
isActive() {
return this.isInList('active');
}
/**
* @returns true if the job is waiting.
*/
async isWaiting() {
return (await this.isInList('wait')) || (await this.isInList('paused'));
}
/**
* @returns the queue name this job belongs to.
*/
get queueName() {
return this.queue.name;
}
/**
* @returns the prefix that is used.
*/
get prefix() {
return this.queue.opts.prefix;
}
/**
* Get current state.
*
* @returns Returns one of these values:
* 'completed', 'failed', 'delayed', 'active', 'waiting', 'waiting-children', 'unknown'.
*/
getState() {
return this.scripts.getState(this.id);
}
/**
* Change delay of a delayed job.
*
* @param delay - milliseconds to be added to current time.
* @returns void
*/
async changeDelay(delay) {
await this.scripts.changeDelay(this.id, delay);
this.delay = delay;
}
/**
* Change job priority.
*
* @returns void
*/
async changePriority(opts) {
await this.scripts.changePriority(this.id, opts.priority, opts.lifo);
}
/**
* Get this jobs children result values if any.
*
* @returns Object mapping children job keys with their values.
*/
async getChildrenValues() {
const client = await this.queue.client;
const result = (await client.hgetall(this.toKey(`${this.id}:processed`)));
if (result) {
return (0, utils_1.parseObjectValues)(result);
}
}
/**
* Get children job keys if this job is a parent and has children.
* @remarks
* Count options before Redis v7.2 works as expected with any quantity of entries
* on processed/unprocessed dependencies, since v7.2 you must consider that count
* won't have any effect until processed/unprocessed dependencies have a length
* greater than 127
* @see https://redis.io/docs/management/optimization/memory-optimization/#redis--72
* @returns dependencies separated by processed and unprocessed.
*/
async getDependencies(opts = {}) {
const client = await this.queue.client;
const multi = client.multi();
if (!opts.processed && !opts.unprocessed) {
multi.hgetall(this.toKey(`${this.id}:processed`));
multi.smembers(this.toKey(`${this.id}:dependencies`));
const [[err1, processed], [err2, unprocessed]] = (await multi.exec());
const transformedProcessed = (0, utils_1.parseObjectValues)(processed);
return { processed: transformedProcessed, unprocessed };
}
else {
const defaultOpts = {
cursor: 0,
count: 20,
};
if (opts.processed) {
const processedOpts = Object.assign(Object.assign({}, defaultOpts), opts.processed);
multi.hscan(this.toKey(`${this.id}:processed`), processedOpts.cursor, 'COUNT', processedOpts.count);
}
if (opts.unprocessed) {
const unprocessedOpts = Object.assign(Object.assign({}, defaultOpts), opts.unprocessed);
multi.sscan(this.toKey(`${this.id}:dependencies`), unprocessedOpts.cursor, 'COUNT', unprocessedOpts.count);
}
const [result1, result2] = (await multi.exec());
const [processedCursor, processed = []] = opts.processed
? result1[1]
: [];
const [unprocessedCursor, unprocessed = []] = opts.unprocessed
? opts.processed
? result2[1]
: result1[1]
: [];
const transformedProcessed = {};
for (let index = 0; index < processed.length; ++index) {
if (index % 2) {
transformedProcessed[processed[index - 1]] = JSON.parse(processed[index]);
}
}
return Object.assign(Object.assign({}, (processedCursor
? {
processed: transformedProcessed,
nextProcessedCursor: Number(processedCursor),
}
: {})), (unprocessedCursor
? { unprocessed, nextUnprocessedCursor: Number(unprocessedCursor) }
: {}));
}
}
/**
* Get children job counts if this job is a parent and has children.
*
* @returns dependencies count separated by processed and unprocessed.
*/
async getDependenciesCount(opts = {}) {
const client = await this.queue.client;
const multi = client.multi();
const updatedOpts = !opts.processed && !opts.unprocessed
? { processed: true, unprocessed: true }
: opts;
if (updatedOpts.processed) {
multi.hlen(this.toKey(`${this.id}:processed`));
}
if (updatedOpts.unprocessed) {
multi.scard(this.toKey(`${this.id}:dependencies`));
}
const [[err1, result1] = [], [err2, result2] = []] = (await multi.exec());
const processed = updatedOpts.processed ? result1 : undefined;
const unprocessed = updatedOpts.unprocessed
? updatedOpts.processed
? result2
: result1
: undefined;
return Object.assign(Object.assign({}, (updatedOpts.processed
? {
processed,
}
: {})), (updatedOpts.unprocessed ? { unprocessed } : {}));
}
/**
* Returns a promise the resolves when the job has completed (containing the return value of the job),
* or rejects when the job has failed (containing the failedReason).
*
* @param queueEvents - Instance of QueueEvents.
* @param ttl - Time in milliseconds to wait for job to finish before timing out.
*/
async waitUntilFinished(queueEvents, ttl) {
await this.queue.waitUntilReady();
const jobId = this.id;
return new Promise(async (resolve, reject) => {
let timeout;
if (ttl) {
timeout = setTimeout(() => onFailed(
/* eslint-disable max-len */
`Job wait ${this.name} timed out before finishing, no finish notification arrived after ${ttl}ms (id=${jobId})`), ttl);
}
function onCompleted(args) {
removeListeners();
resolve(args.returnvalue);
}
function onFailed(args) {
removeListeners();
reject(new Error(args.failedReason || args));
}
const completedEvent = `completed:${jobId}`;
const failedEvent = `failed:${jobId}`;
queueEvents.on(completedEvent, onCompleted);
queueEvents.on(failedEvent, onFailed);
this.queue.on('closing', onFailed);
const removeListeners = () => {
clearInterval(timeout);
queueEvents.removeListener(completedEvent, onCompleted);
queueEvents.removeListener(failedEvent, onFailed);
this.queue.removeListener('closing', onFailed);
};
// Poll once right now to see if the job has already finished. The job may have been completed before we were able
// to register the event handlers on the QueueEvents, so we check here to make sure we're not waiting for an event
// that has already happened. We block checking the job until the queue events object is actually listening to
// Redis so there's no chance that it will miss events.
await queueEvents.waitUntilReady();
const [status, result] = (await this.scripts.isFinished(jobId, true));
const finished = status != 0;
if (finished) {
if (status == -1 || status == 2) {
onFailed({ failedReason: result });
}
else {
onCompleted({ returnvalue: getReturnValue(result) });
}
}
});
}
/**
* Moves the job to the delay set.
*
* @param timestamp - timestamp where the job should be moved back to "wait"
* @param token - token to check job is locked by current worker
* @returns
*/
moveToDelayed(timestamp, token) {
const delay = timestamp - Date.now();
return this.scripts.moveToDelayed(this.id, timestamp, delay > 0 ? delay : 0, token);
}
/**
* Moves the job to the waiting-children set.
*
* @param token - Token to check job is locked by current worker
* @param opts - The options bag for moving a job to waiting-children.
* @returns true if the job was moved
*/
moveToWaitingChildren(token, opts = {}) {
return this.scripts.moveToWaitingChildren(this.id, token, opts);
}
/**
* Promotes a delayed job so that it starts to be processed as soon as possible.
*/
async promote() {
const jobId = this.id;
await this.scripts.promote(jobId);
this.delay = 0;
}
/**
* Attempts to retry the job. Only a job that has failed or completed can be retried.
*
* @param state - completed / failed
* @returns If resolved and return code is 1, then the queue emits a waiting event
* otherwise the operation was not a success and throw the corresponding error. If the promise
* rejects, it indicates that the script failed to execute
*/
retry(state = 'failed') {
this.failedReason = null;
this.finishedOn = null;
this.processedOn = null;
this.returnvalue = null;
return this.scripts.reprocessJob(this, state);
}
/**
* Marks a job to not be retried if it fails (even if attempts has been configured)
*/
discard() {
this.discarded = true;
}
async isInZSet(set) {
const client = await this.queue.client;
const score = await client.zscore(this.queue.toKey(set), this.id);
return score !== null;
}
async isInList(list) {
return this.scripts.isJobInList(this.queue.toKey(list), this.id);
}
/**
* Adds the job to Redis.
*
* @param client -
* @param parentOpts -
* @returns
*/
addJob(client, parentOpts) {
const jobData = this.asJSON();
this.validateOptions(jobData);
return this.scripts.addJob(client, jobData, jobData.opts, this.id, parentOpts);
}
validateOptions(jobData) {
var _a;
const exceedLimit = this.opts.sizeLimit &&
(0, utils_1.lengthInUtf8Bytes)(jobData.data) > this.opts.sizeLimit;
if (exceedLimit) {
throw new Error(`The size of job ${this.name} exceeds the limit ${this.opts.sizeLimit} bytes`);
}
if (this.opts.delay && this.opts.repeat && !((_a = this.opts.repeat) === null || _a === void 0 ? void 0 : _a.count)) {
throw new Error(`Delay and repeat options could not be used together`);
}
if (this.opts.removeDependencyOnFailure && this.opts.failParentOnFailure) {
throw new Error(`RemoveDependencyOnFailure and failParentOnFailure options can not be used together`);
}
if (`${parseInt(this.id, 10)}` === this.id) {
//TODO: throw an error in next breaking change
console.warn('Custom Ids should not be integers: https://github.com/taskforcesh/bullmq/pull/1569');
}
if (this.opts.priority) {
if (Math.trunc(this.opts.priority) !== this.opts.priority) {
throw new Error(`Priority should not be float`);
}
if (this.opts.priority > exports.PRIORITY_LIMIT) {
throw new Error(`Priority should be between 0 and ${exports.PRIORITY_LIMIT}`);
}
}
}
saveStacktrace(multi, err) {
this.stacktrace = this.stacktrace || [];
if (err === null || err === void 0 ? void 0 : err.stack) {
this.stacktrace.push(err.stack);
if (this.opts.stackTraceLimit) {
this.stacktrace = this.stacktrace.slice(0, this.opts.stackTraceLimit);
}
}
const args = this.scripts.saveStacktraceArgs(this.id, JSON.stringify(this.stacktrace), err === null || err === void 0 ? void 0 : err.message);
this.scripts.execCommand(multi, 'saveStacktrace', args);
}
}
exports.Job = Job;
function getTraces(stacktrace) {
const traces = (0, utils_1.tryCatch)(JSON.parse, JSON, [stacktrace]);
if (traces === utils_1.errorObject || !(traces instanceof Array)) {
return [];
}
else {
return traces;
}
}
function getReturnValue(_value) {
const value = (0, utils_1.tryCatch)(JSON.parse, JSON, [_value]);
if (value !== utils_1.errorObject) {
return value;
}
else {
logger('corrupted returnvalue: ' + _value, value);
}
}
//# sourceMappingURL=job.js.map
File diff suppressed because one or more lines are too long
+45
View File
@@ -0,0 +1,45 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
/**
* Wrapper for sandboxing.
*
*/
const lodash_1 = require("lodash");
const child_processor_1 = require("./child-processor");
const enums_1 = require("../enums");
const utils_1 = require("../utils");
exports.default = (send, receiver) => {
const childProcessor = new child_processor_1.ChildProcessor(send);
receiver === null || receiver === void 0 ? void 0 : receiver.on('message', async (msg) => {
try {
switch (msg.cmd) {
case enums_1.ChildCommand.Init:
await childProcessor.init(msg.value);
break;
case enums_1.ChildCommand.Start:
await childProcessor.start(msg.job, msg === null || msg === void 0 ? void 0 : msg.token);
break;
case enums_1.ChildCommand.Stop:
break;
}
}
catch (err) {
console.error('Error handling child message');
}
});
process.on('SIGTERM', () => childProcessor.waitForCurrentJobAndExit());
process.on('SIGINT', () => childProcessor.waitForCurrentJobAndExit());
process.on('uncaughtException', async (err) => {
if (!err.message) {
err = new Error((0, lodash_1.toString)(err));
}
await send({
cmd: enums_1.ParentCommand.Failed,
value: (0, utils_1.errorToJSON)(err),
});
// An uncaughException leaves this process in a potentially undetermined state so
// we must exit
process.exit();
});
};
//# sourceMappingURL=main-base.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"main-base.js","sourceRoot":"","sources":["../../../src/classes/main-base.ts"],"names":[],"mappings":";;AAAA;;;GAGG;AACH,mCAAkC;AAClC,uDAAmD;AACnD,oCAAuD;AACvD,oCAAuC;AAEvC,kBAAe,CACb,IAAiC,EACjC,QAAkE,EAClE,EAAE;IACF,MAAM,cAAc,GAAG,IAAI,gCAAc,CAAC,IAAI,CAAC,CAAC;IAEhD,QAAQ,aAAR,QAAQ,uBAAR,QAAQ,CAAE,EAAE,CAAC,SAAS,EAAE,KAAK,EAAC,GAAG,EAAC,EAAE;QAClC,IAAI;YACF,QAAQ,GAAG,CAAC,GAAmB,EAAE;gBAC/B,KAAK,oBAAY,CAAC,IAAI;oBACpB,MAAM,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;oBACrC,MAAM;gBACR,KAAK,oBAAY,CAAC,KAAK;oBACrB,MAAM,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,KAAK,CAAC,CAAC;oBAChD,MAAM;gBACR,KAAK,oBAAY,CAAC,IAAI;oBACpB,MAAM;aACT;SACF;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;SAC/C;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,cAAc,CAAC,wBAAwB,EAAE,CAAC,CAAC;IACvE,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,cAAc,CAAC,wBAAwB,EAAE,CAAC,CAAC;IAEtE,OAAO,CAAC,EAAE,CAAC,mBAAmB,EAAE,KAAK,EAAE,GAAU,EAAE,EAAE;QACnD,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE;YAChB,GAAG,GAAG,IAAI,KAAK,CAAC,IAAA,iBAAQ,EAAC,GAAG,CAAC,CAAC,CAAC;SAChC;QACD,MAAM,IAAI,CAAC;YACT,GAAG,EAAE,qBAAa,CAAC,MAAM;YACzB,KAAK,EAAE,IAAA,mBAAW,EAAC,GAAG,CAAC;SACxB,CAAC,CAAC;QAEH,iFAAiF;QACjF,eAAe;QACf,OAAO,CAAC,IAAI,EAAE,CAAC;IACjB,CAAC,CAAC,CAAC;AACL,CAAC,CAAC"}
+1
View File
@@ -0,0 +1 @@
export {};
+10
View File
@@ -0,0 +1,10 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
/**
* Worker Thread wrapper for sandboxing
*
*/
const worker_threads_1 = require("worker_threads");
const main_base_1 = require("./main-base");
(0, main_base_1.default)(async (msg) => worker_threads_1.parentPort.postMessage(msg), worker_threads_1.parentPort);
//# sourceMappingURL=main-worker.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"main-worker.js","sourceRoot":"","sources":["../../../src/classes/main-worker.ts"],"names":[],"mappings":";;AAAA;;;GAGG;AACH,mDAA4C;AAC5C,2CAAmC;AAEnC,IAAA,mBAAQ,EAAC,KAAK,EAAE,GAAQ,EAAE,EAAE,CAAC,2BAAU,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,2BAAU,CAAC,CAAC"}
+1
View File
@@ -0,0 +1 @@
export {};
+10
View File
@@ -0,0 +1,10 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
/**
* Child process wrapper for sandboxing.
*
*/
const utils_1 = require("../utils");
const main_base_1 = require("./main-base");
(0, main_base_1.default)((msg) => (0, utils_1.childSend)(process, msg), process);
//# sourceMappingURL=main.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"main.js","sourceRoot":"","sources":["../../../src/classes/main.ts"],"names":[],"mappings":";;AAAA;;;GAGG;AACH,oCAAqC;AACrC,2CAAmC;AAEnC,IAAA,mBAAQ,EAAC,CAAC,GAAQ,EAAE,EAAE,CAAC,IAAA,iBAAS,EAAC,OAAO,EAAE,GAAG,CAAC,EAAE,OAAO,CAAC,CAAC"}
+141
View File
@@ -0,0 +1,141 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.QueueBase = void 0;
const events_1 = require("events");
const utils_1 = require("../utils");
const redis_connection_1 = require("./redis-connection");
const job_1 = require("./job");
const queue_keys_1 = require("./queue-keys");
const scripts_1 = require("./scripts");
/**
* @class QueueBase
* @extends EventEmitter
*
* @description Base class for all classes that need to interact with queues.
* This class is normally not used directly, but extended by the other classes.
*
*/
class QueueBase extends events_1.EventEmitter {
/**
*
* @param name - The name of the queue.
* @param opts - Options for the queue.
* @param Connection - An optional "Connection" class used to instantiate a Connection. This is useful for
* testing with mockups and/or extending the Connection class and passing an alternate implementation.
*/
constructor(name, opts = {}, Connection = redis_connection_1.RedisConnection) {
super();
this.name = name;
this.opts = opts;
this.closed = false;
this.opts = Object.assign({ prefix: 'bull' }, opts);
if (!name) {
throw new Error('Queue name must be provided');
}
if (!opts.connection) {
console.warn([
'BullMQ: DEPRECATION WARNING! Optional instantiation of Queue, Worker, QueueEvents and FlowProducer',
'without providing explicitly a connection or connection options is deprecated. This behaviour will',
'be removed in the next major release',
].join(' '));
}
this.connection = new Connection(opts.connection, (0, utils_1.isRedisInstance)(opts === null || opts === void 0 ? void 0 : opts.connection), opts.blockingConnection, opts.skipVersionCheck);
this.connection.on('error', (error) => this.emit('error', error));
this.connection.on('close', () => {
if (!this.closing) {
this.emit('ioredis:close');
}
});
const queueKeys = new queue_keys_1.QueueKeys(opts.prefix);
this.qualifiedName = queueKeys.getQueueQualifiedName(name);
this.keys = queueKeys.getKeys(name);
this.toKey = (type) => queueKeys.toKey(name, type);
this.scripts = new scripts_1.Scripts(this);
}
/**
* Returns a promise that resolves to a redis client. Normally used only by subclasses.
*/
get client() {
return this.connection.client;
}
/**
* Returns the version of the Redis instance the client is connected to,
*/
get redisVersion() {
return this.connection.redisVersion;
}
/**
* Helper to easily extend Job class calls.
*/
get Job() {
return job_1.Job;
}
/**
* Emits an event. Normally used by subclasses to emit events.
*
* @param event - The emitted event.
* @param args -
* @returns
*/
emit(event, ...args) {
try {
return super.emit(event, ...args);
}
catch (err) {
try {
return super.emit('error', err);
}
catch (err) {
// We give up if the error event also throws an exception.
console.error(err);
return false;
}
}
}
waitUntilReady() {
return this.client;
}
base64Name() {
return Buffer.from(this.name).toString('base64');
}
clientName(suffix = '') {
const queueNameBase64 = this.base64Name();
return `${this.opts.prefix}:${queueNameBase64}${suffix}`;
}
/**
*
* Closes the connection and returns a promise that resolves when the connection is closed.
*/
async close() {
if (!this.closing) {
this.closing = this.connection.close();
}
await this.closing;
this.closed = true;
}
/**
*
* Force disconnects a connection.
*/
disconnect() {
return this.connection.disconnect();
}
async checkConnectionError(fn, delayInMs = utils_1.DELAY_TIME_5) {
try {
return await fn();
}
catch (error) {
if ((0, utils_1.isNotConnectionError)(error)) {
this.emit('error', error);
}
if (!this.closing && delayInMs) {
await (0, utils_1.delay)(delayInMs);
}
else {
return;
}
}
}
}
exports.QueueBase = QueueBase;
//# sourceMappingURL=queue-base.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"queue-base.js","sourceRoot":"","sources":["../../../src/classes/queue-base.ts"],"names":[],"mappings":";;;AAAA,mCAAsC;AAGtC,oCAKkB;AAClB,yDAAqD;AACrD,+BAA4B;AAC5B,6CAAkD;AAClD,uCAAoC;AAEpC;;;;;;;GAOG;AACH,MAAa,SAAU,SAAQ,qBAAY;IAUzC;;;;;;OAMG;IACH,YACkB,IAAY,EACrB,OAAyB,EAAE,EAClC,aAAqC,kCAAe;QAEpD,KAAK,EAAE,CAAC;QAJQ,SAAI,GAAJ,IAAI,CAAQ;QACrB,SAAI,GAAJ,IAAI,CAAuB;QAd1B,WAAM,GAAY,KAAK,CAAC;QAmBhC,IAAI,CAAC,IAAI,mBACP,MAAM,EAAE,MAAM,IACX,IAAI,CACR,CAAC;QAEF,IAAI,CAAC,IAAI,EAAE;YACT,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;SAChD;QAED,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE;YACpB,OAAO,CAAC,IAAI,CACV;gBACE,oGAAoG;gBACpG,oGAAoG;gBACpG,sCAAsC;aACvC,CAAC,IAAI,CAAC,GAAG,CAAC,CACZ,CAAC;SACH;QAED,IAAI,CAAC,UAAU,GAAG,IAAI,UAAU,CAC9B,IAAI,CAAC,UAAU,EACf,IAAA,uBAAe,EAAC,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,UAAU,CAAC,EACjC,IAAI,CAAC,kBAAkB,EACvB,IAAI,CAAC,gBAAgB,CACtB,CAAC;QAEF,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAY,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;QACzE,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YAC/B,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;gBACjB,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;aAC5B;QACH,CAAC,CAAC,CAAC;QAEH,MAAM,SAAS,GAAG,IAAI,sBAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC7C,IAAI,CAAC,aAAa,GAAG,SAAS,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAC3D,IAAI,CAAC,IAAI,GAAG,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,CAAC,KAAK,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC3D,IAAI,CAAC,OAAO,GAAG,IAAI,iBAAO,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC;IAED;;OAEG;IACH,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,IAAc,GAAG;QACf,OAAO,SAAG,CAAC;IACb,CAAC;IAED;;;;;;OAMG;IACH,IAAI,CAAC,KAAsB,EAAE,GAAG,IAAW;QACzC,IAAI;YACF,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,CAAC;SACnC;QAAC,OAAO,GAAG,EAAE;YACZ,IAAI;gBACF,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;aACjC;YAAC,OAAO,GAAG,EAAE;gBACZ,0DAA0D;gBAC1D,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACnB,OAAO,KAAK,CAAC;aACd;SACF;IACH,CAAC;IAED,cAAc;QACZ,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAES,UAAU;QAClB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACnD,CAAC;IAES,UAAU,CAAC,MAAM,GAAG,EAAE;QAC9B,MAAM,eAAe,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;QAC1C,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,eAAe,GAAG,MAAM,EAAE,CAAC;IAC3D,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YACjB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;SACxC;QACD,MAAM,IAAI,CAAC,OAAO,CAAC;QACnB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;IACrB,CAAC;IAED;;;OAGG;IACH,UAAU;QACR,OAAO,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,CAAC;IACtC,CAAC;IAES,KAAK,CAAC,oBAAoB,CAClC,EAAoB,EACpB,SAAS,GAAG,oBAAY;QAExB,IAAI;YACF,OAAO,MAAM,EAAE,EAAE,CAAC;SACnB;QAAC,OAAO,KAAK,EAAE;YACd,IAAI,IAAA,4BAAoB,EAAC,KAAc,CAAC,EAAE;gBACxC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAS,KAAK,CAAC,CAAC;aAClC;YAED,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,SAAS,EAAE;gBAC9B,MAAM,IAAA,aAAK,EAAC,SAAS,CAAC,CAAC;aACxB;iBAAM;gBACL,OAAO;aACR;SACF;IACH,CAAC;CACF;AA7JD,8BA6JC"}
+120
View File
@@ -0,0 +1,120 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.QueueEvents = void 0;
const tslib_1 = require("tslib");
const utils_1 = require("../utils");
const queue_base_1 = require("./queue-base");
/**
* The QueueEvents class is used for listening to the global events
* emitted by a given queue.
*
* This class requires a dedicated redis connection.
*
*/
class QueueEvents extends queue_base_1.QueueBase {
constructor(name, _a = {}, Connection) {
var { connection, autorun = true } = _a, opts = tslib_1.__rest(_a, ["connection", "autorun"]);
super(name, Object.assign(Object.assign({}, opts), { connection: (0, utils_1.isRedisInstance)(connection)
? connection.duplicate()
: connection, blockingConnection: true }), Connection);
this.running = false;
this.opts = Object.assign({
blockingTimeout: 10000,
}, this.opts);
if (autorun) {
this.run().catch(error => this.emit('error', error));
}
}
emit(event, ...args) {
return super.emit(event, ...args);
}
off(eventName, listener) {
super.off(eventName, listener);
return this;
}
on(event, listener) {
super.on(event, listener);
return this;
}
once(event, listener) {
super.once(event, listener);
return this;
}
/**
* Manually starts running the event consumming loop. This shall be used if you do not
* use the default "autorun" option on the constructor.
*/
async run() {
if (!this.running) {
try {
this.running = true;
const client = await this.client;
try {
await client.client('SETNAME', this.clientName(utils_1.QUEUE_EVENT_SUFFIX));
}
catch (err) {
if (!utils_1.clientCommandMessageReg.test(err.message)) {
throw err;
}
}
await this.consumeEvents(client);
}
catch (error) {
this.running = false;
throw error;
}
}
else {
throw new Error('Queue Events is already running.');
}
}
async consumeEvents(client) {
const opts = this.opts;
const key = this.keys.events;
let id = opts.lastEventId || '$';
while (!this.closing) {
// Cast to actual return type, see: https://github.com/DefinitelyTyped/DefinitelyTyped/issues/44301
const data = await this.checkConnectionError(() => client.xread('BLOCK', opts.blockingTimeout, 'STREAMS', key, id));
if (data) {
const stream = data[0];
const events = stream[1];
for (let i = 0; i < events.length; i++) {
id = events[i][0];
const args = (0, utils_1.array2obj)(events[i][1]);
//
// TODO: we may need to have a separate xtream for progress data
// to avoid this hack.
switch (args.event) {
case 'progress':
args.data = JSON.parse(args.data);
break;
case 'completed':
args.returnvalue = JSON.parse(args.returnvalue);
break;
}
const { event } = args, restArgs = tslib_1.__rest(args, ["event"]);
if (event === 'drained') {
this.emit(event, id);
}
else {
this.emit(event, restArgs, id);
this.emit(`${event}:${restArgs.jobId}`, restArgs, id);
}
}
}
}
}
/**
* Stops consuming events and close the underlying Redis connection if necessary.
*
* @returns
*/
close() {
if (!this.closing) {
this.closing = this.disconnect();
}
return this.closing;
}
}
exports.QueueEvents = QueueEvents;
//# sourceMappingURL=queue-events.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"queue-events.js","sourceRoot":"","sources":["../../../src/classes/queue-events.ts"],"names":[],"mappings":";;;;AAMA,oCAKkB;AAClB,6CAAyC;AAgJzC;;;;;;GAMG;AACH,MAAa,WAAY,SAAQ,sBAAS;IAGxC,YACE,IAAY,EACZ,KAA8D,EAAE,EAChE,UAAmC;YADnC,EAAE,UAAU,EAAE,OAAO,GAAG,IAAI,OAAoC,EAA/B,IAAI,sBAArC,yBAAuC,CAAF;QAGrC,KAAK,CACH,IAAI,kCAEC,IAAI,KACP,UAAU,EAAE,IAAA,uBAAe,EAAC,UAAU,CAAC;gBACrC,CAAC,CAAe,UAAW,CAAC,SAAS,EAAE;gBACvC,CAAC,CAAC,UAAU,EACd,kBAAkB,EAAE,IAAI,KAE1B,UAAU,CACX,CAAC;QAjBI,YAAO,GAAG,KAAK,CAAC;QAmBtB,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,MAAM,CACvB;YACE,eAAe,EAAE,KAAK;SACvB,EACD,IAAI,CAAC,IAAI,CACV,CAAC;QAEF,IAAI,OAAO,EAAE;YACX,IAAI,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;SACtD;IACH,CAAC;IAED,IAAI,CACF,KAAQ,EACR,GAAG,IAAwC;QAE3C,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,CAAC;IACpC,CAAC;IAED,GAAG,CACD,SAAY,EACZ,QAAgC;QAEhC,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAC/B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,EAAE,CACA,KAAQ,EACR,QAAgC;QAEhC,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAC1B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CACF,KAAQ,EACR,QAAgC;QAEhC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAC5B,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,GAAG;QACP,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YACjB,IAAI;gBACF,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;gBACpB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC;gBAEjC,IAAI;oBACF,MAAM,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,UAAU,CAAC,0BAAkB,CAAC,CAAC,CAAC;iBACrE;gBAAC,OAAO,GAAG,EAAE;oBACZ,IAAI,CAAC,+BAAuB,CAAC,IAAI,CAAS,GAAI,CAAC,OAAO,CAAC,EAAE;wBACvD,MAAM,GAAG,CAAC;qBACX;iBACF;gBAED,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;aAClC;YAAC,OAAO,KAAK,EAAE;gBACd,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;gBACrB,MAAM,KAAK,CAAC;aACb;SACF;aAAM;YACL,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;SACrD;IACH,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,MAAmB;QAC7C,MAAM,IAAI,GAAuB,IAAI,CAAC,IAAI,CAAC;QAE3C,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;QAC7B,IAAI,EAAE,GAAG,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;QAEjC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE;YACpB,mGAAmG;YACnG,MAAM,IAAI,GAAkB,MAAM,IAAI,CAAC,oBAAoB,CAAC,GAAG,EAAE,CAC/D,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,eAAgB,EAAE,SAAS,EAAE,GAAG,EAAE,EAAE,CAAC,CACjE,CAAC;YACF,IAAI,IAAI,EAAE;gBACR,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;gBACvB,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;oBACtC,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBAClB,MAAM,IAAI,GAAG,IAAA,iBAAS,EAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBAErC,EAAE;oBACF,gEAAgE;oBAChE,sBAAsB;oBACtB,QAAQ,IAAI,CAAC,KAAK,EAAE;wBAClB,KAAK,UAAU;4BACb,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;4BAClC,MAAM;wBACR,KAAK,WAAW;4BACd,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;4BAChD,MAAM;qBACT;oBAED,MAAM,EAAE,KAAK,KAAkB,IAAI,EAAjB,QAAQ,kBAAK,IAAI,EAA7B,SAAsB,CAAO,CAAC;oBAEpC,IAAI,KAAK,KAAK,SAAS,EAAE;wBACvB,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;qBACtB;yBAAM;wBACL,IAAI,CAAC,IAAI,CAAC,KAAY,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC;wBACtC,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,IAAI,QAAQ,CAAC,KAAK,EAAS,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC;qBAC9D;iBACF;aACF;SACF;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YACjB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;SAClC;QACD,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;CACF;AAlJD,kCAkJC"}
+405
View File
@@ -0,0 +1,405 @@
/*eslint-env node */
'use strict';
Object.defineProperty(exports, "__esModule", { value: true });
exports.QueueGetters = void 0;
const queue_base_1 = require("./queue-base");
const job_1 = require("./job");
const utils_1 = require("../utils");
/**
*
* @class QueueGetters
* @extends QueueBase
*
* @description Provides different getters for different aspects of a queue.
*/
class QueueGetters extends queue_base_1.QueueBase {
getJob(jobId) {
return this.Job.fromId(this, jobId);
}
commandByType(types, count, callback) {
return types.map((type) => {
type = type === 'waiting' ? 'wait' : type; // alias
const key = this.toKey(type);
switch (type) {
case 'completed':
case 'failed':
case 'delayed':
case 'prioritized':
case 'repeat':
case 'waiting-children':
return callback(key, count ? 'zcard' : 'zrange');
case 'active':
case 'wait':
case 'paused':
return callback(key, count ? 'llen' : 'lrange');
}
});
}
/**
* Helper to easily extend Job class calls.
*/
get Job() {
return job_1.Job;
}
sanitizeJobTypes(types) {
const currentTypes = typeof types === 'string' ? [types] : types;
if (Array.isArray(currentTypes) && currentTypes.length > 0) {
const sanitizedTypes = [...currentTypes];
if (sanitizedTypes.indexOf('waiting') !== -1) {
sanitizedTypes.push('paused');
}
return [...new Set(sanitizedTypes)];
}
return [
'active',
'completed',
'delayed',
'failed',
'paused',
'prioritized',
'waiting',
'waiting-children',
];
}
/**
Returns the number of jobs waiting to be processed. This includes jobs that are
"waiting" or "delayed" or "prioritized" or "waiting-children".
*/
async count() {
const count = await this.getJobCountByTypes('waiting', 'paused', 'delayed', 'prioritized', 'waiting-children');
return count;
}
/**
* Returns the time to live for a rate limited key in milliseconds.
* @returns -2 if the key does not exist.
* -1 if the key exists but has no associated expire.
* @see {@link https://redis.io/commands/pttl/}
*/
async getRateLimitTtl() {
const client = await this.client;
return client.pttl(this.keys.limiter);
}
/**
* Job counts by type
*
* Queue#getJobCountByTypes('completed') => completed count
* Queue#getJobCountByTypes('completed,failed') => completed + failed count
* Queue#getJobCountByTypes('completed', 'failed') => completed + failed count
* Queue#getJobCountByTypes('completed', 'waiting', 'failed') => completed + waiting + failed count
*/
async getJobCountByTypes(...types) {
const result = await this.getJobCounts(...types);
return Object.values(result).reduce((sum, count) => sum + count, 0);
}
/**
* Returns the job counts for each type specified or every list/set in the queue by default.
*
* @returns An object, key (type) and value (count)
*/
async getJobCounts(...types) {
const currentTypes = this.sanitizeJobTypes(types);
const responses = await this.scripts.getCounts(currentTypes);
const counts = {};
responses.forEach((res, index) => {
counts[currentTypes[index]] = res || 0;
});
return counts;
}
/**
* Get current job state.
*
* @returns Returns one of these values:
* 'completed', 'failed', 'delayed', 'active', 'waiting', 'waiting-children', 'unknown'.
*/
getJobState(jobId) {
return this.scripts.getState(jobId);
}
/**
* Returns the number of jobs in completed status.
*/
getCompletedCount() {
return this.getJobCountByTypes('completed');
}
/**
* Returns the number of jobs in failed status.
*/
getFailedCount() {
return this.getJobCountByTypes('failed');
}
/**
* Returns the number of jobs in delayed status.
*/
getDelayedCount() {
return this.getJobCountByTypes('delayed');
}
/**
* Returns the number of jobs in active status.
*/
getActiveCount() {
return this.getJobCountByTypes('active');
}
/**
* Returns the number of jobs in prioritized status.
*/
getPrioritizedCount() {
return this.getJobCountByTypes('prioritized');
}
/**
* Returns the number of jobs in waiting or paused statuses.
*/
getWaitingCount() {
return this.getJobCountByTypes('waiting');
}
/**
* Returns the number of jobs in waiting-children status.
*/
getWaitingChildrenCount() {
return this.getJobCountByTypes('waiting-children');
}
/**
* Returns the jobs that are in the "waiting" status.
* @param start - zero based index from where to start returning jobs.
* @param end - zero based index where to stop returning jobs.
*/
getWaiting(start = 0, end = -1) {
return this.getJobs(['waiting'], start, end, true);
}
/**
* Returns the jobs that are in the "waiting-children" status.
* I.E. parent jobs that have at least one child that has not completed yet.
* @param start - zero based index from where to start returning jobs.
* @param end - zero based index where to stop returning jobs.
*/
getWaitingChildren(start = 0, end = -1) {
return this.getJobs(['waiting-children'], start, end, true);
}
/**
* Returns the jobs that are in the "active" status.
* @param start - zero based index from where to start returning jobs.
* @param end - zero based index where to stop returning jobs.
*/
getActive(start = 0, end = -1) {
return this.getJobs(['active'], start, end, true);
}
/**
* Returns the jobs that are in the "delayed" status.
* @param start - zero based index from where to start returning jobs.
* @param end - zero based index where to stop returning jobs.
*/
getDelayed(start = 0, end = -1) {
return this.getJobs(['delayed'], start, end, true);
}
/**
* Returns the jobs that are in the "prioritized" status.
* @param start - zero based index from where to start returning jobs.
* @param end - zero based index where to stop returning jobs.
*/
getPrioritized(start = 0, end = -1) {
return this.getJobs(['prioritized'], start, end, true);
}
/**
* Returns the jobs that are in the "completed" status.
* @param start - zero based index from where to start returning jobs.
* @param end - zero based index where to stop returning jobs.
*/
getCompleted(start = 0, end = -1) {
return this.getJobs(['completed'], start, end, false);
}
/**
* Returns the jobs that are in the "failed" status.
* @param start - zero based index from where to start returning jobs.
* @param end - zero based index where to stop returning jobs.
*/
getFailed(start = 0, end = -1) {
return this.getJobs(['failed'], start, end, false);
}
/**
* Returns the qualified job ids and the raw job data (if available) of the
* children jobs of the given parent job.
* It is possible to get either the already processed children, in this case
* an array of qualified job ids and their result values will be returned,
* or the pending children, in this case an array of qualified job ids will
* be returned.
* A qualified job id is a string representing the job id in a given queue,
* for example: "bull:myqueue:jobid".
*
* @param parentId The id of the parent job
* @param type "processed" | "pending"
* @param opts
*
* @returns { items: { id: string, v?: any, err?: string } [], jobs: JobJsonRaw[], total: number}
*/
async getDependencies(parentId, type, start, end) {
const key = this.toKey(type == 'processed'
? `${parentId}:processed`
: `${parentId}:dependencies`);
const { items, total, jobs } = await this.scripts.paginate(key, {
start,
end,
fetchJobs: true,
});
return {
items,
jobs,
total,
};
}
async getRanges(types, start = 0, end = 1, asc = false) {
const multiCommands = [];
this.commandByType(types, false, (key, command) => {
switch (command) {
case 'lrange':
multiCommands.push('lrange');
break;
case 'zrange':
multiCommands.push('zrange');
break;
}
});
const responses = await this.scripts.getRanges(types, start, end, asc);
let results = [];
responses.forEach((response, index) => {
const result = response || [];
if (asc && multiCommands[index] === 'lrange') {
results = results.concat(result.reverse());
}
else {
results = results.concat(result);
}
});
return [...new Set(results)];
}
/**
* Returns the jobs that are on the given statuses (note that JobType is synonym for job status)
* @param types - the statuses of the jobs to return.
* @param start - zero based index from where to start returning jobs.
* @param end - zero based index where to stop returning jobs.
* @param asc - if true, the jobs will be returned in ascending order.
*/
async getJobs(types, start = 0, end = -1, asc = false) {
const currentTypes = this.sanitizeJobTypes(types);
const jobIds = await this.getRanges(currentTypes, start, end, asc);
return Promise.all(jobIds.map(jobId => this.Job.fromId(this, jobId)));
}
/**
* Returns the logs for a given Job.
* @param jobId - the id of the job to get the logs for.
* @param start - zero based index from where to start returning jobs.
* @param end - zero based index where to stop returning jobs.
* @param asc - if true, the jobs will be returned in ascending order.
*/
async getJobLogs(jobId, start = 0, end = -1, asc = true) {
const client = await this.client;
const multi = client.multi();
const logsKey = this.toKey(jobId + ':logs');
if (asc) {
multi.lrange(logsKey, start, end);
}
else {
multi.lrange(logsKey, -(end + 1), -(start + 1));
}
multi.llen(logsKey);
const result = (await multi.exec());
if (!asc) {
result[0][1].reverse();
}
return {
logs: result[0][1],
count: result[1][1],
};
}
async baseGetClients(suffix) {
const client = await this.client;
const clients = (await client.client('LIST'));
try {
const list = this.parseClientList(clients, suffix);
return list;
}
catch (err) {
if (!utils_1.clientCommandMessageReg.test(err.message)) {
throw err;
}
return [];
}
}
/**
* Get the worker list related to the queue. i.e. all the known
* workers that are available to process jobs for this queue.
* Note: GCP does not support SETNAME, so this call will not work
*
* @returns - Returns an array with workers info.
*/
getWorkers() {
return this.baseGetClients(utils_1.WORKER_SUFFIX);
}
/**
* Get queue events list related to the queue.
* Note: GCP does not support SETNAME, so this call will not work
*
* @returns - Returns an array with queue events info.
*/
async getQueueEvents() {
return this.baseGetClients(utils_1.QUEUE_EVENT_SUFFIX);
}
/**
* Get queue metrics related to the queue.
*
* This method returns the gathered metrics for the queue.
* The metrics are represented as an array of job counts
* per unit of time (1 minute).
*
* @param start - Start point of the metrics, where 0
* is the newest point to be returned.
* @param end - End point of the metrics, where -1 is the
* oldest point to be returned.
*
* @returns - Returns an object with queue metrics.
*/
async getMetrics(type, start = 0, end = -1) {
const client = await this.client;
const metricsKey = this.toKey(`metrics:${type}`);
const dataKey = `${metricsKey}:data`;
const multi = client.multi();
multi.hmget(metricsKey, 'count', 'prevTS', 'prevCount');
multi.lrange(dataKey, start, end);
multi.llen(dataKey);
const [hmget, range, len] = (await multi.exec());
const [err, [count, prevTS, prevCount]] = hmget;
const [err2, data] = range;
const [err3, numPoints] = len;
if (err || err2) {
throw err || err2 || err3;
}
return {
meta: {
count: parseInt(count || '0', 10),
prevTS: parseInt(prevTS || '0', 10),
prevCount: parseInt(prevCount || '0', 10),
},
data,
count: numPoints,
};
}
parseClientList(list, suffix = '') {
const lines = list.split('\n');
const clients = [];
lines.forEach((line) => {
const client = {};
const keyValues = line.split(' ');
keyValues.forEach(function (keyValue) {
const index = keyValue.indexOf('=');
const key = keyValue.substring(0, index);
const value = keyValue.substring(index + 1);
client[key] = value;
});
const name = client['name'];
if (name && name === `${this.clientName()}${suffix ? `${suffix}` : ''}`) {
client['name'] = this.name;
clients.push(client);
}
});
return clients;
}
}
exports.QueueGetters = QueueGetters;
//# sourceMappingURL=queue-getters.js.map
File diff suppressed because one or more lines are too long
+41
View File
@@ -0,0 +1,41 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.QueueKeys = void 0;
class QueueKeys {
constructor(prefix = 'bull') {
this.prefix = prefix;
}
getKeys(name) {
const keys = {};
[
'',
'active',
'wait',
'waiting-children',
'paused',
'id',
'delayed',
'prioritized',
'stalled-check',
'completed',
'failed',
'stalled',
'repeat',
'limiter',
'meta',
'events',
'pc',
].forEach(key => {
keys[key] = this.toKey(name, key);
});
return keys;
}
toKey(name, type) {
return `${this.getQueueQualifiedName(name)}:${type}`;
}
getQueueQualifiedName(name) {
return `${this.prefix}:${name}`;
}
}
exports.QueueKeys = QueueKeys;
//# sourceMappingURL=queue-keys.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"queue-keys.js","sourceRoot":"","sources":["../../../src/classes/queue-keys.ts"],"names":[],"mappings":";;;AAEA,MAAa,SAAS;IACpB,YAA4B,SAAiB,MAAM;QAAvB,WAAM,GAAN,MAAM,CAAiB;IAAG,CAAC;IAEvD,OAAO,CAAC,IAAY;QAClB,MAAM,IAAI,GAAgC,EAAE,CAAC;QAC7C;YACE,EAAE;YACF,QAAQ;YACR,MAAM;YACN,kBAAkB;YAClB,QAAQ;YACR,IAAI;YACJ,SAAS;YACT,aAAa;YACb,eAAe;YACf,WAAW;YACX,QAAQ;YACR,SAAS;YACT,QAAQ;YACR,SAAS;YACT,MAAM;YACN,QAAQ;YACR,IAAI;SACL,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;YACd,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,IAAY,EAAE,IAAY;QAC9B,OAAO,GAAG,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;IACvD,CAAC;IAED,qBAAqB,CAAC,IAAY;QAChC,OAAO,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC;IAClC,CAAC;CACF;AArCD,8BAqCC"}
+344
View File
@@ -0,0 +1,344 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.Queue = void 0;
const lodash_1 = require("lodash");
const uuid_1 = require("uuid");
const job_1 = require("./job");
const queue_getters_1 = require("./queue-getters");
const repeat_1 = require("./repeat");
const version_1 = require("../version");
/**
* Queue
*
* This class provides methods to add jobs to a queue and some othe high-level
* administration such as pausing or deleting queues.
*
*/
class Queue extends queue_getters_1.QueueGetters {
constructor(name, opts, Connection) {
var _a;
super(name, Object.assign({ blockingConnection: false }, opts), Connection);
this.token = (0, uuid_1.v4)();
this.libName = 'bullmq';
this.jobsOpts = (_a = (0, lodash_1.get)(opts, 'defaultJobOptions')) !== null && _a !== void 0 ? _a : {};
this.waitUntilReady()
.then(client => {
if (!this.closing && !(opts === null || opts === void 0 ? void 0 : opts.skipMetasUpdate)) {
return client.hmset(this.keys.meta, this.metaValues);
}
})
.catch(err => {
// We ignore this error to avoid warnings. The error can still
// be received by listening to event 'error'
});
}
emit(event, ...args) {
return super.emit(event, ...args);
}
off(eventName, listener) {
super.off(eventName, listener);
return this;
}
on(event, listener) {
super.on(event, listener);
return this;
}
once(event, listener) {
super.once(event, listener);
return this;
}
/**
* Returns this instance current default job options.
*/
get defaultJobOptions() {
return Object.assign({}, this.jobsOpts);
}
get metaValues() {
var _a, _b, _c, _d;
return {
'opts.maxLenEvents': (_d = (_c = (_b = (_a = this.opts) === null || _a === void 0 ? void 0 : _a.streams) === null || _b === void 0 ? void 0 : _b.events) === null || _c === void 0 ? void 0 : _c.maxLen) !== null && _d !== void 0 ? _d : 10000,
version: `${this.libName}:${version_1.version}`,
};
}
/**
* Get library version.
*
* @returns the content of the meta.library field.
*/
async getVersion() {
const client = await this.client;
return await client.hget(this.keys.meta, 'version');
}
get repeat() {
return new Promise(async (resolve) => {
if (!this._repeat) {
this._repeat = new repeat_1.Repeat(this.name, Object.assign(Object.assign({}, this.opts), { connection: await this.client }));
this._repeat.on('error', e => this.emit.bind(this, e));
}
resolve(this._repeat);
});
}
/**
* Adds a new job to the queue.
*
* @param name - Name of the job to be added to the queue,.
* @param data - Arbitrary data to append to the job.
* @param opts - Job options that affects how the job is going to be processed.
*/
async add(name, data, opts) {
if (opts && opts.repeat) {
return (await this.repeat).addNextRepeatableJob(name, data, Object.assign(Object.assign({}, this.jobsOpts), opts), true);
}
else {
const jobId = opts === null || opts === void 0 ? void 0 : opts.jobId;
if (jobId == '0' || (jobId === null || jobId === void 0 ? void 0 : jobId.startsWith('0:'))) {
throw new Error("JobId cannot be '0' or start with 0:");
}
const job = await this.Job.create(this, name, data, Object.assign(Object.assign(Object.assign({}, this.jobsOpts), opts), { jobId }));
this.emit('waiting', job);
return job;
}
}
/**
* Adds an array of jobs to the queue. This method may be faster than adding
* one job at a time in a sequence.
*
* @param jobs - The array of jobs to add to the queue. Each job is defined by 3
* properties, 'name', 'data' and 'opts'. They follow the same signature as 'Queue.add'.
*/
addBulk(jobs) {
return this.Job.createBulk(this, jobs.map(job => {
var _a;
return ({
name: job.name,
data: job.data,
opts: Object.assign(Object.assign(Object.assign({}, this.jobsOpts), job.opts), { jobId: (_a = job.opts) === null || _a === void 0 ? void 0 : _a.jobId }),
});
}));
}
/**
* Pauses the processing of this queue globally.
*
* We use an atomic RENAME operation on the wait queue. Since
* we have blocking calls with BRPOPLPUSH on the wait queue, as long as the queue
* is renamed to 'paused', no new jobs will be processed (the current ones
* will run until finalized).
*
* Adding jobs requires a LUA script to check first if the paused list exist
* and in that case it will add it there instead of the wait list.
*/
async pause() {
await this.scripts.pause(true);
this.emit('paused');
}
/**
* Close the queue instance.
*
*/
async close() {
if (!this.closing) {
if (this._repeat) {
await this._repeat.close();
}
}
return super.close();
}
/**
* Resumes the processing of this queue globally.
*
* The method reverses the pause operation by resuming the processing of the
* queue.
*/
async resume() {
await this.scripts.pause(false);
this.emit('resumed');
}
/**
* Returns true if the queue is currently paused.
*/
async isPaused() {
const client = await this.client;
const pausedKeyExists = await client.hexists(this.keys.meta, 'paused');
return pausedKeyExists === 1;
}
/**
* Get all repeatable meta jobs.
*
* @param start - Offset of first job to return.
* @param end - Offset of last job to return.
* @param asc - Determine the order in which jobs are returned based on their
* next execution time.
*/
async getRepeatableJobs(start, end, asc) {
return (await this.repeat).getRepeatableJobs(start, end, asc);
}
/**
* Removes a repeatable job.
*
* Note: you need to use the exact same repeatOpts when deleting a repeatable job
* than when adding it.
*
* @see removeRepeatableByKey
*
* @param name - job name
* @param repeatOpts -
* @param jobId -
* @returns
*/
async removeRepeatable(name, repeatOpts, jobId) {
const repeat = await this.repeat;
const removed = await repeat.removeRepeatable(name, repeatOpts, jobId);
return !removed;
}
/**
* Removes a repeatable job by its key. Note that the key is the one used
* to store the repeatable job metadata and not one of the job iterations
* themselves. You can use "getRepeatableJobs" in order to get the keys.
*
* @see getRepeatableJobs
*
* @param repeatJobKey - to the repeatable job.
* @returns
*/
async removeRepeatableByKey(key) {
const repeat = await this.repeat;
const removed = await repeat.removeRepeatableByKey(key);
return !removed;
}
/**
* Removes the given job from the queue as well as all its
* dependencies.
*
* @param jobId - The id of the job to remove
* @param opts - Options to remove a job
* @returns 1 if it managed to remove the job or 0 if the job or
* any of its dependencies were locked.
*/
remove(jobId, { removeChildren = true } = {}) {
return this.scripts.remove(jobId, removeChildren);
}
/**
* Updates the given job's progress.
*
* @param jobId - The id of the job to update
* @param progress - number or object to be saved as progress.
*/
async updateJobProgress(jobId, progress) {
return this.scripts.updateProgress(jobId, progress);
}
/**
* Logs one row of job's log data.
*
* @param jobId - The job id to log against.
* @param logRow - string with log data to be logged.
* @param keepLogs - max number of log entries to keep (0 for unlimited).
*
* @returns The total number of log entries for this job so far.
*/
async addJobLog(jobId, logRow, keepLogs) {
return job_1.Job.addJobLog(this, jobId, logRow, keepLogs);
}
/**
* Drains the queue, i.e., removes all jobs that are waiting
* or delayed, but not active, completed or failed.
*
* @param delayed - Pass true if it should also clean the
* delayed jobs.
*/
drain(delayed = false) {
return this.scripts.drain(delayed);
}
/**
* Cleans jobs from a queue. Similar to drain but keeps jobs within a certain
* grace period.
*
* @param grace - The grace period
* @param limit - Max number of jobs to clean
* @param type - The type of job to clean
* Possible values are completed, wait, active, paused, delayed, failed. Defaults to completed.
* @returns Id jobs from the deleted records
*/
async clean(grace, limit, type = 'completed') {
const maxCount = limit || Infinity;
const maxCountPerCall = Math.min(10000, maxCount);
const timestamp = Date.now() - grace;
let deletedCount = 0;
const deletedJobsIds = [];
while (deletedCount < maxCount) {
const jobsIds = await this.scripts.cleanJobsInSet(type, timestamp, maxCountPerCall);
this.emit('cleaned', jobsIds, type);
deletedCount += jobsIds.length;
deletedJobsIds.push(...jobsIds);
if (jobsIds.length < maxCountPerCall) {
break;
}
}
return deletedJobsIds;
}
/**
* Completely destroys the queue and all of its contents irreversibly.
* This method will the *pause* the queue and requires that there are no
* active jobs. It is possible to bypass this requirement, i.e. not
* having active jobs using the "force" option.
*
* Note: This operation requires to iterate on all the jobs stored in the queue
* and can be slow for very large queues.
*
* @param opts - Obliterate options.
*/
async obliterate(opts) {
await this.pause();
let cursor = 0;
do {
cursor = await this.scripts.obliterate(Object.assign({ force: false, count: 1000 }, opts));
} while (cursor);
}
/**
* Retry all the failed jobs.
*
* @param opts: { count: number; state: FinishedStatus; timestamp: number}
* - count number to limit how many jobs will be moved to wait status per iteration,
* - state failed by default or completed.
* - timestamp from which timestamp to start moving jobs to wait status, default Date.now().
*
* @returns
*/
async retryJobs(opts = {}) {
let cursor = 0;
do {
cursor = await this.scripts.retryJobs(opts.state, opts.count, opts.timestamp);
} while (cursor);
}
/**
* Promote all the delayed jobs.
*
* @param opts: { count: number }
* - count number to limit how many jobs will be moved to wait status per iteration
*
* @returns
*/
async promoteJobs(opts = {}) {
let cursor = 0;
do {
cursor = await this.scripts.promoteJobs(opts.count);
} while (cursor);
}
/**
* Trim the event stream to an approximately maxLength.
*
* @param maxLength -
*/
async trimEvents(maxLength) {
const client = await this.client;
return client.xtrim(this.keys.events, 'MAXLEN', '~', maxLength);
}
/**
* Delete old priority helper key.
*/
async removeDeprecatedPriorityKey() {
const client = await this.client;
return client.del(this.toKey('priority'));
}
}
exports.Queue = Queue;
//# sourceMappingURL=queue.js.map
File diff suppressed because one or more lines are too long
+233
View File
@@ -0,0 +1,233 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.RedisConnection = void 0;
const events_1 = require("events");
const ioredis_1 = require("ioredis");
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
const utils_1 = require("ioredis/built/utils");
const utils_2 = require("../utils");
const version_1 = require("../version");
const scripts = require("../scripts");
const overrideMessage = [
'BullMQ: WARNING! Your redis options maxRetriesPerRequest must be null',
'and will be overridden by BullMQ.',
].join(' ');
const deprecationMessage = [
'BullMQ: DEPRECATION WARNING! Your redis options maxRetriesPerRequest must be null.',
'On the next versions having this settings will throw an exception',
].join(' ');
class RedisConnection extends events_1.EventEmitter {
constructor(opts, shared = false, blocking = true, skipVersionCheck = false) {
super();
this.shared = shared;
this.blocking = blocking;
this.capabilities = {
canDoubleTimeout: false,
};
if (!(0, utils_2.isRedisInstance)(opts)) {
this.checkBlockingOptions(overrideMessage, opts);
this.opts = Object.assign({ port: 6379, host: '127.0.0.1', retryStrategy: function (times) {
return Math.max(Math.min(Math.exp(times), 20000), 1000);
} }, opts);
if (this.blocking) {
this.opts.maxRetriesPerRequest = null;
}
}
else {
this._client = opts;
// Test if the redis instance is using keyPrefix
// and if so, throw an error.
if (this._client.options.keyPrefix) {
throw new Error('BullMQ: ioredis does not support ioredis prefixes, use the prefix option instead.');
}
if ((0, utils_2.isRedisCluster)(this._client)) {
this.opts = this._client.options.redisOptions;
}
else {
this.opts = this._client.options;
}
this.checkBlockingOptions(deprecationMessage, this.opts);
}
this.skipVersionCheck =
skipVersionCheck || !!(this.opts && this.opts.skipVersionCheck);
this.handleClientError = (err) => {
this.emit('error', err);
};
this.handleClientClose = () => {
this.emit('close');
};
this.handleClientReady = () => {
this.emit('ready');
};
this.initializing = this.init();
this.initializing.catch(err => this.emit('error', err));
}
checkBlockingOptions(msg, options) {
if (this.blocking && options && options.maxRetriesPerRequest) {
console.error(msg);
}
}
/**
* Waits for a redis client to be ready.
* @param redis - client
*/
static async waitUntilReady(client) {
if (client.status === 'ready') {
return;
}
if (client.status === 'wait') {
return client.connect();
}
if (client.status === 'end') {
throw new Error(utils_1.CONNECTION_CLOSED_ERROR_MSG);
}
let handleReady;
let handleEnd;
let handleError;
try {
await new Promise((resolve, reject) => {
let lastError;
handleError = (err) => {
lastError = err;
};
handleReady = () => {
resolve();
};
handleEnd = () => {
reject(lastError || new Error(utils_1.CONNECTION_CLOSED_ERROR_MSG));
};
(0, utils_2.increaseMaxListeners)(client, 3);
client.once('ready', handleReady);
client.on('end', handleEnd);
client.once('error', handleError);
});
}
finally {
client.removeListener('end', handleEnd);
client.removeListener('error', handleError);
client.removeListener('ready', handleReady);
(0, utils_2.decreaseMaxListeners)(client, 3);
}
}
get client() {
return this.initializing;
}
loadCommands(version, providedScripts) {
const finalScripts = providedScripts || scripts;
for (const property in finalScripts) {
// Only define the command if not already defined
const commandName = `${finalScripts[property].name}:${version}`;
if (!this._client[commandName]) {
this._client.defineCommand(commandName, {
numberOfKeys: finalScripts[property].keys,
lua: finalScripts[property].content,
});
}
}
}
async init() {
if (!this._client) {
this._client = new ioredis_1.default(this.opts);
}
(0, utils_2.increaseMaxListeners)(this._client, 3);
this._client.on('error', this.handleClientError);
// ioredis treats connection errors as a different event ('close')
this._client.on('close', this.handleClientClose);
this._client.on('ready', this.handleClientReady);
await RedisConnection.waitUntilReady(this._client);
this.loadCommands(version_1.version);
this.version = await this.getRedisVersion();
if (this.skipVersionCheck !== true && !this.closing) {
if ((0, utils_2.isRedisVersionLowerThan)(this.version, RedisConnection.minimumVersion)) {
throw new Error(`Redis version needs to be greater or equal than ${RedisConnection.minimumVersion} Current: ${this.version}`);
}
if ((0, utils_2.isRedisVersionLowerThan)(this.version, RedisConnection.recommendedMinimumVersion)) {
console.warn(`It is highly recommended to use a minimum Redis version of ${RedisConnection.recommendedMinimumVersion}
Current: ${this.version}`);
}
}
this.capabilities = {
canDoubleTimeout: !(0, utils_2.isRedisVersionLowerThan)(this.version, '6.0.0'),
};
return this._client;
}
async disconnect(wait = true) {
const client = await this.client;
if (client.status !== 'end') {
let _resolve, _reject;
if (!wait) {
return client.disconnect();
}
const disconnecting = new Promise((resolve, reject) => {
(0, utils_2.increaseMaxListeners)(client, 2);
client.once('end', resolve);
client.once('error', reject);
_resolve = resolve;
_reject = reject;
});
client.disconnect();
try {
await disconnecting;
}
finally {
(0, utils_2.decreaseMaxListeners)(client, 2);
client.removeListener('end', _resolve);
client.removeListener('error', _reject);
}
}
}
async reconnect() {
const client = await this.client;
return client.connect();
}
async close() {
if (!this.closing) {
this.closing = true;
try {
await this.initializing;
if (!this.shared) {
await this._client.quit();
}
}
catch (error) {
if ((0, utils_2.isNotConnectionError)(error)) {
throw error;
}
}
finally {
this._client.off('error', this.handleClientError);
this._client.off('close', this.handleClientClose);
this._client.off('ready', this.handleClientReady);
(0, utils_2.decreaseMaxListeners)(this._client, 3);
this.removeAllListeners();
}
}
}
async getRedisVersion() {
const doc = await this._client.info();
const redisPrefix = 'redis_version:';
const maxMemoryPolicyPrefix = 'maxmemory_policy:';
const lines = doc.split('\r\n');
let redisVersion;
for (let i = 0; i < lines.length; i++) {
if (lines[i].indexOf(maxMemoryPolicyPrefix) === 0) {
const maxMemoryPolicy = lines[i].substr(maxMemoryPolicyPrefix.length);
if (maxMemoryPolicy !== 'noeviction') {
console.warn(`IMPORTANT! Eviction policy is ${maxMemoryPolicy}. It should be "noeviction"`);
}
}
if (lines[i].indexOf(redisPrefix) === 0) {
redisVersion = lines[i].substr(redisPrefix.length);
}
}
return redisVersion;
}
get redisVersion() {
return this.version;
}
}
exports.RedisConnection = RedisConnection;
RedisConnection.minimumVersion = '5.0.0';
RedisConnection.recommendedMinimumVersion = '6.2.0';
//# sourceMappingURL=redis-connection.js.map
File diff suppressed because one or more lines are too long
+151
View File
@@ -0,0 +1,151 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.getNextMillis = exports.Repeat = void 0;
const tslib_1 = require("tslib");
const cron_parser_1 = require("cron-parser");
const crypto_1 = require("crypto");
const queue_base_1 = require("./queue-base");
class Repeat extends queue_base_1.QueueBase {
constructor(name, opts, Connection) {
super(name, opts, Connection);
this.repeatStrategy =
(opts.settings && opts.settings.repeatStrategy) || exports.getNextMillis;
this.repeatKeyHashAlgorithm =
(opts.settings && opts.settings.repeatKeyHashAlgorithm) || 'md5';
}
async addNextRepeatableJob(name, data, opts, skipCheckExists) {
var _a;
// HACK: This is a temporary fix to enable easy migration from bullmq <3.0.0
// to >= 3.0.0. It should be removed when moving to 4.x.
const repeatOpts = Object.assign({}, opts.repeat);
(_a = repeatOpts.pattern) !== null && _a !== void 0 ? _a : (repeatOpts.pattern = repeatOpts.cron);
delete repeatOpts.cron;
const prevMillis = opts.prevMillis || 0;
const currentCount = repeatOpts.count ? repeatOpts.count + 1 : 1;
if (typeof repeatOpts.limit !== 'undefined' &&
currentCount > repeatOpts.limit) {
return;
}
let now = Date.now();
if (!(typeof repeatOpts.endDate === undefined) &&
now > new Date(repeatOpts.endDate).getTime()) {
return;
}
now = prevMillis < now ? now : prevMillis;
const nextMillis = await this.repeatStrategy(now, repeatOpts, name);
const pattern = repeatOpts.pattern;
const hasImmediately = Boolean((repeatOpts.every || pattern) && repeatOpts.immediately);
const offset = hasImmediately ? now - nextMillis : undefined;
if (nextMillis) {
// We store the undecorated opts.jobId into the repeat options
if (!prevMillis && opts.jobId) {
repeatOpts.jobId = opts.jobId;
}
const repeatJobKey = getRepeatKey(name, repeatOpts);
let repeatableExists = true;
if (!skipCheckExists) {
// Check that the repeatable job hasn't been removed
// TODO: a lua script would be better here
const client = await this.client;
repeatableExists = !!(await client.zscore(this.keys.repeat, repeatJobKey));
}
const { immediately } = repeatOpts, filteredRepeatOpts = tslib_1.__rest(repeatOpts, ["immediately"]);
// The job could have been deleted since this check
if (repeatableExists) {
return this.createNextJob(name, nextMillis, repeatJobKey, Object.assign(Object.assign({}, opts), { repeat: Object.assign({ offset }, filteredRepeatOpts) }), data, currentCount, hasImmediately);
}
}
}
async createNextJob(name, nextMillis, repeatJobKey, opts, data, currentCount, hasImmediately) {
const client = await this.client;
//
// Generate unique job id for this iteration.
//
const jobId = this.getRepeatJobId(name, nextMillis, this.hash(repeatJobKey), opts.repeat.jobId);
const now = Date.now();
const delay = nextMillis + (opts.repeat.offset ? opts.repeat.offset : 0) - now;
const mergedOpts = Object.assign(Object.assign({}, opts), { jobId, delay: delay < 0 || hasImmediately ? 0 : delay, timestamp: now, prevMillis: nextMillis, repeatJobKey });
mergedOpts.repeat = Object.assign(Object.assign({}, opts.repeat), { count: currentCount });
await client.zadd(this.keys.repeat, nextMillis.toString(), repeatJobKey);
return this.Job.create(this, name, data, mergedOpts);
}
async removeRepeatable(name, repeat, jobId) {
const repeatJobKey = getRepeatKey(name, Object.assign(Object.assign({}, repeat), { jobId }));
const repeatJobId = this.getRepeatJobId(name, '', this.hash(repeatJobKey), jobId || repeat.jobId);
return this.scripts.removeRepeatable(repeatJobId, repeatJobKey);
}
async removeRepeatableByKey(repeatJobKey) {
const data = this.keyToData(repeatJobKey);
const repeatJobId = this.getRepeatJobId(data.name, '', this.hash(repeatJobKey), data.id);
return this.scripts.removeRepeatable(repeatJobId, repeatJobKey);
}
keyToData(key, next) {
const data = key.split(':');
const pattern = data.slice(4).join(':') || null;
return {
key,
name: data[0],
id: data[1] || null,
endDate: parseInt(data[2]) || null,
tz: data[3] || null,
pattern,
next,
};
}
async getRepeatableJobs(start = 0, end = -1, asc = false) {
const client = await this.client;
const key = this.keys.repeat;
const result = asc
? await client.zrange(key, start, end, 'WITHSCORES')
: await client.zrevrange(key, start, end, 'WITHSCORES');
const jobs = [];
for (let i = 0; i < result.length; i += 2) {
jobs.push(this.keyToData(result[i], parseInt(result[i + 1])));
}
return jobs;
}
async getRepeatableCount() {
const client = await this.client;
return client.zcard(this.toKey('repeat'));
}
hash(str) {
return (0, crypto_1.createHash)(this.repeatKeyHashAlgorithm).update(str).digest('hex');
}
getRepeatJobId(name, nextMillis, namespace, jobId) {
const checksum = this.hash(`${name}${jobId || ''}${namespace}`);
return `repeat:${checksum}:${nextMillis}`;
// return `repeat:${jobId || ''}:${name}:${namespace}:${nextMillis}`;
//return `repeat:${name}:${namespace}:${nextMillis}`;
}
}
exports.Repeat = Repeat;
function getRepeatKey(name, repeat) {
const endDate = repeat.endDate ? new Date(repeat.endDate).getTime() : '';
const tz = repeat.tz || '';
const pattern = repeat.pattern;
const suffix = (pattern ? pattern : String(repeat.every)) || '';
const jobId = repeat.jobId ? repeat.jobId : '';
return `${name}:${jobId}:${endDate}:${tz}:${suffix}`;
}
const getNextMillis = (millis, opts) => {
const pattern = opts.pattern;
if (pattern && opts.every) {
throw new Error('Both .pattern and .every options are defined for this repeatable job');
}
if (opts.every) {
return (Math.floor(millis / opts.every) * opts.every +
(opts.immediately ? 0 : opts.every));
}
const currentDate = opts.startDate && new Date(opts.startDate) > new Date(millis)
? new Date(opts.startDate)
: new Date(millis);
const interval = (0, cron_parser_1.parseExpression)(pattern, Object.assign(Object.assign({}, opts), { currentDate }));
try {
return interval.next().getTime();
}
catch (e) {
// Ignore error
}
};
exports.getNextMillis = getNextMillis;
//# sourceMappingURL=repeat.js.map
File diff suppressed because one or more lines are too long
+65
View File
@@ -0,0 +1,65 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const enums_1 = require("../enums");
const sandbox = (processFile, childPool) => {
return async function process(job, token) {
const child = await childPool.retain(processFile);
let msgHandler;
let exitHandler;
await child.send({
cmd: enums_1.ChildCommand.Start,
job: job.asJSONSandbox(),
token,
});
const done = new Promise((resolve, reject) => {
msgHandler = async (msg) => {
var _a, _b;
switch (msg.cmd) {
case enums_1.ParentCommand.Completed:
resolve(msg.value);
break;
case enums_1.ParentCommand.Failed:
case enums_1.ParentCommand.Error: {
const err = new Error();
Object.assign(err, msg.value);
reject(err);
break;
}
case enums_1.ParentCommand.Progress:
await job.updateProgress(msg.value);
break;
case enums_1.ParentCommand.Log:
await job.log(msg.value);
break;
case enums_1.ParentCommand.MoveToDelayed:
await job.moveToDelayed((_a = msg.value) === null || _a === void 0 ? void 0 : _a.timestamp, (_b = msg.value) === null || _b === void 0 ? void 0 : _b.token);
break;
case enums_1.ParentCommand.Update:
await job.updateData(msg.value);
break;
}
};
exitHandler = (exitCode, signal) => {
reject(new Error('Unexpected exit code: ' + exitCode + ' signal: ' + signal));
};
child.on('message', msgHandler);
child.on('exit', exitHandler);
});
try {
await done;
return done;
}
finally {
child.off('message', msgHandler);
child.off('exit', exitHandler);
if (child.exitCode !== null || /SIG.*/.test(`${child.signalCode}`)) {
childPool.remove(child);
}
else {
childPool.release(child);
}
}
};
};
exports.default = sandbox;
//# sourceMappingURL=sandbox.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"sandbox.js","sourceRoot":"","sources":["../../../src/classes/sandbox.ts"],"names":[],"mappings":";;AAAA,oCAAuD;AAKvD,MAAM,OAAO,GAAG,CACd,WAAgB,EAChB,SAAoB,EACpB,EAAE;IACF,OAAO,KAAK,UAAU,OAAO,CAAC,GAAiB,EAAE,KAAc;QAC7D,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QAClD,IAAI,UAAe,CAAC;QACpB,IAAI,WAAgB,CAAC;QAErB,MAAM,KAAK,CAAC,IAAI,CAAC;YACf,GAAG,EAAE,oBAAY,CAAC,KAAK;YACvB,GAAG,EAAE,GAAG,CAAC,aAAa,EAAE;YACxB,KAAK;SACN,CAAC,CAAC;QAEH,MAAM,IAAI,GAAe,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACvD,UAAU,GAAG,KAAK,EAAE,GAAiB,EAAE,EAAE;;gBACvC,QAAQ,GAAG,CAAC,GAAG,EAAE;oBACf,KAAK,qBAAa,CAAC,SAAS;wBAC1B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;wBACnB,MAAM;oBACR,KAAK,qBAAa,CAAC,MAAM,CAAC;oBAC1B,KAAK,qBAAa,CAAC,KAAK,CAAC,CAAC;wBACxB,MAAM,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC;wBACxB,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;wBAC9B,MAAM,CAAC,GAAG,CAAC,CAAC;wBACZ,MAAM;qBACP;oBACD,KAAK,qBAAa,CAAC,QAAQ;wBACzB,MAAM,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;wBACpC,MAAM;oBACR,KAAK,qBAAa,CAAC,GAAG;wBACpB,MAAM,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;wBACzB,MAAM;oBACR,KAAK,qBAAa,CAAC,aAAa;wBAC9B,MAAM,GAAG,CAAC,aAAa,CAAC,MAAA,GAAG,CAAC,KAAK,0CAAE,SAAS,EAAE,MAAA,GAAG,CAAC,KAAK,0CAAE,KAAK,CAAC,CAAC;wBAChE,MAAM;oBACR,KAAK,qBAAa,CAAC,MAAM;wBACvB,MAAM,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;wBAChC,MAAM;iBACT;YACH,CAAC,CAAC;YAEF,WAAW,GAAG,CAAC,QAAa,EAAE,MAAW,EAAE,EAAE;gBAC3C,MAAM,CACJ,IAAI,KAAK,CAAC,wBAAwB,GAAG,QAAQ,GAAG,WAAW,GAAG,MAAM,CAAC,CACtE,CAAC;YACJ,CAAC,CAAC;YAEF,KAAK,CAAC,EAAE,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;YAChC,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,IAAI;YACF,MAAM,IAAI,CAAC;YACX,OAAO,IAAI,CAAC;SACb;gBAAS;YACR,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;YACjC,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;YAE/B,IAAI,KAAK,CAAC,QAAQ,KAAK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,UAAU,EAAE,CAAC,EAAE;gBAClE,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;aACzB;iBAAM;gBACL,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;aAC1B;SACF;IACH,CAAC,CAAC;AACJ,CAAC,CAAC;AAEF,kBAAe,OAAO,CAAC"}
+782
View File
@@ -0,0 +1,782 @@
/**
* Includes all the scripts needed by the queue and jobs.
*/
/*eslint-env node */
'use strict';
Object.defineProperty(exports, "__esModule", { value: true });
exports.raw2NextJobData = exports.Scripts = void 0;
const msgpackr_1 = require("msgpackr");
const packer = new msgpackr_1.Packr({
useRecords: false,
encodeUndefinedAsNil: true,
});
const pack = packer.pack;
const enums_1 = require("../enums");
const utils_1 = require("../utils");
const version_1 = require("../version");
class Scripts {
constructor(queue) {
this.queue = queue;
const queueKeys = this.queue.keys;
this.version = version_1.version;
this.moveToFinishedKeys = [
queueKeys.wait,
queueKeys.active,
queueKeys.prioritized,
queueKeys.events,
queueKeys.stalled,
queueKeys.limiter,
queueKeys.delayed,
queueKeys.paused,
queueKeys.meta,
queueKeys.pc,
undefined,
undefined,
undefined,
];
}
execCommand(client, commandName, args) {
const commandNameWithVersion = `${commandName}:${this.version}`;
return client[commandNameWithVersion](args);
}
async isJobInList(listKey, jobId) {
const client = await this.queue.client;
let result;
if ((0, utils_1.isRedisVersionLowerThan)(this.queue.redisVersion, '6.0.6')) {
result = await this.execCommand(client, 'isJobInList', [listKey, jobId]);
}
else {
result = await client.lpos(listKey, jobId);
}
return Number.isInteger(result);
}
async addDelayedJob(client, job, encodedOpts, args) {
const queueKeys = this.queue.keys;
const keys = [
queueKeys.wait,
queueKeys.paused,
queueKeys.meta,
queueKeys.id,
queueKeys.delayed,
queueKeys.completed,
queueKeys.events,
];
keys.push(pack(args), job.data, encodedOpts);
return this.execCommand(client, 'addDelayedJob', keys);
}
async addPrioritizedJob(client, job, encodedOpts, args) {
const queueKeys = this.queue.keys;
const keys = [
queueKeys.wait,
queueKeys.paused,
queueKeys.meta,
queueKeys.id,
queueKeys.prioritized,
queueKeys.completed,
queueKeys.events,
queueKeys.pc,
];
keys.push(pack(args), job.data, encodedOpts);
return this.execCommand(client, 'addPrioritizedJob', keys);
}
async addParentJob(client, job, encodedOpts, args) {
const queueKeys = this.queue.keys;
const keys = [
queueKeys.meta,
queueKeys.id,
queueKeys.completed,
queueKeys.events,
];
keys.push(pack(args), job.data, encodedOpts);
return this.execCommand(client, 'addParentJob', keys);
}
async addJob(client, job, opts, jobId, parentOpts = {}) {
const queueKeys = this.queue.keys;
const parent = job.parent
? Object.assign(Object.assign({}, job.parent), { fpof: opts.fpof, rdof: opts.rdof }) : null;
const args = [
queueKeys[''],
typeof jobId !== 'undefined' ? jobId : '',
job.name,
job.timestamp,
job.parentKey || null,
parentOpts.waitChildrenKey || null,
parentOpts.parentDependenciesKey || null,
parent,
job.repeatJobKey,
];
let encodedOpts;
if (opts.repeat) {
const repeat = Object.assign({}, opts.repeat);
if (repeat.startDate) {
repeat.startDate = +new Date(repeat.startDate);
}
if (repeat.endDate) {
repeat.endDate = +new Date(repeat.endDate);
}
encodedOpts = pack(Object.assign(Object.assign({}, opts), { repeat }));
}
else {
encodedOpts = pack(opts);
}
let result;
if (parentOpts.waitChildrenKey) {
result = await this.addParentJob(client, job, encodedOpts, args);
}
else if (opts.delay) {
result = await this.addDelayedJob(client, job, encodedOpts, args);
}
else if (opts.priority) {
result = await this.addPrioritizedJob(client, job, encodedOpts, args);
}
else {
const keys = [
queueKeys.wait,
queueKeys.paused,
queueKeys.meta,
queueKeys.id,
queueKeys.completed,
queueKeys.events,
];
keys.push(pack(args), job.data, encodedOpts);
result = await this.execCommand(client, 'addStandardJob', keys);
}
if (result < 0) {
throw this.finishedErrors(result, parentOpts.parentKey, 'addJob');
}
return result;
}
async pause(pause) {
const client = await this.queue.client;
let src = 'wait', dst = 'paused';
if (!pause) {
src = 'paused';
dst = 'wait';
}
const keys = [src, dst, 'meta', 'prioritized'].map((name) => this.queue.toKey(name));
keys.push(this.queue.keys.events);
return this.execCommand(client, 'pause', keys.concat([pause ? 'paused' : 'resumed']));
}
removeRepeatableArgs(repeatJobId, repeatJobKey) {
const queueKeys = this.queue.keys;
const keys = [queueKeys.repeat, queueKeys.delayed];
const args = [repeatJobId, repeatJobKey, queueKeys['']];
return keys.concat(args);
}
async removeRepeatable(repeatJobId, repeatJobKey) {
const client = await this.queue.client;
const args = this.removeRepeatableArgs(repeatJobId, repeatJobKey);
return this.execCommand(client, 'removeRepeatable', args);
}
async remove(jobId, removeChildren) {
const client = await this.queue.client;
const keys = [''].map(name => this.queue.toKey(name));
return this.execCommand(client, 'removeJob', keys.concat([jobId, removeChildren ? 1 : 0]));
}
async extendLock(jobId, token, duration, client) {
client = client || (await this.queue.client);
const args = [
this.queue.toKey(jobId) + ':lock',
this.queue.keys.stalled,
token,
duration,
jobId,
];
return this.execCommand(client, 'extendLock', args);
}
async updateData(job, data) {
const client = await this.queue.client;
const keys = [this.queue.toKey(job.id)];
const dataJson = JSON.stringify(data);
const result = await this.execCommand(client, 'updateData', keys.concat([dataJson]));
if (result < 0) {
throw this.finishedErrors(result, job.id, 'updateData');
}
}
async updateProgress(jobId, progress) {
const client = await this.queue.client;
const keys = [
this.queue.toKey(jobId),
this.queue.keys.events,
this.queue.keys.meta,
];
const progressJson = JSON.stringify(progress);
const result = await this.execCommand(client, 'updateProgress', keys.concat([jobId, progressJson]));
if (result < 0) {
throw this.finishedErrors(result, jobId, 'updateProgress');
}
}
moveToFinishedArgs(job, val, propVal, shouldRemove, target, token, timestamp, fetchNext = true) {
var _a, _b, _c, _d, _e;
const queueKeys = this.queue.keys;
const opts = this.queue.opts;
const workerKeepJobs = target === 'completed' ? opts.removeOnComplete : opts.removeOnFail;
const metricsKey = this.queue.toKey(`metrics:${target}`);
const keys = this.moveToFinishedKeys;
keys[10] = queueKeys[target];
keys[11] = this.queue.toKey((_a = job.id) !== null && _a !== void 0 ? _a : '');
keys[12] = metricsKey;
const keepJobs = this.getKeepJobs(shouldRemove, workerKeepJobs);
const args = [
job.id,
timestamp,
propVal,
typeof val === 'undefined' ? 'null' : val,
target,
JSON.stringify({ jobId: job.id, val: val }),
!fetchNext || this.queue.closing ? 0 : 1,
queueKeys[''],
pack({
token,
keepJobs,
limiter: opts.limiter,
lockDuration: opts.lockDuration,
attempts: job.opts.attempts,
attemptsMade: job.attemptsMade,
maxMetricsSize: ((_b = opts.metrics) === null || _b === void 0 ? void 0 : _b.maxDataPoints)
? (_c = opts.metrics) === null || _c === void 0 ? void 0 : _c.maxDataPoints
: '',
fpof: !!((_d = job.opts) === null || _d === void 0 ? void 0 : _d.failParentOnFailure),
rdof: !!((_e = job.opts) === null || _e === void 0 ? void 0 : _e.removeDependencyOnFailure),
}),
];
return keys.concat(args);
}
getKeepJobs(shouldRemove, workerKeepJobs) {
if (typeof shouldRemove === 'undefined') {
return workerKeepJobs || { count: shouldRemove ? 0 : -1 };
}
return typeof shouldRemove === 'object'
? shouldRemove
: typeof shouldRemove === 'number'
? { count: shouldRemove }
: { count: shouldRemove ? 0 : -1 };
}
async moveToFinished(jobId, args) {
const client = await this.queue.client;
const result = await this.execCommand(client, 'moveToFinished', args);
if (result < 0) {
throw this.finishedErrors(result, jobId, 'moveToFinished', 'active');
}
else {
if (typeof result !== 'undefined') {
return raw2NextJobData(result);
}
}
}
finishedErrors(code, jobId, command, state) {
switch (code) {
case enums_1.ErrorCode.JobNotExist:
return new Error(`Missing key for job ${jobId}. ${command}`);
case enums_1.ErrorCode.JobLockNotExist:
return new Error(`Missing lock for job ${jobId}. ${command}`);
case enums_1.ErrorCode.JobNotInState:
return new Error(`Job ${jobId} is not in the ${state} state. ${command}`);
case enums_1.ErrorCode.JobPendingDependencies:
return new Error(`Job ${jobId} has pending dependencies. ${command}`);
case enums_1.ErrorCode.ParentJobNotExist:
return new Error(`Missing key for parent job ${jobId}. ${command}`);
case enums_1.ErrorCode.JobLockMismatch:
return new Error(`Lock mismatch for job ${jobId}. Cmd ${command} from ${state}`);
default:
return new Error(`Unknown code ${code} error for ${jobId}. ${command}`);
}
}
drainArgs(delayed) {
const queueKeys = this.queue.keys;
const keys = [
queueKeys.wait,
queueKeys.paused,
delayed ? queueKeys.delayed : '',
queueKeys.prioritized,
];
const args = [queueKeys['']];
return keys.concat(args);
}
async drain(delayed) {
const client = await this.queue.client;
const args = this.drainArgs(delayed);
return this.execCommand(client, 'drain', args);
}
getRangesArgs(types, start, end, asc) {
const queueKeys = this.queue.keys;
const transformedTypes = types.map(type => {
return type === 'waiting' ? 'wait' : type;
});
const keys = [queueKeys['']];
const args = [start, end, asc ? '1' : '0', ...transformedTypes];
return keys.concat(args);
}
async getRanges(types, start = 0, end = 1, asc = false) {
const client = await this.queue.client;
const args = this.getRangesArgs(types, start, end, asc);
return this.execCommand(client, 'getRanges', args);
}
getCountsArgs(types) {
const queueKeys = this.queue.keys;
const transformedTypes = types.map(type => {
return type === 'waiting' ? 'wait' : type;
});
const keys = [queueKeys['']];
const args = [...transformedTypes];
return keys.concat(args);
}
async getCounts(types) {
const client = await this.queue.client;
const args = this.getCountsArgs(types);
return this.execCommand(client, 'getCounts', args);
}
moveToCompletedArgs(job, returnvalue, removeOnComplete, token, fetchNext = false) {
const timestamp = Date.now();
return this.moveToFinishedArgs(job, returnvalue, 'returnvalue', removeOnComplete, 'completed', token, timestamp, fetchNext);
}
moveToFailedArgs(job, failedReason, removeOnFailed, token, fetchNext = false) {
const timestamp = Date.now();
return this.moveToFinishedArgs(job, failedReason, 'failedReason', removeOnFailed, 'failed', token, timestamp, fetchNext);
}
async isFinished(jobId, returnValue = false) {
const client = await this.queue.client;
const keys = ['completed', 'failed', jobId].map((key) => {
return this.queue.toKey(key);
});
return this.execCommand(client, 'isFinished', keys.concat([jobId, returnValue ? '1' : '']));
}
async getState(jobId) {
const client = await this.queue.client;
const keys = [
'completed',
'failed',
'delayed',
'active',
'wait',
'paused',
'waiting-children',
'prioritized',
].map((key) => {
return this.queue.toKey(key);
});
if ((0, utils_1.isRedisVersionLowerThan)(this.queue.redisVersion, '6.0.6')) {
return this.execCommand(client, 'getState', keys.concat([jobId]));
}
return this.execCommand(client, 'getStateV2', keys.concat([jobId]));
}
async changeDelay(jobId, delay) {
const client = await this.queue.client;
const args = this.changeDelayArgs(jobId, delay);
const result = await this.execCommand(client, 'changeDelay', args);
if (result < 0) {
throw this.finishedErrors(result, jobId, 'changeDelay', 'delayed');
}
}
changeDelayArgs(jobId, delay) {
//
// Bake in the job id first 12 bits into the timestamp
// to guarantee correct execution order of delayed jobs
// (up to 4096 jobs per given timestamp or 4096 jobs apart per timestamp)
//
// WARNING: Jobs that are so far apart that they wrap around will cause FIFO to fail
//
let timestamp = Date.now() + delay;
if (timestamp > 0) {
timestamp = timestamp * 0x1000 + (+jobId & 0xfff);
}
const keys = ['delayed', jobId].map(name => {
return this.queue.toKey(name);
});
keys.push.apply(keys, [this.queue.keys.events]);
return keys.concat([delay, JSON.stringify(timestamp), jobId]);
}
async changePriority(jobId, priority = 0, lifo = false) {
const client = await this.queue.client;
const args = this.changePriorityArgs(jobId, priority, lifo);
const result = await this.execCommand(client, 'changePriority', args);
if (result < 0) {
throw this.finishedErrors(result, jobId, 'changePriority');
}
}
changePriorityArgs(jobId, priority = 0, lifo = false) {
const keys = [
this.queue.keys.wait,
this.queue.keys.paused,
this.queue.keys.meta,
this.queue.keys.prioritized,
this.queue.keys.pc,
];
return keys.concat([
priority,
this.queue.toKey(jobId),
jobId,
lifo ? 1 : 0,
]);
}
// Note: We have an issue here with jobs using custom job ids
moveToDelayedArgs(jobId, timestamp, token, delay) {
//
// Bake in the job id first 12 bits into the timestamp
// to guarantee correct execution order of delayed jobs
// (up to 4096 jobs per given timestamp or 4096 jobs apart per timestamp)
//
// WARNING: Jobs that are so far apart that they wrap around will cause FIFO to fail
//
timestamp = Math.max(0, timestamp !== null && timestamp !== void 0 ? timestamp : 0);
if (timestamp > 0) {
timestamp = timestamp * 0x1000 + (+jobId & 0xfff);
}
const keys = [
'wait',
'active',
'prioritized',
'delayed',
jobId,
].map(name => {
return this.queue.toKey(name);
});
keys.push.apply(keys, [
this.queue.keys.events,
this.queue.keys.paused,
this.queue.keys.meta,
]);
return keys.concat([
this.queue.keys[''],
Date.now(),
JSON.stringify(timestamp),
jobId,
token,
delay,
]);
}
saveStacktraceArgs(jobId, stacktrace, failedReason) {
const keys = [this.queue.toKey(jobId)];
return keys.concat([stacktrace, failedReason]);
}
moveToWaitingChildrenArgs(jobId, token, opts) {
const timestamp = Date.now();
const childKey = (0, utils_1.getParentKey)(opts.child);
const keys = [`${jobId}:lock`, 'active', 'waiting-children', jobId].map(name => {
return this.queue.toKey(name);
});
return keys.concat([
token,
childKey !== null && childKey !== void 0 ? childKey : '',
JSON.stringify(timestamp),
jobId,
]);
}
async moveToDelayed(jobId, timestamp, delay, token = '0') {
const client = await this.queue.client;
const args = this.moveToDelayedArgs(jobId, timestamp, token, delay);
const result = await this.execCommand(client, 'moveToDelayed', args);
if (result < 0) {
throw this.finishedErrors(result, jobId, 'moveToDelayed', 'active');
}
}
/**
* Move parent job to waiting-children state.
*
* @returns true if job is successfully moved, false if there are pending dependencies.
* @throws JobNotExist
* This exception is thrown if jobId is missing.
* @throws JobLockNotExist
* This exception is thrown if job lock is missing.
* @throws JobNotInState
* This exception is thrown if job is not in active state.
*/
async moveToWaitingChildren(jobId, token, opts = {}) {
const client = await this.queue.client;
const args = this.moveToWaitingChildrenArgs(jobId, token, opts);
const result = await this.execCommand(client, 'moveToWaitingChildren', args);
switch (result) {
case 0:
return true;
case 1:
return false;
default:
throw this.finishedErrors(result, jobId, 'moveToWaitingChildren', 'active');
}
}
/**
* Remove jobs in a specific state.
*
* @returns Id jobs from the deleted records.
*/
async cleanJobsInSet(set, timestamp, limit = 0) {
const client = await this.queue.client;
return this.execCommand(client, 'cleanJobsInSet', [
this.queue.toKey(set),
this.queue.toKey('events'),
this.queue.toKey(''),
timestamp,
limit,
set,
]);
}
retryJobArgs(jobId, lifo, token) {
const keys = [
'active',
'wait',
'paused',
jobId,
'meta',
].map(name => {
return this.queue.toKey(name);
});
keys.push(this.queue.keys.events, this.queue.keys.delayed, this.queue.keys.prioritized, this.queue.keys.pc);
const pushCmd = (lifo ? 'R' : 'L') + 'PUSH';
return keys.concat([
this.queue.toKey(''),
Date.now(),
pushCmd,
jobId,
token,
]);
}
moveJobsToWaitArgs(state, count, timestamp) {
const keys = [
this.queue.toKey(''),
this.queue.keys.events,
this.queue.toKey(state),
this.queue.toKey('wait'),
this.queue.toKey('paused'),
this.queue.toKey('meta'),
];
const args = [count, timestamp, state];
return keys.concat(args);
}
async retryJobs(state = 'failed', count = 1000, timestamp = new Date().getTime()) {
const client = await this.queue.client;
const args = this.moveJobsToWaitArgs(state, count, timestamp);
return this.execCommand(client, 'moveJobsToWait', args);
}
async promoteJobs(count = 1000) {
const client = await this.queue.client;
const args = this.moveJobsToWaitArgs('delayed', count, Number.MAX_VALUE);
return this.execCommand(client, 'moveJobsToWait', args);
}
/**
* Attempts to reprocess a job
*
* @param job -
* @param state - The expected job state. If the job is not found
* on the provided state, then it's not reprocessed. Supported states: 'failed', 'completed'
*
* @returns Returns a promise that evaluates to a return code:
* 1 means the operation was a success
* 0 means the job does not exist
* -1 means the job is currently locked and can't be retried.
* -2 means the job was not found in the expected set
*/
async reprocessJob(job, state) {
const client = await this.queue.client;
const keys = [
this.queue.toKey(job.id),
this.queue.keys.events,
this.queue.toKey(state),
this.queue.keys.wait,
this.queue.keys.meta,
this.queue.keys.paused,
];
const args = [
job.id,
(job.opts.lifo ? 'R' : 'L') + 'PUSH',
state === 'failed' ? 'failedReason' : 'returnvalue',
state,
];
const result = await this.execCommand(client, 'reprocessJob', keys.concat(args));
switch (result) {
case 1:
return;
default:
throw this.finishedErrors(result, job.id, 'reprocessJob', state);
}
}
async moveToActive(client, token, jobId) {
const opts = this.queue.opts;
const queueKeys = this.queue.keys;
const keys = [
queueKeys.wait,
queueKeys.active,
queueKeys.prioritized,
queueKeys.events,
queueKeys.stalled,
queueKeys.limiter,
queueKeys.delayed,
queueKeys.paused,
queueKeys.meta,
queueKeys.pc,
];
const args = [
queueKeys[''],
Date.now(),
jobId || '',
pack({
token,
lockDuration: opts.lockDuration,
limiter: opts.limiter,
}),
];
const result = await this.execCommand(client, 'moveToActive', keys.concat(args));
return raw2NextJobData(result);
}
async promote(jobId) {
const client = await this.queue.client;
const keys = [
this.queue.keys.delayed,
this.queue.keys.wait,
this.queue.keys.paused,
this.queue.keys.meta,
this.queue.keys.prioritized,
this.queue.keys.pc,
this.queue.keys.events,
];
const args = [this.queue.toKey(''), jobId];
const code = await this.execCommand(client, 'promote', keys.concat(args));
if (code < 0) {
throw this.finishedErrors(code, jobId, 'promote', 'delayed');
}
}
/**
* Looks for unlocked jobs in the active queue.
*
* The job was being worked on, but the worker process died and it failed to renew the lock.
* We call these jobs 'stalled'. This is the most common case. We resolve these by moving them
* back to wait to be re-processed. To prevent jobs from cycling endlessly between active and wait,
* (e.g. if the job handler keeps crashing),
* we limit the number stalled job recoveries to settings.maxStalledCount.
*/
async moveStalledJobsToWait() {
const client = await this.queue.client;
const opts = this.queue.opts;
const keys = [
this.queue.keys.stalled,
this.queue.keys.wait,
this.queue.keys.active,
this.queue.keys.failed,
this.queue.keys['stalled-check'],
this.queue.keys.meta,
this.queue.keys.paused,
this.queue.keys.events,
];
const args = [
opts.maxStalledCount,
this.queue.toKey(''),
Date.now(),
opts.stalledInterval,
];
return this.execCommand(client, 'moveStalledJobsToWait', keys.concat(args));
}
/**
* Moves a job back from Active to Wait.
* This script is used when a job has been manually rate limited and needs
* to be moved back to wait from active status.
*
* @param client - Redis client
* @param jobId - Job id
* @returns
*/
async moveJobFromActiveToWait(jobId, token) {
const client = await this.queue.client;
const lockKey = `${this.queue.toKey(jobId)}:lock`;
const keys = [
this.queue.keys.active,
this.queue.keys.wait,
this.queue.keys.stalled,
lockKey,
this.queue.keys.paused,
this.queue.keys.meta,
this.queue.keys.limiter,
this.queue.keys.prioritized,
this.queue.keys.events,
];
const args = [jobId, token, this.queue.toKey(jobId)];
const pttl = await this.execCommand(client, 'moveJobFromActiveToWait', keys.concat(args));
return pttl < 0 ? 0 : pttl;
}
async obliterate(opts) {
const client = await this.queue.client;
const keys = [
this.queue.keys.meta,
this.queue.toKey(''),
];
const args = [opts.count, opts.force ? 'force' : null];
const result = await this.execCommand(client, 'obliterate', keys.concat(args));
if (result < 0) {
switch (result) {
case -1:
throw new Error('Cannot obliterate non-paused queue');
case -2:
throw new Error('Cannot obliterate queue with active jobs');
}
}
return result;
}
/**
* Paginate a set or hash keys.
* @param opts
*
*/
async paginate(key, opts) {
const client = await this.queue.client;
const keys = [key];
const maxIterations = 5;
const pageSize = opts.end >= 0 ? opts.end - opts.start + 1 : Infinity;
let cursor = '0', offset = 0, items, total, rawJobs, page = [], jobs = [];
do {
const args = [
opts.start + page.length,
opts.end,
cursor,
offset,
maxIterations,
];
if (opts.fetchJobs) {
args.push(1);
}
[cursor, offset, items, total, rawJobs] = await this.execCommand(client, 'paginate', keys.concat(args));
page = page.concat(items);
if (rawJobs && rawJobs.length) {
jobs = jobs.concat(rawJobs.map(utils_1.array2obj));
}
// Important to keep this coercive inequality (!=) instead of strict inequality (!==)
} while (cursor != '0' && page.length < pageSize);
// If we get an array of arrays, it means we are paginating a hash
if (page.length && Array.isArray(page[0])) {
const result = [];
for (let index = 0; index < page.length; index++) {
const [id, value] = page[index];
try {
result.push({ id, v: JSON.parse(value) });
}
catch (err) {
result.push({ id, err: err.message });
}
}
return {
cursor,
items: result,
total,
jobs,
};
}
else {
return {
cursor,
items: page.map(item => ({ id: item })),
total,
jobs,
};
}
}
}
exports.Scripts = Scripts;
function raw2NextJobData(raw) {
if (raw) {
const result = [null, raw[1], raw[2], raw[3]];
if (raw[0]) {
result[0] = (0, utils_1.array2obj)(raw[0]);
}
return result;
}
return [];
}
exports.raw2NextJobData = raw2NextJobData;
//# sourceMappingURL=scripts.js.map
File diff suppressed because one or more lines are too long
+634
View File
@@ -0,0 +1,634 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.Worker = void 0;
const fs = require("fs");
const url_1 = require("url");
const path = require("path");
const uuid_1 = require("uuid");
// Note: this Polyfill is only needed for Node versions < 15.4.0
const node_abort_controller_1 = require("node-abort-controller");
const utils_1 = require("../utils");
const queue_base_1 = require("./queue-base");
const repeat_1 = require("./repeat");
const child_pool_1 = require("./child-pool");
const job_1 = require("./job");
const redis_connection_1 = require("./redis-connection");
const sandbox_1 = require("./sandbox");
const async_fifo_queue_1 = require("./async-fifo-queue");
const errors_1 = require("./errors");
// 10 seconds is the maximum time a BRPOPLPUSH can block.
const maximumBlockTimeout = 10;
/**
*
* This class represents a worker that is able to process jobs from the queue.
* As soon as the class is instantiated and a connection to Redis is established
* it will start processing jobs.
*
*/
class Worker extends queue_base_1.QueueBase {
static RateLimitError() {
return new errors_1.RateLimitError();
}
constructor(name, processor, opts = {}, Connection) {
super(name, Object.assign(Object.assign({}, opts), { blockingConnection: true }), Connection);
this.abortDelayController = null;
this.blockUntil = 0;
this.drained = false;
this.extendLocksTimer = null;
this.limitUntil = 0;
this.waiting = null;
this.running = false;
this.opts = Object.assign({ drainDelay: 5, concurrency: 1, lockDuration: 30000, maxStalledCount: 1, stalledInterval: 30000, autorun: true, runRetryDelay: 15000 }, this.opts);
if (this.opts.stalledInterval <= 0) {
throw new Error('stalledInterval must be greater than 0');
}
this.concurrency = this.opts.concurrency;
this.opts.lockRenewTime =
this.opts.lockRenewTime || this.opts.lockDuration / 2;
this.id = (0, uuid_1.v4)();
if (processor) {
if (typeof processor === 'function') {
this.processFn = processor;
}
else {
// SANDBOXED
if (processor instanceof url_1.URL) {
if (!fs.existsSync(processor)) {
throw new Error(`URL ${processor} does not exist in the local file system`);
}
processor = processor.href;
}
else {
const supportedFileTypes = ['.js', '.ts', '.flow', '.cjs'];
const processorFile = processor +
(supportedFileTypes.includes(path.extname(processor)) ? '' : '.js');
if (!fs.existsSync(processorFile)) {
throw new Error(`File ${processorFile} does not exist`);
}
}
const mainFile = this.opts.useWorkerThreads
? 'main-worker.js'
: 'main.js';
let mainFilePath = path.join(path.dirname(module.filename), `${mainFile}`);
try {
fs.statSync(mainFilePath); // would throw if file not exists
}
catch (_) {
mainFilePath = path.join(process.cwd(), `dist/cjs/classes/${mainFile}`);
fs.statSync(mainFilePath);
}
this.childPool = new child_pool_1.ChildPool({
mainFile: mainFilePath,
useWorkerThreads: this.opts.useWorkerThreads,
});
this.processFn = (0, sandbox_1.default)(processor, this.childPool).bind(this);
}
if (this.opts.autorun) {
this.run().catch(error => this.emit('error', error));
}
}
const connectionName = this.clientName(utils_1.WORKER_SUFFIX);
this.blockingConnection = new redis_connection_1.RedisConnection((0, utils_1.isRedisInstance)(opts.connection)
? opts.connection.duplicate({ connectionName })
: Object.assign(Object.assign({}, opts.connection), { connectionName }), false, true, opts.skipVersionCheck);
this.blockingConnection.on('error', error => this.emit('error', error));
this.blockingConnection.on('ready', () => setTimeout(() => this.emit('ready'), 0));
}
emit(event, ...args) {
return super.emit(event, ...args);
}
off(eventName, listener) {
super.off(eventName, listener);
return this;
}
on(event, listener) {
super.on(event, listener);
return this;
}
once(event, listener) {
super.once(event, listener);
return this;
}
callProcessJob(job, token) {
return this.processFn(job, token);
}
createJob(data, jobId) {
return this.Job.fromJSON(this, data, jobId);
}
/**
*
* Waits until the worker is ready to start processing jobs.
* In general only useful when writing tests.
*
*/
async waitUntilReady() {
await super.waitUntilReady();
return this.blockingConnection.client;
}
set concurrency(concurrency) {
if (typeof concurrency !== 'number' ||
concurrency < 1 ||
!isFinite(concurrency)) {
throw new Error('concurrency must be a finite number greater than 0');
}
this.opts.concurrency = concurrency;
}
get repeat() {
return new Promise(async (resolve) => {
if (!this._repeat) {
const connection = await this.client;
this._repeat = new repeat_1.Repeat(this.name, Object.assign(Object.assign({}, this.opts), { connection }));
this._repeat.on('error', e => this.emit.bind(this, e));
}
resolve(this._repeat);
});
}
async run() {
if (!this.processFn) {
throw new Error('No process function is defined.');
}
if (this.running) {
throw new Error('Worker is already running.');
}
try {
this.running = true;
if (this.closing) {
return;
}
await this.startStalledCheckTimer();
const jobsInProgress = new Set();
this.startLockExtenderTimer(jobsInProgress);
const asyncFifoQueue = (this.asyncFifoQueue =
new async_fifo_queue_1.AsyncFifoQueue());
let tokenPostfix = 0;
const client = await this.client;
const bclient = await this.blockingConnection.client;
while (!this.closing) {
let numTotal = asyncFifoQueue.numTotal();
while (!this.waiting &&
numTotal < this.opts.concurrency &&
(!this.limitUntil || numTotal == 0)) {
const token = `${this.id}:${tokenPostfix++}`;
const fetchedJob = this.retryIfFailed(() => this._getNextJob(client, bclient, token, { block: true }), this.opts.runRetryDelay);
asyncFifoQueue.add(fetchedJob);
numTotal = asyncFifoQueue.numTotal();
if (this.waiting && numTotal > 1) {
// We have a job waiting but we have others that we could start processing already
break;
}
// We await here so that we fetch jobs in sequence, this is important to avoid unnecessary calls
// to Redis in high concurrency scenarios.
const job = await fetchedJob;
// No more jobs waiting but we have others that could start processing already
if (!job && numTotal > 1) {
break;
}
// If there are potential jobs to be processed and blockUntil is set, we should exit to avoid waiting
// for processing this job.
if (this.blockUntil) {
break;
}
}
// Since there can be undefined jobs in the queue (when a job fails or queue is empty)
// we iterate until we find a job.
let job;
do {
job = await asyncFifoQueue.fetch();
} while (!job &&
asyncFifoQueue.numTotal() > 0 &&
asyncFifoQueue.numQueued() > 0);
if (job) {
const token = job.token;
asyncFifoQueue.add(this.retryIfFailed(() => this.processJob(job, token, () => asyncFifoQueue.numTotal() <= this.opts.concurrency, jobsInProgress), this.opts.runRetryDelay));
}
}
this.running = false;
return asyncFifoQueue.waitAll();
}
catch (error) {
this.running = false;
throw error;
}
}
/**
* Returns a promise that resolves to the next job in queue.
* @param token - worker token to be assigned to retrieved job
* @returns a Job or undefined if no job was available in the queue.
*/
async getNextJob(token, { block = true } = {}) {
return this._getNextJob(await this.client, await this.blockingConnection.client, token, { block });
}
async _getNextJob(client, bclient, token, { block = true } = {}) {
var _a;
if (this.paused) {
if (block) {
await this.paused;
}
else {
return;
}
}
if (this.closing) {
return;
}
if (this.drained && block && !this.limitUntil && !this.waiting) {
this.waiting = this.waitForJob(bclient);
try {
const jobId = await this.waiting;
return this.moveToActive(client, token, jobId);
}
catch (err) {
// Swallow error if locally paused or closing since we did force a disconnection
if (!(this.paused || this.closing) &&
(0, utils_1.isNotConnectionError)(err)) {
throw err;
}
}
finally {
this.waiting = null;
}
}
else {
if (this.limitUntil) {
(_a = this.abortDelayController) === null || _a === void 0 ? void 0 : _a.abort();
this.abortDelayController = new node_abort_controller_1.AbortController();
await this.delay(this.limitUntil, this.abortDelayController);
}
return this.moveToActive(client, token);
}
}
/**
* Overrides the rate limit to be active for the next jobs.
*
* @param expireTimeMs - expire time in ms of this rate limit.
*/
async rateLimit(expireTimeMs) {
await this.client.then(client => client.set(this.keys.limiter, Number.MAX_SAFE_INTEGER, 'PX', expireTimeMs));
}
async moveToActive(client, token, jobId) {
// If we get the special delayed job ID, we pick the delay as the next
// block timeout.
if (jobId && jobId.startsWith('0:')) {
this.blockUntil = parseInt(jobId.split(':')[1]) || 0;
// Remove marker from active list.
await client.lrem(this.keys.active, 1, jobId);
if (this.blockUntil > 0) {
return;
}
}
const [jobData, id, limitUntil, delayUntil] = await this.scripts.moveToActive(client, token, jobId);
this.updateDelays(limitUntil, delayUntil);
return this.nextJobFromJobData(jobData, id, token);
}
async waitForJob(bclient) {
if (this.paused) {
return;
}
try {
const opts = this.opts;
if (!this.closing) {
let blockTimeout = Math.max(this.blockUntil
? (this.blockUntil - Date.now()) / 1000
: opts.drainDelay, 0);
let jobId;
// Blocking for less than 50ms is useless.
if (blockTimeout > 0.05) {
blockTimeout = this.blockingConnection.capabilities.canDoubleTimeout
? blockTimeout
: Math.ceil(blockTimeout);
// We restrict the maximum block timeout to 10 second to avoid
// blocking the connection for too long in the case of reconnections
// reference: https://github.com/taskforcesh/bullmq/issues/1658
blockTimeout = Math.min(blockTimeout, maximumBlockTimeout);
jobId = await bclient.brpoplpush(this.keys.wait, this.keys.active, blockTimeout);
}
else {
jobId = await bclient.rpoplpush(this.keys.wait, this.keys.active);
}
this.blockUntil = 0;
return jobId;
}
}
catch (error) {
if ((0, utils_1.isNotConnectionError)(error)) {
this.emit('error', error);
}
if (!this.closing) {
await this.delay();
}
}
finally {
this.waiting = null;
}
}
/**
*
* This function is exposed only for testing purposes.
*/
async delay(milliseconds, abortController) {
await (0, utils_1.delay)(milliseconds || utils_1.DELAY_TIME_1, abortController);
}
updateDelays(limitUntil = 0, delayUntil = 0) {
this.limitUntil = Math.max(limitUntil, 0) || 0;
this.blockUntil = Math.max(delayUntil, 0) || 0;
}
async nextJobFromJobData(jobData, jobId, token) {
if (!jobData) {
if (!this.drained) {
this.emit('drained');
this.drained = true;
}
}
else {
this.drained = false;
const job = this.createJob(jobData, jobId);
job.token = token;
if (job.opts.repeat) {
const repeat = await this.repeat;
await repeat.addNextRepeatableJob(job.name, job.data, job.opts);
}
return job;
}
}
async processJob(job, token, fetchNextCallback = () => true, jobsInProgress) {
if (!job || this.closing || this.paused) {
return;
}
const handleCompleted = async (result) => {
if (!this.connection.closing) {
const completed = await job.moveToCompleted(result, token, fetchNextCallback() && !(this.closing || this.paused));
this.emit('completed', job, result, 'active');
const [jobData, jobId, limitUntil, delayUntil] = completed || [];
this.updateDelays(limitUntil, delayUntil);
return this.nextJobFromJobData(jobData, jobId, token);
}
};
const handleFailed = async (err) => {
if (!this.connection.closing) {
try {
if (err.message == errors_1.RATE_LIMIT_ERROR) {
this.limitUntil = await this.moveLimitedBackToWait(job, token);
return;
}
if (err instanceof errors_1.DelayedError ||
err.message == 'DelayedError' ||
err instanceof errors_1.WaitingChildrenError ||
err.name == 'WaitingChildrenError') {
return;
}
await job.moveToFailed(err, token);
this.emit('failed', job, err, 'active');
}
catch (err) {
this.emit('error', err);
// It probably means that the job has lost the lock before completion
// A worker will (or already has) moved the job back
// to the waiting list (as stalled)
}
}
};
this.emit('active', job, 'waiting');
const inProgressItem = { job, ts: Date.now() };
try {
jobsInProgress.add(inProgressItem);
const result = await this.callProcessJob(job, token);
return await handleCompleted(result);
}
catch (err) {
return handleFailed(err);
}
finally {
jobsInProgress.delete(inProgressItem);
}
}
/**
*
* Pauses the processing of this queue only for this worker.
*/
async pause(doNotWaitActive) {
if (!this.paused) {
this.paused = new Promise(resolve => {
this.resumeWorker = function () {
resolve();
this.paused = null; // Allow pause to be checked externally for paused state.
this.resumeWorker = null;
};
});
await (!doNotWaitActive && this.whenCurrentJobsFinished());
this.emit('paused');
}
}
/**
*
* Resumes processing of this worker (if paused).
*/
resume() {
if (this.resumeWorker) {
this.resumeWorker();
this.emit('resumed');
}
}
/**
*
* Checks if worker is paused.
*
* @returns true if worker is paused, false otherwise.
*/
isPaused() {
return !!this.paused;
}
/**
*
* Checks if worker is currently running.
*
* @returns true if worker is running, false otherwise.
*/
isRunning() {
return this.running;
}
/**
*
* Closes the worker and related redis connections.
*
* This method waits for current jobs to finalize before returning.
*
* @param force - Use force boolean parameter if you do not want to wait for
* current jobs to be processed.
*
* @returns Promise that resolves when the worker has been closed.
*/
close(force = false) {
if (this.closing) {
return this.closing;
}
this.closing = (async () => {
var _a;
this.emit('closing', 'closing queue');
(_a = this.abortDelayController) === null || _a === void 0 ? void 0 : _a.abort();
const client = await this.blockingConnection.client;
this.resume();
await Promise.resolve()
.finally(() => {
return force || this.whenCurrentJobsFinished(false);
})
.finally(() => {
var _a;
const closePoolPromise = (_a = this.childPool) === null || _a === void 0 ? void 0 : _a.clean();
if (force) {
// since we're not waiting for the job to end attach
// an error handler to avoid crashing the whole process
closePoolPromise === null || closePoolPromise === void 0 ? void 0 : closePoolPromise.catch(err => {
console.error(err); // TODO: emit error in next breaking change version
});
return;
}
return closePoolPromise;
})
.finally(() => clearTimeout(this.extendLocksTimer))
.finally(() => clearTimeout(this.stalledCheckTimer))
.finally(() => client.disconnect())
.finally(() => this.connection.close())
.finally(() => this.emit('closed'));
this.closed = true;
})();
return this.closing;
}
/**
*
* Manually starts the stalled checker.
* The check will run once as soon as this method is called, and
* then every opts.stalledInterval milliseconds until the worker is closed.
* Note: Normally you do not need to call this method, since the stalled checker
* is automatically started when the worker starts processing jobs after
* calling run. However if you want to process the jobs manually you need
* to call this method to start the stalled checker.
*
* @see {@link https://docs.bullmq.io/patterns/manually-fetching-jobs}
*/
async startStalledCheckTimer() {
if (!this.opts.skipStalledCheck) {
clearTimeout(this.stalledCheckTimer);
if (!this.closing) {
try {
await this.checkConnectionError(() => this.moveStalledJobsToWait());
this.stalledCheckTimer = setTimeout(async () => {
await this.startStalledCheckTimer();
}, this.opts.stalledInterval);
}
catch (err) {
this.emit('error', err);
}
}
}
}
startLockExtenderTimer(jobsInProgress) {
if (!this.opts.skipLockRenewal) {
clearTimeout(this.extendLocksTimer);
if (!this.closed) {
this.extendLocksTimer = setTimeout(async () => {
// Get all the jobs whose locks expire in less than 1/2 of the lockRenewTime
const now = Date.now();
const jobsToExtend = [];
for (const item of jobsInProgress) {
const { job, ts } = item;
if (!ts) {
item.ts = now;
continue;
}
if (ts + this.opts.lockRenewTime / 2 < now) {
item.ts = now;
jobsToExtend.push(job);
}
}
try {
if (jobsToExtend.length) {
await this.extendLocks(jobsToExtend);
}
}
catch (err) {
this.emit('error', err);
}
this.startLockExtenderTimer(jobsInProgress);
}, this.opts.lockRenewTime / 2);
}
}
}
/**
* Returns a promise that resolves when active jobs are cleared
*
* @returns
*/
async whenCurrentJobsFinished(reconnect = true) {
//
// Force reconnection of blocking connection to abort blocking redis call immediately.
//
if (this.waiting) {
// If we are not going to reconnect, we will not wait for the disconnection.
await this.blockingConnection.disconnect(reconnect);
}
else {
reconnect = false;
}
if (this.asyncFifoQueue) {
await this.asyncFifoQueue.waitAll();
}
reconnect && (await this.blockingConnection.reconnect());
}
async retryIfFailed(fn, delayInMs) {
const retry = 1;
do {
try {
return await fn();
}
catch (err) {
this.emit('error', err);
if (delayInMs) {
await this.delay(delayInMs);
}
else {
return;
}
}
} while (retry);
}
async extendLocks(jobs) {
try {
const multi = (await this.client).multi();
for (const job of jobs) {
await this.scripts.extendLock(job.id, job.token, this.opts.lockDuration, multi);
}
const result = (await multi.exec());
for (const [err, jobId] of result) {
if (err) {
// TODO: signal process function that the job has been lost.
this.emit('error', new Error(`could not renew lock for job ${jobId}`));
}
}
}
catch (err) {
this.emit('error', err);
}
}
async moveStalledJobsToWait() {
const chunkSize = 50;
const [failed, stalled] = await this.scripts.moveStalledJobsToWait();
stalled.forEach((jobId) => this.emit('stalled', jobId, 'active'));
const jobPromises = [];
for (let i = 0; i < failed.length; i++) {
jobPromises.push(job_1.Job.fromId(this, failed[i]));
if ((i + 1) % chunkSize === 0) {
this.notifyFailedJobs(await Promise.all(jobPromises));
jobPromises.length = 0;
}
}
this.notifyFailedJobs(await Promise.all(jobPromises));
}
notifyFailedJobs(failedJobs) {
failedJobs.forEach((job) => this.emit('failed', job, new Error('job stalled more than allowable limit'), 'active'));
}
moveLimitedBackToWait(job, token) {
return this.scripts.moveJobFromActiveToWait(job.id, token);
}
}
exports.Worker = Worker;
//# sourceMappingURL=worker.js.map
File diff suppressed because one or more lines are too long
+124
View File
@@ -0,0 +1,124 @@
--[[
Adds a delayed job to the queue by doing the following:
- Increases the job counter if needed.
- Creates a new job key with the job data.
- computes timestamp.
- adds to delayed zset.
- Emits a global event 'delayed' if the job is delayed.
Input:
KEYS[1] 'wait',
KEYS[2] 'paused'
KEYS[3] 'meta'
KEYS[4] 'id'
KEYS[5] 'delayed'
KEYS[6] 'completed'
KEYS[7] events stream key
ARGV[1] msgpacked arguments array
[1] key prefix,
[2] custom id (use custom instead of one generated automatically)
[3] name
[4] timestamp
[5] parentKey?
x [6] waitChildrenKey key.
[7] parent dependencies key.
[8] parent? {id, queueKey}
[9] repeat job key
ARGV[2] Json stringified job data
ARGV[3] msgpacked options
Output:
jobId - OK
-5 - Missing parent key
]]
local waitKey = KEYS[1]
local pausedKey = KEYS[2]
local metaKey = KEYS[3]
local idKey = KEYS[4]
local delayedKey = KEYS[5]
local completedKey = KEYS[6]
local eventsKey = KEYS[7]
local jobId
local jobIdKey
local rcall = redis.call
local args = cmsgpack.unpack(ARGV[1])
local data = ARGV[2]
local opts = cmsgpack.unpack(ARGV[3])
local parentKey = args[5]
local repeatJobKey = args[9]
local parent = args[8]
local parentData
-- Includes
--- @include "includes/storeJob"
--- @include "includes/addDelayMarkerIfNeeded"
--- @include "includes/getTargetQueueList"
--- @include "includes/getNextDelayedTimestamp"
--- @include "includes/updateExistingJobsParent"
--- @include "includes/getOrSetMaxEvents"
if parentKey ~= nil then
if rcall("EXISTS", parentKey) ~= 1 then return -5 end
parentData = cjson.encode(parent)
end
local jobCounter = rcall("INCR", idKey)
local maxEvents = getOrSetMaxEvents(metaKey)
local parentDependenciesKey = args[7]
local timestamp = args[4]
if args[2] == "" then
jobId = jobCounter
jobIdKey = args[1] .. jobId
else
-- Refactor to: handleDuplicateJob.lua
jobId = args[2]
jobIdKey = args[1] .. jobId
if rcall("EXISTS", jobIdKey) == 1 then
updateExistingJobsParent(parentKey, parent, parentData,
parentDependenciesKey, completedKey, jobIdKey,
jobId, timestamp)
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
"duplicated", "jobId", jobId)
return jobId .. "" -- convert to string
end
end
-- Store the job.
local delay, priority = storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2],
opts, timestamp, parentKey, parentData,
repeatJobKey)
-- Compute delayed timestamp and the score.
local delayedTimestamp = (delay > 0 and (timestamp + delay)) or 0
local score = delayedTimestamp * 0x1000 + bit.band(jobCounter, 0xfff)
rcall("ZADD", delayedKey, score, jobId)
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
"jobId", jobId, "delay", delayedTimestamp)
-- If wait list is empty, and this delayed job is the next one to be processed,
-- then we need to signal the workers by adding a dummy job (jobId 0:delay) to the wait list.
local target = getTargetQueueList(metaKey, KEYS[1], KEYS[2])
addDelayMarkerIfNeeded(target, delayedKey)
-- Check if this job is a child of another job, if so add it to the parents dependencies
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
-- fail in this case.
if parentDependenciesKey ~= nil then
rcall("SADD", parentDependenciesKey, jobIdKey)
end
return jobId .. "" -- convert to string
+101
View File
@@ -0,0 +1,101 @@
--[[
Adds a parent job to the queue by doing the following:
- Increases the job counter if needed.
- Creates a new job key with the job data.
- adds the job to the waiting-children zset
Input:
KEYS[1] 'meta'
KEYS[2] 'id'
KEYS[3] 'completed'
KEYS[4] events stream key
ARGV[1] msgpacked arguments array
[1] key prefix,
[2] custom id (will not generate one automatically)
[3] name
[4] timestamp
[5] parentKey?
[6] waitChildrenKey key.
[7] parent dependencies key.
[8] parent? {id, queueKey}
[9] repeat job key
ARGV[2] Json stringified job data
ARGV[3] msgpacked options
Output:
jobId - OK
-5 - Missing parent key
]]
local metaKey = KEYS[1]
local idKey = KEYS[2]
local completedKey = KEYS[3]
local eventsKey = KEYS[4]
local jobId
local jobIdKey
local rcall = redis.call
local args = cmsgpack.unpack(ARGV[1])
local data = ARGV[2]
local opts = cmsgpack.unpack(ARGV[3])
local parentKey = args[5]
local repeatJobKey = args[9]
local parent = args[8]
local parentData
-- Includes
--- @include "includes/storeJob"
--- @include "includes/updateExistingJobsParent"
--- @include "includes/getOrSetMaxEvents"
if parentKey ~= nil then
if rcall("EXISTS", parentKey) ~= 1 then return -5 end
parentData = cjson.encode(parent)
end
local jobCounter = rcall("INCR", idKey)
local maxEvents = getOrSetMaxEvents(metaKey)
local parentDependenciesKey = args[7]
local timestamp = args[4]
if args[2] == "" then
jobId = jobCounter
jobIdKey = args[1] .. jobId
else
jobId = args[2]
jobIdKey = args[1] .. jobId
if rcall("EXISTS", jobIdKey) == 1 then
updateExistingJobsParent(parentKey, parent, parentData,
parentDependenciesKey, completedKey, jobIdKey,
jobId, timestamp)
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
"duplicated", "jobId", jobId)
return jobId .. "" -- convert to string
end
end
-- Store the job.
storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2], opts, timestamp,
parentKey, parentData, repeatJobKey)
local waitChildrenKey = args[6]
rcall("ZADD", waitChildrenKey, timestamp, jobId)
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
"waiting-children", "jobId", jobId)
-- Check if this job is a child of another job, if so add it to the parents dependencies
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
-- fail in this case.
if parentDependenciesKey ~= nil then
rcall("SADD", parentDependenciesKey, jobIdKey)
end
return jobId .. "" -- convert to string
+117
View File
@@ -0,0 +1,117 @@
--[[
Adds a priotitized job to the queue by doing the following:
- Increases the job counter if needed.
- Creates a new job key with the job data.
- Adds the job to the "added" list so that workers gets notified.
Input:
KEYS[1] 'wait',
KEYS[2] 'paused'
KEYS[3] 'meta'
KEYS[4] 'id'
KEYS[5] 'prioritized'
KEYS[6] 'completed'
KEYS[7] events stream key
KEYS[8] 'pc' priority counter
ARGV[1] msgpacked arguments array
[1] key prefix,
[2] custom id (will not generate one automatically)
[3] name
[4] timestamp
[5] parentKey?
[6] waitChildrenKey key.
[7] parent dependencies key.
[8] parent? {id, queueKey}
[9] repeat job key
ARGV[2] Json stringified job data
ARGV[3] msgpacked options
Output:
jobId - OK
-5 - Missing parent key
]]
local waitKey = KEYS[1]
local pausedKey = KEYS[2]
local metaKey = KEYS[3]
local idKey = KEYS[4]
local priorityKey = KEYS[5]
local completedKey = KEYS[6]
local eventsKey = KEYS[7]
local priorityCounterKey = KEYS[8]
local jobId
local jobIdKey
local rcall = redis.call
local args = cmsgpack.unpack(ARGV[1])
local data = ARGV[2]
local opts = cmsgpack.unpack(ARGV[3])
local parentKey = args[5]
local repeatJobKey = args[9]
local parent = args[8]
local parentData
-- Includes
--- @include "includes/storeJob"
--- @include "includes/addJobWithPriority"
--- @include "includes/getTargetQueueList"
--- @include "includes/updateExistingJobsParent"
--- @include "includes/getOrSetMaxEvents"
if parentKey ~= nil then
if rcall("EXISTS", parentKey) ~= 1 then return -5 end
parentData = cjson.encode(parent)
end
local jobCounter = rcall("INCR", idKey)
local maxEvents = getOrSetMaxEvents(metaKey)
local parentDependenciesKey = args[7]
local timestamp = args[4]
if args[2] == "" then
jobId = jobCounter
jobIdKey = args[1] .. jobId
else
jobId = args[2]
jobIdKey = args[1] .. jobId
if rcall("EXISTS", jobIdKey) == 1 then
updateExistingJobsParent(parentKey, parent, parentData,
parentDependenciesKey, completedKey, jobIdKey,
jobId, timestamp)
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
"duplicated", "jobId", jobId)
return jobId .. "" -- convert to string
end
end
-- Store the job.
local delay, priority = storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2],
opts, timestamp, parentKey, parentData,
repeatJobKey)
local target, paused = getTargetQueueList(metaKey, waitKey, pausedKey)
addJobWithPriority(waitKey, priorityKey, priority, paused, jobId,
priorityCounterKey)
-- Emit waiting event
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting",
"jobId", jobId)
-- Check if this job is a child of another job, if so add it to the parents dependencies
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
-- fail in this case.
if parentDependenciesKey ~= nil then
rcall("SADD", parentDependenciesKey, jobIdKey)
end
return jobId .. "" -- convert to string
+117
View File
@@ -0,0 +1,117 @@
--[[
Adds a job to the queue by doing the following:
- Increases the job counter if needed.
- Creates a new job key with the job data.
- if delayed:
- computes timestamp.
- adds to delayed zset.
- Emits a global event 'delayed' if the job is delayed.
- if not delayed
- Adds the jobId to the wait/paused list in one of three ways:
- LIFO
- FIFO
- prioritized.
- Adds the job to the "added" list so that workers gets notified.
Input:
KEYS[1] 'wait',
KEYS[2] 'paused'
KEYS[3] 'meta'
KEYS[4] 'id'
KEYS[5] 'completed'
KEYS[6] events stream key
ARGV[1] msgpacked arguments array
[1] key prefix,
[2] custom id (will not generate one automatically)
[3] name
[4] timestamp
[5] parentKey?
[6] waitChildrenKey key.
[7] parent dependencies key.
[8] parent? {id, queueKey}
[9] repeat job key
ARGV[2] Json stringified job data
ARGV[3] msgpacked options
Output:
jobId - OK
-5 - Missing parent key
]]
local eventsKey = KEYS[6]
local jobId
local jobIdKey
local rcall = redis.call
local args = cmsgpack.unpack(ARGV[1])
local data = ARGV[2]
local opts = cmsgpack.unpack(ARGV[3])
local parentKey = args[5]
local repeatJobKey = args[9]
local parent = args[8]
local parentData
-- Includes
--- @include "includes/storeJob"
--- @include "includes/updateExistingJobsParent"
--- @include "includes/getTargetQueueList"
--- @include "includes/getOrSetMaxEvents"
if parentKey ~= nil then
if rcall("EXISTS", parentKey) ~= 1 then return -5 end
parentData = cjson.encode(parent)
end
local jobCounter = rcall("INCR", KEYS[4])
local metaKey = KEYS[3]
local maxEvents = getOrSetMaxEvents(metaKey)
local parentDependenciesKey = args[7]
local timestamp = args[4]
if args[2] == "" then
jobId = jobCounter
jobIdKey = args[1] .. jobId
else
jobId = args[2]
jobIdKey = args[1] .. jobId
if rcall("EXISTS", jobIdKey) == 1 then
updateExistingJobsParent(parentKey, parent, parentData,
parentDependenciesKey, KEYS[5], jobIdKey,
jobId, timestamp)
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
"duplicated", "jobId", jobId)
return jobId .. "" -- convert to string
end
end
-- Store the job.
storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2], opts, timestamp,
parentKey, parentData, repeatJobKey)
local target, paused = getTargetQueueList(metaKey, KEYS[1], KEYS[2])
-- LIFO or FIFO
local pushCmd = opts['lifo'] and 'RPUSH' or 'LPUSH'
rcall(pushCmd, target, jobId)
-- Emit waiting event
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting",
"jobId", jobId)
-- Check if this job is a child of another job, if so add it to the parents dependencies
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
-- fail in this case.
if parentDependenciesKey ~= nil then
rcall("SADD", parentDependenciesKey, jobIdKey)
end
return jobId .. "" -- convert to string
+40
View File
@@ -0,0 +1,40 @@
--[[
Change job delay when it is in delayed set.
Input:
KEYS[1] delayed key
KEYS[2] job key
KEYS[3] events stream
ARGV[1] delay
ARGV[2] delayedTimestamp
ARGV[3] the id of the job
Output:
0 - OK
-1 - Missing job.
-3 - Job not in delayed set.
Events:
- delayed key.
]]
local rcall = redis.call
if rcall("EXISTS", KEYS[2]) == 1 then
local jobId = ARGV[3]
local score = tonumber(ARGV[2])
local delayedTimestamp = (score / 0x1000)
local numRemovedElements = rcall("ZREM", KEYS[1], jobId)
if numRemovedElements < 1 then
return -3
end
rcall("HSET", KEYS[2], "delay", tonumber(ARGV[1]))
rcall("ZADD", KEYS[1], score, jobId)
rcall("XADD", KEYS[3], "*", "event", "delayed", "jobId", jobId, "delay", delayedTimestamp)
return 0
else
return -1
end
+52
View File
@@ -0,0 +1,52 @@
--[[
Change job priority
Input:
KEYS[1] 'wait',
KEYS[2] 'paused'
KEYS[3] 'meta'
KEYS[4] 'prioritized'
KEYS[5] 'pc' priority counter
ARGV[1] priority value
ARGV[2] job key
ARGV[3] job id
ARGV[4] lifo
Output:
0 - OK
-1 - Missing job
]]
local jobKey = ARGV[2]
local jobId = ARGV[3]
local priority = tonumber(ARGV[1])
local rcall = redis.call
-- Includes
--- @include "includes/addJobWithPriority"
--- @include "includes/getTargetQueueList"
if rcall("EXISTS", jobKey) == 1 then
local target, paused = getTargetQueueList(KEYS[3], KEYS[1], KEYS[2])
if rcall("ZREM", KEYS[4], jobId) > 0 then
addJobWithPriority(KEYS[1], KEYS[4], priority, paused, jobId, KEYS[5])
else
local numRemovedElements = rcall("LREM", target, -1, jobId)
if numRemovedElements > 0 then
-- Standard or priority add
if priority == 0 then
-- LIFO or FIFO
local pushCmd = ARGV[4] == '1' and 'RPUSH' or 'LPUSH';
rcall(pushCmd, target, jobId)
else
addJobWithPriority(KEYS[1], KEYS[4], priority, paused, jobId, KEYS[5])
end
end
end
rcall("HSET", jobKey, "priority", priority)
return 0
else
return -1
end
+54
View File
@@ -0,0 +1,54 @@
--[[
Remove jobs from the specific set.
Input:
KEYS[1] set key,
KEYS[2] events stream key
ARGV[1] jobKey prefix
ARGV[2] timestamp
ARGV[3] limit the number of jobs to be removed. 0 is unlimited
ARGV[4] set name, can be any of 'wait', 'active', 'paused', 'delayed', 'completed', or 'failed'
]]
local rcall = redis.call
local rangeStart = 0
local rangeEnd = -1
local limit = tonumber(ARGV[3])
-- If we're only deleting _n_ items, avoid retrieving all items
-- for faster performance
--
-- Start from the tail of the list, since that's where oldest elements
-- are generally added for FIFO lists
if limit > 0 then
rangeStart = -1 - limit + 1
rangeEnd = -1
end
-- Includes
--- @include "includes/cleanList"
--- @include "includes/cleanSet"
local result
if ARGV[4] == "active" then
result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], false)
elseif ARGV[4] == "delayed" then
rangeEnd = "+inf"
result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
{"processedOn", "timestamp"}, false)
elseif ARGV[4] == "prioritized" then
rangeEnd = "+inf"
result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
{"timestamp"}, false)
elseif ARGV[4] == "wait" or ARGV[4] == "paused" then
result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], true)
else
rangeEnd = ARGV[2]
result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
{"finishedOn"}, true)
end
rcall("XADD", KEYS[2], "*", "event", "cleaned", "count", result[2])
return result[1]
+26
View File
@@ -0,0 +1,26 @@
--[[
Drains the queue, removes all jobs that are waiting
or delayed, but not active, completed or failed
Input:
KEYS[1] 'wait',
KEYS[2] 'paused'
KEYS[3] 'delayed'
KEYS[4] 'prioritized'
ARGV[1] queue key prefix
]]
local rcall = redis.call
local queueBaseKey = ARGV[1]
--- @include "includes/removeListJobs"
--- @include "includes/removeZSetJobs"
removeListJobs(KEYS[1], true, queueBaseKey, 0) --wait
removeListJobs(KEYS[2], true, queueBaseKey, 0) --paused
if KEYS[3] ~= "" then
removeZSetJobs(KEYS[3], true, queueBaseKey, 0) --delayed
end
removeZSetJobs(KEYS[4], true, queueBaseKey, 0) --prioritized
+23
View File
@@ -0,0 +1,23 @@
--[[
Extend lock and removes the job from the stalled set.
Input:
KEYS[1] 'lock',
KEYS[2] 'stalled'
ARGV[1] token
ARGV[2] lock duration in milliseconds
ARGV[3] jobid
Output:
"1" if lock extented succesfully.
]]
local rcall = redis.call
if rcall("GET", KEYS[1]) == ARGV[1] then
-- if rcall("SET", KEYS[1], ARGV[1], "PX", ARGV[2], "XX") then
if rcall("SET", KEYS[1], ARGV[1], "PX", ARGV[2]) then
rcall("SREM", KEYS[2], ARGV[3])
return 1
end
end
return 0
+35
View File
@@ -0,0 +1,35 @@
--[[
Get counts per provided states
Input:
KEYS[1] 'prefix'
ARGV[1...] types
]]
local rcall = redis.call;
local prefix = KEYS[1]
local results = {}
for i = 1, #ARGV do
local stateKey = prefix .. ARGV[i]
if ARGV[i] == "wait" or ARGV[i] == "paused" then
local marker = rcall("LINDEX", stateKey, -1)
if marker and string.sub(marker, 1, 2) == "0:" then
local count = rcall("LLEN", stateKey)
if count > 1 then
rcall("RPOP", stateKey)
results[#results+1] = count-1
else
results[#results+1] = 0
end
else
results[#results+1] = rcall("LLEN", stateKey)
end
elseif ARGV[i] == "active" then
results[#results+1] = rcall("LLEN", stateKey)
else
results[#results+1] = rcall("ZCARD", stateKey)
end
end
return results
+69
View File
@@ -0,0 +1,69 @@
--[[
Get job ids per provided states
Input:
KEYS[1] 'prefix'
ARGV[1] start
ARGV[2] end
ARGV[3] asc
ARGV[4...] types
]]
local rcall = redis.call
local prefix = KEYS[1]
local rangeStart = tonumber(ARGV[1])
local rangeEnd = tonumber(ARGV[2])
local asc = ARGV[3]
local results = {}
local function getRangeInList(listKey, asc, rangeStart, rangeEnd, results)
if asc == "1" then
local modifiedRangeStart
local modifiedRangeEnd
if rangeStart == -1 then
modifiedRangeStart = 0
else
modifiedRangeStart = -(rangeStart + 1)
end
if rangeEnd == -1 then
modifiedRangeEnd = 0
else
modifiedRangeEnd = -(rangeEnd + 1)
end
results[#results+1] = rcall("LRANGE", listKey,
modifiedRangeEnd,
modifiedRangeStart)
else
results[#results+1] = rcall("LRANGE", listKey, rangeStart, rangeEnd)
end
end
for i = 4, #ARGV do
local stateKey = prefix .. ARGV[i]
if ARGV[i] == "wait" or ARGV[i] == "paused" then
local marker = rcall("LINDEX", stateKey, -1)
if marker and string.sub(marker, 1, 2) == "0:" then
local count = rcall("LLEN", stateKey)
if count > 1 then
rcall("RPOP", stateKey)
getRangeInList(stateKey, asc, rangeStart, rangeEnd, results)
else
results[#results+1] = {}
end
else
getRangeInList(stateKey, asc, rangeStart, rangeEnd, results)
end
elseif ARGV[i] == "active" then
getRangeInList(stateKey, asc, rangeStart, rangeEnd, results)
else
if asc == "1" then
results[#results+1] = rcall("ZRANGE", stateKey, rangeStart, rangeEnd)
else
results[#results+1] = rcall("ZREVRANGE", stateKey, rangeStart, rangeEnd)
end
end
end
return results
+65
View File
@@ -0,0 +1,65 @@
--[[
Get a job state
Input:
KEYS[1] 'completed' key,
KEYS[2] 'failed' key
KEYS[3] 'delayed' key
KEYS[4] 'active' key
KEYS[5] 'wait' key
KEYS[6] 'paused' key
KEYS[7] 'waiting-children' key
KEYS[8] 'prioritized' key
ARGV[1] job id
Output:
'completed'
'failed'
'delayed'
'active'
'prioritized'
'waiting'
'waiting-children'
'unknown'
]]
local rcall = redis.call
if rcall("ZSCORE", KEYS[1], ARGV[1]) ~= false then
return "completed"
end
if rcall("ZSCORE", KEYS[2], ARGV[1]) ~= false then
return "failed"
end
if rcall("ZSCORE", KEYS[3], ARGV[1]) ~= false then
return "delayed"
end
if rcall("ZSCORE", KEYS[8], ARGV[1]) ~= false then
return "prioritized"
end
-- Includes
--- @include "includes/checkItemInList"
local active_items = rcall("LRANGE", KEYS[4] , 0, -1)
if checkItemInList(active_items, ARGV[1]) ~= nil then
return "active"
end
local wait_items = rcall("LRANGE", KEYS[5] , 0, -1)
if checkItemInList(wait_items, ARGV[1]) ~= nil then
return "waiting"
end
local paused_items = rcall("LRANGE", KEYS[6] , 0, -1)
if checkItemInList(paused_items, ARGV[1]) ~= nil then
return "waiting"
end
if rcall("ZSCORE", KEYS[7], ARGV[1]) ~= false then
return "waiting-children"
end
return "unknown"
+58
View File
@@ -0,0 +1,58 @@
--[[
Get a job state
Input:
KEYS[1] 'completed' key,
KEYS[2] 'failed' key
KEYS[3] 'delayed' key
KEYS[4] 'active' key
KEYS[5] 'wait' key
KEYS[6] 'paused' key
KEYS[7] 'waiting-children' key
KEYS[8] 'prioritized' key
ARGV[1] job id
Output:
'completed'
'failed'
'delayed'
'active'
'waiting'
'waiting-children'
'unknown'
]]
local rcall = redis.call
if rcall("ZSCORE", KEYS[1], ARGV[1]) ~= false then
return "completed"
end
if rcall("ZSCORE", KEYS[2], ARGV[1]) ~= false then
return "failed"
end
if rcall("ZSCORE", KEYS[3], ARGV[1]) ~= false then
return "delayed"
end
if rcall("ZSCORE", KEYS[8], ARGV[1]) ~= false then
return "prioritized"
end
if rcall("LPOS", KEYS[4] , ARGV[1]) ~= false then
return "active"
end
if rcall("LPOS", KEYS[5] , ARGV[1]) ~= false then
return "waiting"
end
if rcall("LPOS", KEYS[6] , ARGV[1]) ~= false then
return "waiting"
end
if rcall("ZSCORE", KEYS[7] , ARGV[1]) ~= false then
return "waiting-children"
end
return "unknown"
@@ -0,0 +1,27 @@
--[[
Add delay marker if needed.
]]
-- Includes
--- @include "getNextDelayedTimestamp"
local function addDelayMarkerIfNeeded(targetKey, delayedKey)
local waitLen = rcall("LLEN", targetKey)
if waitLen <= 1 then
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
if nextTimestamp ~= nil then
-- Check if there is already a marker with older timestamp
-- if there is, we need to replace it.
if waitLen == 1 then
local marker = rcall("LINDEX", targetKey, 0)
local oldTimestamp = tonumber(marker:sub(3))
if oldTimestamp and oldTimestamp > nextTimestamp then
rcall("LSET", targetKey, 0, "0:" .. nextTimestamp)
end
else
-- if there is no marker, then we need to add one
rcall("LPUSH", targetKey, "0:" .. nextTimestamp)
end
end
end
end
+15
View File
@@ -0,0 +1,15 @@
--[[
Function to add job considering priority.
]]
-- Includes
--- @include "addPriorityMarkerIfNeeded"
--- @include "getPriorityScore"
local function addJobWithPriority(waitKey, prioritizedKey, priority, paused, jobId, priorityCounterKey)
local score = getPriorityScore(priority, priorityCounterKey)
rcall("ZADD", prioritizedKey, score, jobId)
if not paused then
addPriorityMarkerIfNeeded(waitKey)
end
end
@@ -0,0 +1,12 @@
--[[
Function priority marker to wait if needed
in order to wake up our workers and to respect priority
order as much as possible
]]
local function addPriorityMarkerIfNeeded(waitKey)
local waitLen = rcall("LLEN", waitKey)
if waitLen == 0 then
rcall("LPUSH", waitKey, "0:0")
end
end
+18
View File
@@ -0,0 +1,18 @@
--[[
Function to loop in batches.
Just a bit of warning, some commands as ZREM
could receive a maximum of 7000 parameters per call.
]]
local function batches(n, batchSize)
local i = 0
return function()
local from = i * batchSize + 1
i = i + 1
if (from <= n) then
local to = math.min(from + batchSize - 1, n)
return from, to
end
end
end
+12
View File
@@ -0,0 +1,12 @@
--[[
Functions to check if a item belongs to a list.
]]
local function checkItemInList(list, item)
for _, v in pairs(list) do
if v == item then
return 1
end
end
return nil
end
+139
View File
@@ -0,0 +1,139 @@
--[[
Move stalled jobs to wait.
Input:
stalledKey 'stalled' (SET)
waitKey 'wait', (LIST)
activeKey 'active', (LIST)
failedKey 'failed', (ZSET)
stalledCheckKey 'stalled-check', (KEY)
metaKey 'meta', (KEY)
pausedKey 'paused', (LIST)
eventStreamKey 'event stream' (STREAM)
maxStalledJobCount Max stalled job count
queueKeyPrefix queue.toKey('')
timestamp timestamp
maxCheckTime max check time
Events:
'stalled' with stalled job id.
]]
local rcall = redis.call
-- Includes
--- @include "batches"
--- @include "getTargetQueueList"
--- @include "removeJob"
--- @include "removeJobsByMaxAge"
--- @include "removeJobsByMaxCount"
--- @include "trimEvents"
-- Check if we need to check for stalled jobs now.
local function checkStalledJobs(stalledKey, waitKey, activeKey, failedKey,
stalledCheckKey, metaKey, pausedKey,
eventStreamKey, maxStalledJobCount,
queueKeyPrefix, timestamp, maxCheckTime)
if rcall("EXISTS", stalledCheckKey) == 1 then return {{}, {}} end
rcall("SET", stalledCheckKey, timestamp, "PX", maxCheckTime)
-- Trim events before emiting them to avoid trimming events emitted in this script
trimEvents(metaKey, eventStreamKey)
-- Move all stalled jobs to wait
local stalling = rcall('SMEMBERS', stalledKey)
local stalled = {}
local failed = {}
if (#stalling > 0) then
rcall('DEL', stalledKey)
local MAX_STALLED_JOB_COUNT = tonumber(maxStalledJobCount)
-- Remove from active list
for i, jobId in ipairs(stalling) do
if string.sub(jobId, 1, 2) == "0:" then
-- If the jobId is a delay marker ID we just remove it.
rcall("LREM", activeKey, 1, jobId)
else
local jobKey = queueKeyPrefix .. jobId
-- Check that the lock is also missing, then we can handle this job as really stalled.
if (rcall("EXISTS", jobKey .. ":lock") == 0) then
-- Remove from the active queue.
local removed = rcall("LREM", activeKey, 1, jobId)
if (removed > 0) then
-- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
local stalledCount =
rcall("HINCRBY", jobKey, "stalledCounter", 1)
if (stalledCount > MAX_STALLED_JOB_COUNT) then
local rawOpts = rcall("HGET", jobKey, "opts")
local opts = cjson.decode(rawOpts)
local removeOnFailType = type(opts["removeOnFail"])
rcall("ZADD", failedKey, timestamp, jobId)
local failedReason =
"job stalled more than allowable limit"
rcall("HMSET", jobKey, "failedReason", failedReason,
"finishedOn", timestamp)
rcall("XADD", eventStreamKey, "*", "event",
"failed", "jobId", jobId, 'prev', 'active',
'failedReason', failedReason)
if removeOnFailType == "number" then
removeJobsByMaxCount(opts["removeOnFail"],
failedKey, queueKeyPrefix)
elseif removeOnFailType == "boolean" then
if opts["removeOnFail"] then
removeJob(jobId, false, queueKeyPrefix)
rcall("ZREM", failedKey, jobId)
end
elseif removeOnFailType ~= "nil" then
local maxAge = opts["removeOnFail"]["age"]
local maxCount = opts["removeOnFail"]["count"]
if maxAge ~= nil then
removeJobsByMaxAge(timestamp, maxAge,
failedKey, queueKeyPrefix)
end
if maxCount ~= nil and maxCount > 0 then
removeJobsByMaxCount(maxCount, failedKey,
queueKeyPrefix)
end
end
table.insert(failed, jobId)
else
local target =
getTargetQueueList(metaKey, waitKey, pausedKey)
-- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
rcall("RPUSH", target, jobId)
rcall("XADD", eventStreamKey, "*", "event",
"waiting", "jobId", jobId, 'prev', 'active')
-- Emit the stalled event
rcall("XADD", eventStreamKey, "*", "event",
"stalled", "jobId", jobId)
table.insert(stalled, jobId)
end
end
end
end
end
end
-- Mark potentially stalled jobs
local active = rcall('LRANGE', activeKey, 0, -1)
if (#active > 0) then
for from, to in batches(#active, 7000) do
rcall('SADD', stalledKey, unpack(active, from, to))
end
end
return {failed, stalled}
end
+47
View File
@@ -0,0 +1,47 @@
--[[
Function to clean job list.
Returns jobIds and deleted count number.
]]
-- Includes
--- @include "getTimestamp"
--- @include "removeJob"
local function cleanList(listKey, jobKeyPrefix, rangeStart, rangeEnd,
timestamp, isWaiting)
local jobs = rcall("LRANGE", listKey, rangeStart, rangeEnd)
local deleted = {}
local deletedCount = 0
local jobTS
local deletionMarker = ''
local jobIdsLen = #jobs
for i, job in ipairs(jobs) do
if limit > 0 and deletedCount >= limit then
break
end
local jobKey = jobKeyPrefix .. job
if (isWaiting or rcall("EXISTS", jobKey .. ":lock") == 0) then
-- Find the right timestamp of the job to compare to maxTimestamp:
-- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed
-- * processedOn represents when the job was last attempted, but it doesn't get populated until
-- the job is first tried
-- * timestamp is the original job submission time
-- Fetch all three of these (in that order) and use the first one that is set so that we'll leave jobs
-- that have been active within the grace period:
jobTS = getTimestamp(jobKey, {"finishedOn", "processedOn", "timestamp"})
if (not jobTS or jobTS <= timestamp) then
-- replace the entry with a deletion marker; the actual deletion will
-- occur at the end of the script
rcall("LSET", listKey, rangeEnd - jobIdsLen + i, deletionMarker)
removeJob(job, true, jobKeyPrefix)
deletedCount = deletedCount + 1
table.insert(deleted, job)
end
end
end
rcall("LREM", listKey, 0, deletionMarker)
return {deleted, deletedCount}
end
+45
View File
@@ -0,0 +1,45 @@
--[[
Function to clean job set.
Returns jobIds and deleted count number.
]]
-- Includes
--- @include "batches"
--- @include "getJobsInZset"
--- @include "getTimestamp"
--- @include "removeJob"
local function cleanSet(setKey, jobKeyPrefix, rangeEnd, timestamp, limit, attributes, isFinished)
local jobs = getJobsInZset(setKey, rangeEnd, limit)
local deleted = {}
local deletedCount = 0
local jobTS
for i, job in ipairs(jobs) do
if limit > 0 and deletedCount >= limit then
break
end
local jobKey = jobKeyPrefix .. job
if isFinished then
removeJob(job, true, jobKeyPrefix)
deletedCount = deletedCount + 1
table.insert(deleted, job)
else
-- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed
jobTS = getTimestamp(jobKey, attributes)
if (not jobTS or jobTS <= timestamp) then
removeJob(job, true, jobKeyPrefix)
deletedCount = deletedCount + 1
table.insert(deleted, job)
end
end
end
if(#deleted > 0) then
for from, to in batches(#deleted, 7000) do
rcall("ZREM", setKey, unpack(deleted, from, to))
end
end
return {deleted, deletedCount}
end
+46
View File
@@ -0,0 +1,46 @@
--[[
Functions to collect metrics based on a current and previous count of jobs.
Granualarity is fixed at 1 minute.
]]
--- @include "batches"
local function collectMetrics(metaKey, dataPointsList, maxDataPoints,
timestamp)
-- Increment current count
local count = rcall("HINCRBY", metaKey, "count", 1) - 1
-- Compute how many data points we need to add to the list, N.
local prevTS = rcall("HGET", metaKey, "prevTS")
if not prevTS then
-- If prevTS is nil, set it to the current timestamp
rcall("HSET", metaKey, "prevTS", timestamp, "prevCount", 0)
return
end
local N = math.floor((timestamp - prevTS) / 60000)
if N > 0 then
local delta = count - rcall("HGET", metaKey, "prevCount")
-- If N > 1, add N-1 zeros to the list
if N > 1 then
local points = {}
points[1] = delta
for i = 2, N do
points[i] = 0
end
for from, to in batches(#points, 7000) do
rcall("LPUSH", dataPointsList, unpack(points, from, to))
end
else
-- LPUSH delta to the list
rcall("LPUSH", dataPointsList, delta)
end
-- LTRIM to keep list to its max size
rcall("LTRIM", dataPointsList, 0, maxDataPoints - 1)
-- update prev count with current count
rcall("HSET", metaKey, "prevCount", count, "prevTS", timestamp)
end
end
+12
View File
@@ -0,0 +1,12 @@
--[[
Functions to destructure job key.
Just a bit of warning, these functions may be a bit slow and affect performance significantly.
]]
local getJobIdFromKey = function (jobKey)
return string.match(jobKey, ".*:(.*)")
end
local getJobKeyPrefix = function (jobKey, jobId)
return string.sub(jobKey, 0, #jobKey - #jobId)
end
+70
View File
@@ -0,0 +1,70 @@
--[[
Function to achieve pagination for a set or hash.
This function simulates pagination in the most efficient way possible
for a set using sscan or hscan.
The main limitation is that sets are not order preserving, so the
pagination is not stable. This means that if the set is modified
between pages, the same element may appear in different pages.
]] -- Maximum number of elements to be returned by sscan per iteration.
local maxCount = 100
-- Finds the cursor, and returns the first elements available for the requested page.
local function findPage(key, command, pageStart, pageSize, cursor, offset,
maxIterations, fetchJobs)
local items = {}
local jobs = {}
local iterations = 0
repeat
-- Iterate over the set using sscan/hscan.
local result = rcall(command, key, cursor, "COUNT", maxCount)
cursor = result[1]
local members = result[2]
local step = 1
if command == "HSCAN" then
step = 2
end
if #members == 0 then
-- If the result is empty, we can return the result.
return cursor, offset, items, jobs
end
local chunkStart = offset
local chunkEnd = offset + #members / step
local pageEnd = pageStart + pageSize
if chunkEnd < pageStart then
-- If the chunk is before the page, we can skip it.
offset = chunkEnd
elseif chunkStart > pageEnd then
-- If the chunk is after the page, we can return the result.
return cursor, offset, items, jobs
else
-- If the chunk is overlapping the page, we need to add the elements to the result.
for i = 1, #members, step do
if offset >= pageEnd then
return cursor, offset, items, jobs
end
if offset >= pageStart then
local index = #items + 1
if fetchJobs ~= nil then
jobs[#jobs+1] = rcall("HGETALL", members[i])
end
if step == 2 then
items[index] = {members[i], members[i + 1]}
else
items[index] = members[i]
end
end
offset = offset + 1
end
end
iterations = iterations + 1
until cursor == "0" or iterations >= maxIterations
return cursor, offset, items, jobs
end
+11
View File
@@ -0,0 +1,11 @@
-- We use ZRANGEBYSCORE to make the case where we're deleting a limited number
-- of items in a sorted set only run a single iteration. If we simply used
-- ZRANGE, we may take a long time traversing through jobs that are within the
-- grace period.
local function getJobsInZset(zsetKey, rangeEnd, limit)
if limit > 0 then
return rcall("ZRANGEBYSCORE", zsetKey, 0, rangeEnd, "LIMIT", 0, limit)
else
return rcall("ZRANGEBYSCORE", zsetKey, 0, rangeEnd)
end
end
@@ -0,0 +1,13 @@
--[[
Function to return the next delayed job timestamp.
]]
local function getNextDelayedTimestamp(delayedKey)
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
if #result then
local nextTimestamp = tonumber(result[2])
if (nextTimestamp ~= nil) then
nextTimestamp = nextTimestamp / 0x1000
end
return nextTimestamp
end
end
+9
View File
@@ -0,0 +1,9 @@
local function getOrSetMaxEvents(metaKey)
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
if not maxEvents then
maxEvents = 10000
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
end
return maxEvents
end
+7
View File
@@ -0,0 +1,7 @@
--[[
Function to get priority score.
]]
local function getPriorityScore(priority, priorityCounterKey)
local prioCounter = rcall("INCR", priorityCounterKey)
return priority * 0x100000000 + prioCounter % 0x100000000
end
+14
View File
@@ -0,0 +1,14 @@
local function getRateLimitTTL(maxJobs, rateLimiterKey)
if maxJobs and maxJobs <= tonumber(rcall("GET", rateLimiterKey) or 0) then
local pttl = rcall("PTTL", rateLimiterKey)
if pttl == 0 then
rcall("DEL", rateLimiterKey)
end
if pttl > 0 then
return pttl
end
end
return 0
end
+12
View File
@@ -0,0 +1,12 @@
--[[
Function to check for the meta.paused key to decide if we are paused or not
(since an empty list and !EXISTS are not really the same).
]]
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
return waitKey, false
else
return pausedKey, true
end
end
+19
View File
@@ -0,0 +1,19 @@
--[[
Function to get the latest saved timestamp.
]]
local function getTimestamp(jobKey, attributes)
if #attributes == 1 then
return rcall("HGET", jobKey, attributes[1])
end
local jobTs
for _, ts in ipairs(rcall("HMGET", jobKey, unpack(attributes))) do
if (ts) then
jobTs = ts
break
end
end
return jobTs
end
+7
View File
@@ -0,0 +1,7 @@
--[[
Function to get ZSet items.
]]
local function getZSetItems(keyName, max)
return rcall('ZRANGE', keyName, 0, max - 1)
end
+33
View File
@@ -0,0 +1,33 @@
--[[
Function to recursively check if there are no locks
on the jobs to be removed.
returns:
boolean
]]
local function isLocked( prefix, jobId, removeChildren)
local jobKey = prefix .. jobId;
-- Check if this job is locked
local lockKey = jobKey .. ':lock'
local lock = rcall("GET", lockKey)
if not lock then
if removeChildren == "1" then
local dependencies = rcall("SMEMBERS", jobKey .. ":dependencies")
if (#dependencies > 0) then
for i, childJobKey in ipairs(dependencies) do
-- We need to get the jobId for this job.
local childJobId = getJobIdFromKey(childJobKey)
local childJobPrefix = getJobKeyPrefix(childJobKey, childJobId)
local result = isLocked( childJobPrefix, childJobId, removeChildren )
if result then
return true
end
end
end
end
return false
end
return true
end
@@ -0,0 +1,13 @@
--[[
Function to move job from prioritized state to active.
]]
local function moveJobFromPriorityToActive(priorityKey, activeKey, priorityCounterKey)
local prioritizedJob = rcall("ZPOPMIN", priorityKey)
if #prioritizedJob > 0 then
rcall("LPUSH", activeKey, prioritizedJob[1])
return prioritizedJob[1]
else
rcall("DEL", priorityCounterKey)
end
end
@@ -0,0 +1,38 @@
--[[
Function to recursively move from waitingChildren to failed.
]]
-- Includes
--- @include "moveParentToWaitIfNeeded"
local function moveParentFromWaitingChildrenToFailed( parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
if rcall("ZREM", parentQueueKey .. ":waiting-children", parentId) == 1 then
rcall("ZADD", parentQueueKey .. ":failed", timestamp, parentId)
local failedReason = "child " .. jobIdKey .. " failed"
rcall("HMSET", parentKey, "failedReason", failedReason, "finishedOn", timestamp)
rcall("XADD", parentQueueKey .. ":events", "*", "event", "failed", "jobId", parentId, "failedReason",
failedReason, "prev", "waiting-children")
local rawParentData = rcall("HGET", parentKey, "parent")
if rawParentData ~= false then
local parentData = cjson.decode(rawParentData)
if parentData['fpof'] then
moveParentFromWaitingChildrenToFailed(
parentData['queueKey'],
parentData['queueKey'] .. ':' .. parentData['id'],
parentData['id'],
parentKey,
timestamp
)
elseif parentData['rdof'] then
local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
local grandParentDependenciesSet = grandParentKey .. ":dependencies"
if rcall("SREM", grandParentDependenciesSet, parentKey) == 1 then
moveParentToWaitIfNeeded(parentData['queueKey'], grandParentDependenciesSet,
grandParentKey, parentData['id'], timestamp)
end
end
end
end
end
@@ -0,0 +1,42 @@
--[[
Validate and move parent to active if needed.
]]
-- Includes
--- @include "addDelayMarkerIfNeeded"
--- @include "addJobWithPriority"
--- @include "getTargetQueueList"
local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
local isParentActive = rcall("ZSCORE", parentQueueKey .. ":waiting-children", parentId)
if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then
rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
local parentWaitKey = parentQueueKey .. ":wait"
local parentTarget, paused = getTargetQueueList(parentQueueKey .. ":meta", parentWaitKey,
parentQueueKey .. ":paused")
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
local priority = tonumber(jobAttributes[1]) or 0
local delay = tonumber(jobAttributes[2]) or 0
if delay > 0 then
local delayedTimestamp = tonumber(timestamp) + delay
local score = delayedTimestamp * 0x1000
local parentDelayedKey = parentQueueKey .. ":delayed"
rcall("ZADD", parentDelayedKey, score, parentId)
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId,
"delay", delayedTimestamp)
addDelayMarkerIfNeeded(parentTarget, parentDelayedKey)
else
if priority == 0 then
rcall("RPUSH", parentTarget, parentId)
else
addJobWithPriority(parentWaitKey, parentQueueKey .. ":prioritized", priority, paused,
parentId, parentQueueKey .. ":pc")
end
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId,
"prev", "waiting-children")
end
end
end
@@ -0,0 +1,73 @@
--[[
Function to move job from wait state to active.
Input:
keys[1] wait key
keys[2] active key
keys[3] prioritized key
keys[4] stream events key
keys[5] stalled key
-- Rate limiting
keys[6] rate limiter key
keys[7] delayed key
keys[8] paused key
keys[9] meta key
keys[10] pc priority counter
opts - token - lock token
opts - lockDuration
opts - limiter
]]
-- Includes
--- @include "pushBackJobWithPriority"
local function prepareJobForProcessing(keys, keyPrefix, targetKey, jobId, processedOn,
maxJobs, expireTime, opts)
local jobKey = keyPrefix .. jobId
-- Check if we need to perform rate limiting.
if maxJobs then
local rateLimiterKey = keys[6];
-- check if we exceeded rate limit, we need to remove the job and return expireTime
if expireTime > 0 then
-- remove from active queue and add back to the wait list
rcall("LREM", keys[2], 1, jobId)
local priority = tonumber(rcall("HGET", jobKey, "priority")) or 0
if priority == 0 then
rcall("RPUSH", targetKey, jobId)
else
pushBackJobWithPriority(keys[3], priority, jobId)
end
-- Return when we can process more jobs
return {0, 0, expireTime, 0}
end
local jobCounter = tonumber(rcall("INCR", rateLimiterKey))
if jobCounter == 1 then
local limiterDuration = opts['limiter'] and opts['limiter']['duration']
local integerDuration = math.floor(math.abs(limiterDuration))
rcall("PEXPIRE", rateLimiterKey, integerDuration)
end
end
local lockKey = jobKey .. ':lock'
-- get a lock
if opts['token'] ~= "0" then
rcall("SET", lockKey, opts['token'], "PX", opts['lockDuration'])
end
rcall("XADD", keys[4], "*", "event", "active", "jobId", jobId, "prev", "waiting")
rcall("HSET", jobKey, "processedOn", processedOn)
rcall("HINCRBY", jobKey, "attemptsMade", 1)
return {rcall("HGETALL", jobKey), jobId, 0, 0} -- get job data
end
+44
View File
@@ -0,0 +1,44 @@
--[[
Updates the delay set, by moving delayed jobs that should
be processed now to "wait".
Events:
'waiting'
]]
-- Includes
--- @include "addPriorityMarkerIfNeeded"
--- @include "getPriorityScore"
-- Try to get as much as 1000 jobs at once
local function promoteDelayedJobs(delayedKey, waitKey, targetKey, prioritizedKey,
eventStreamKey, prefix, timestamp, paused, priorityCounterKey)
local jobs = rcall("ZRANGEBYSCORE", delayedKey, 0, (timestamp + 1) * 0x1000, "LIMIT", 0, 1000)
if (#jobs > 0) then
rcall("ZREM", delayedKey, unpack(jobs))
for _, jobId in ipairs(jobs) do
local jobKey = prefix .. jobId
local priority =
tonumber(rcall("HGET", jobKey, "priority")) or 0
if priority == 0 then
-- LIFO or FIFO
rcall("LPUSH", targetKey, jobId)
else
local score = getPriorityScore(priority, priorityCounterKey)
rcall("ZADD", prioritizedKey, score, jobId)
end
-- Emit waiting event
rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId",
jobId, "prev", "delayed")
rcall("HSET", jobKey, "delay", 0)
end
if not paused then
addPriorityMarkerIfNeeded(targetKey)
end
end
end
@@ -0,0 +1,9 @@
--[[
Function to push back job considering priority in front of same prioritized jobs.
]]
local function pushBackJobWithPriority(prioritizedKey, priority, jobId)
-- in order to put it at front of same prioritized jobs
-- we consider prioritized counter as 0
local score = priority * 0x100000000
rcall("ZADD", prioritizedKey, score, jobId)
end
+13
View File
@@ -0,0 +1,13 @@
--[[
Function to remove job.
]]
-- Includes
--- @include "removeParentDependencyKey"
local function removeJob(jobId, hard, baseKey)
local jobKey = baseKey .. jobId
removeParentDependencyKey(jobKey, hard, nil, baseKey)
rcall("DEL", jobKey, jobKey .. ':logs',
jobKey .. ':dependencies', jobKey .. ':processed')
end
@@ -0,0 +1,35 @@
--[[
Function to remove from any state.
returns:
prev state
]]
local function removeJobFromAnyState( prefix, jobId)
-- We start with the ZSCORE checks, since they have O(1) complexity
if rcall("ZSCORE", prefix .. "completed", jobId) then
rcall("ZREM", prefix .. "completed", jobId)
return "completed"
elseif rcall("ZSCORE", prefix .. "waiting-children", jobId) then
rcall("ZREM", prefix .. "waiting-children", jobId)
return "waiting-children"
elseif rcall("ZSCORE", prefix .. "delayed", jobId) then
rcall("ZREM", prefix .. "delayed", jobId)
return "delayed"
elseif rcall("ZSCORE", prefix .. "failed", jobId) then
rcall("ZREM", prefix .. "failed", jobId)
return "failed"
elseif rcall("ZSCORE", prefix .. "prioritized", jobId) then
rcall("ZREM", prefix .. "prioritized", jobId)
return "prioritized"
-- We remove only 1 element from the list, since we assume they are not added multiple times
elseif rcall("LREM", prefix .. "wait", 1, jobId) == 1 then
return "wait"
elseif rcall("LREM", prefix .. "paused", 1, jobId) == 1 then
return "paused"
elseif rcall("LREM", prefix .. "active", 1, jobId) == 1 then
return "active"
end
return "unknown"
end
+13
View File
@@ -0,0 +1,13 @@
--[[
Functions to remove jobs.
]]
-- Includes
--- @include "removeJob"
local function removeJobs(keys, hard, baseKey, max)
for i, key in ipairs(keys) do
removeJob(key, hard, baseKey)
end
return max - #keys
end
+15
View File
@@ -0,0 +1,15 @@
--[[
Functions to remove jobs by max age.
]]
-- Includes
--- @include "removeJob"
local function removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
local start = timestamp - maxAge * 1000
local jobIds = rcall("ZREVRANGEBYSCORE", targetSet, start, "-inf")
for i, jobId in ipairs(jobIds) do
removeJob(jobId, false, prefix)
end
rcall("ZREMRANGEBYSCORE", targetSet, "-inf", start)
end
@@ -0,0 +1,15 @@
--[[
Functions to remove jobs by max count.
]]
-- Includes
--- @include "removeJob"
local function removeJobsByMaxCount(maxCount, targetSet, prefix)
local start = maxCount
local jobIds = rcall("ZREVRANGE", targetSet, start, -1)
for i, jobId in ipairs(jobIds) do
removeJob(jobId, false, prefix)
end
rcall("ZREMRANGEBYRANK", targetSet, 0, -(maxCount + 1))
end
+17
View File
@@ -0,0 +1,17 @@
--[[
Functions to remove jobs.
]]
-- Includes
--- @include "removeJobs"
local function getListItems(keyName, max)
return rcall('LRANGE', keyName, 0, max - 1)
end
local function removeListJobs(keyName, hard, baseKey, max)
local jobs = getListItems(keyName, max)
local count = removeJobs(jobs, hard, baseKey, max)
rcall("LTRIM", keyName, #jobs, -1)
return count
end
@@ -0,0 +1,77 @@
--[[
Check if this job has a parent. If so we will just remove it from
the parent child list, but if it is the last child we should move the parent to "wait/paused"
which requires code from "moveToFinished"
]]
--- @include "destructureJobKey"
--- @include "getTargetQueueList"
local function moveParentToWait(parentPrefix, parentId, emitEvent)
local parentTarget = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "wait", parentPrefix .. "paused")
rcall("RPUSH", parentTarget, parentId)
if emitEvent then
local parentEventStream = parentPrefix .. "events"
rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
end
end
local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey)
if parentKey then
local parentDependenciesKey = parentKey .. ":dependencies"
local result = rcall("SREM", parentDependenciesKey, jobKey)
if result > 0 then
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
if pendingDependencies == 0 then
local parentId = getJobIdFromKey(parentKey)
local parentPrefix = getJobKeyPrefix(parentKey, parentId)
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
if numRemovedElements == 1 then
if hard then
if parentPrefix == baseKey then
removeParentDependencyKey(parentKey, hard, nil, baseKey)
rcall("DEL", parentKey, parentKey .. ':logs',
parentKey .. ':dependencies', parentKey .. ':processed')
else
moveParentToWait(parentPrefix, parentId)
end
else
moveParentToWait(parentPrefix, parentId, true)
end
end
end
end
else
local missedParentKey = rcall("HGET", jobKey, "parentKey")
if( (type(missedParentKey) == "string") and missedParentKey ~= "" and (rcall("EXISTS", missedParentKey) == 1)) then
local parentDependenciesKey = missedParentKey .. ":dependencies"
local result = rcall("SREM", parentDependenciesKey, jobKey)
if result > 0 then
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
if pendingDependencies == 0 then
local parentId = getJobIdFromKey(missedParentKey)
local parentPrefix = getJobKeyPrefix(missedParentKey, parentId)
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
if numRemovedElements == 1 then
if hard then
if parentPrefix == baseKey then
removeParentDependencyKey(missedParentKey, hard, nil, baseKey)
rcall("DEL", missedParentKey, missedParentKey .. ':logs',
missedParentKey .. ':dependencies', missedParentKey .. ':processed')
else
moveParentToWait(parentPrefix, parentId)
end
else
moveParentToWait(parentPrefix, parentId, true)
end
end
end
end
end
end
end

Some files were not shown because too many files have changed in this diff Show More