Initial commit of working RSS Aggregator build

This commit is contained in:
jmadmin committed 2026-05-12 17:06:28 -03:00
1 parent ea3a2ca53e
commit 7ac2f6e384
4962 files changed
+1032666

No files matched your search

+43
View File
@@ -0,0 +1,43 @@
/**
* AsyncFifoQueue
*
* A minimal FIFO queue for asynchronous operations. Allows adding asynchronous operations
* and consume them in the order they are resolved.
*
* TODO: Optimize using a linked list for the queue instead of an array.
* Current implementation requires memory copies when shifting the queue.
* For a linked linked implementation, we can exploit the fact that the
* maximum number of elements in the list will never exceen the concurrency factor
* of the worker, so the nodes of the list could be pre-allocated.
*/
export declare class AsyncFifoQueue<T> {
private ignoreErrors;
/**
* A queue of completed promises. As the pending
* promises are resolved, they are added to this queue.
*/
private queue;
/**
* A set of pending promises.
*/
private pending;
/**
* The next promise to be resolved. As soon as a pending promise
* is resolved, this promise is resolved with the result of the
* pending promise.
*/
private nextPromise;
private resolve;
private reject;
constructor(ignoreErrors?: boolean);
add(promise: Promise<T>): void;
waitAll(): Promise<void>;
numTotal(): number;
numPending(): number;
numQueued(): number;
private resolvePromise;
private rejectPromise;
private newPromise;
private wait;
fetch(): Promise<T | void>;
}
+93
View File
@@ -0,0 +1,93 @@
/**
* AsyncFifoQueue
*
* A minimal FIFO queue for asynchronous operations. Allows adding asynchronous operations
* and consume them in the order they are resolved.
*
* TODO: Optimize using a linked list for the queue instead of an array.
* Current implementation requires memory copies when shifting the queue.
* For a linked linked implementation, we can exploit the fact that the
* maximum number of elements in the list will never exceen the concurrency factor
* of the worker, so the nodes of the list could be pre-allocated.
*/
export class AsyncFifoQueue {
constructor(ignoreErrors = false) {
this.ignoreErrors = ignoreErrors;
/**
* A queue of completed promises. As the pending
* promises are resolved, they are added to this queue.
*/
this.queue = [];
/**
* A set of pending promises.
*/
this.pending = new Set();
this.newPromise();
}
add(promise) {
this.pending.add(promise);
promise
.then(data => {
this.pending.delete(promise);
if (this.queue.length === 0) {
this.resolvePromise(data);
}
this.queue.push(data);
})
.catch(err => {
// Ignore errors
if (this.ignoreErrors) {
this.queue.push(undefined);
}
this.pending.delete(promise);
this.rejectPromise(err);
});
}
async waitAll() {
await Promise.all(this.pending);
}
numTotal() {
return this.pending.size + this.queue.length;
}
numPending() {
return this.pending.size;
}
numQueued() {
return this.queue.length;
}
resolvePromise(data) {
this.resolve(data);
this.newPromise();
}
rejectPromise(err) {
this.reject(err);
this.newPromise();
}
newPromise() {
this.nextPromise = new Promise((resolve, reject) => {
this.resolve = resolve;
this.reject = reject;
});
}
async wait() {
return this.nextPromise;
}
async fetch() {
if (this.pending.size === 0 && this.queue.length === 0) {
return;
}
while (this.queue.length === 0) {
try {
await this.wait();
}
catch (err) {
// Ignore errors
if (!this.ignoreErrors) {
console.error('Unexpected Error in AsyncFifoQueue', err);
}
}
}
return this.queue.shift();
}
}
//# sourceMappingURL=async-fifo-queue.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"async-fifo-queue.js","sourceRoot":"","sources":["../../../src/classes/async-fifo-queue.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,cAAc;IAqBzB,YAAoB,eAAe,KAAK;QAApB,iBAAY,GAAZ,YAAY,CAAQ;QApBxC;;;WAGG;QACK,UAAK,GAAsB,EAAE,CAAC;QAEtC;;WAEG;QACK,YAAO,GAAG,IAAI,GAAG,EAAc,CAAC;QAYtC,IAAI,CAAC,UAAU,EAAE,CAAC;IACpB,CAAC;IAEM,GAAG,CAAC,OAAmB;QAC5B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAE1B,OAAO;aACJ,IAAI,CAAC,IAAI,CAAC,EAAE;YACX,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAE7B,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;gBAC3B,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;aAC3B;YACD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,CAAC,CAAC;aACD,KAAK,CAAC,GAAG,CAAC,EAAE;YACX,gBAAgB;YAChB,IAAI,IAAI,CAAC,YAAY,EAAE;gBACrB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;aAC5B;YACD,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC7B,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;IACP,CAAC;IAEM,KAAK,CAAC,OAAO;QAClB,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAClC,CAAC;IAEM,QAAQ;QACb,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAC/C,CAAC;IAEM,UAAU;QACf,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;IAC3B,CAAC;IAEM,SAAS;QACd,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAC3B,CAAC;IAEO,cAAc,CAAC,IAAO;QAC5B,IAAI,CAAC,OAAQ,CAAC,IAAI,CAAC,CAAC;QACpB,IAAI,CAAC,UAAU,EAAE,CAAC;IACpB,CAAC;IAEO,aAAa,CAAC,GAAQ;QAC5B,IAAI,CAAC,MAAO,CAAC,GAAG,CAAC,CAAC;QAClB,IAAI,CAAC,UAAU,EAAE,CAAC;IACpB,CAAC;IAEO,UAAU;QAChB,IAAI,CAAC,WAAW,GAAG,IAAI,OAAO,CAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAChE,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;YACvB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACvB,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,IAAI;QAChB,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAEM,KAAK,CAAC,KAAK;QAChB,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;YACtD,OAAO;SACR;QACD,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;YAC9B,IAAI;gBACF,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;aACnB;YAAC,OAAO,GAAG,EAAE;gBACZ,gBAAgB;gBAChB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;oBACtB,OAAO,CAAC,KAAK,CAAC,oCAAoC,EAAE,GAAG,CAAC,CAAC;iBAC1D;aACF;SACF;QACD,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IAC5B,CAAC;CACF"}
+10
View File
@@ -0,0 +1,10 @@
import { BackoffOptions, MinimalJob } from '../interfaces';
import { BackoffStrategy } from '../types';
export interface BuiltInStrategies {
[index: string]: (delay: number) => BackoffStrategy;
}
export declare class Backoffs {
static builtinStrategies: BuiltInStrategies;
static normalize(backoff: number | BackoffOptions): BackoffOptions | undefined;
static calculate(backoff: BackoffOptions, attemptsMade: number, err: Error, job: MinimalJob, customStrategy?: BackoffStrategy): Promise<number> | number | undefined;
}
+44
View File
@@ -0,0 +1,44 @@
export class Backoffs {
static normalize(backoff) {
if (Number.isFinite(backoff)) {
return {
type: 'fixed',
delay: backoff,
};
}
else if (backoff) {
return backoff;
}
}
static calculate(backoff, attemptsMade, err, job, customStrategy) {
if (backoff) {
const strategy = lookupStrategy(backoff, customStrategy);
return strategy(attemptsMade, backoff.type, err, job);
}
}
}
Backoffs.builtinStrategies = {
fixed: function (delay) {
return function () {
return delay;
};
},
exponential: function (delay) {
return function (attemptsMade) {
return Math.round(Math.pow(2, attemptsMade - 1) * delay);
};
},
};
function lookupStrategy(backoff, customStrategy) {
if (backoff.type in Backoffs.builtinStrategies) {
return Backoffs.builtinStrategies[backoff.type](backoff.delay);
}
else if (customStrategy) {
return customStrategy;
}
else {
throw new Error(`Unknown backoff strategy ${backoff.type}.
If a custom backoff strategy is used, specify it when the queue is created.`);
}
}
//# sourceMappingURL=backoffs.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"backoffs.js","sourceRoot":"","sources":["../../../src/classes/backoffs.ts"],"names":[],"mappings":"AAOA,MAAM,OAAO,QAAQ;IAenB,MAAM,CAAC,SAAS,CACd,OAAgC;QAEhC,IAAI,MAAM,CAAC,QAAQ,CAAS,OAAO,CAAC,EAAE;YACpC,OAAO;gBACL,IAAI,EAAE,OAAO;gBACb,KAAK,EAAU,OAAO;aACvB,CAAC;SACH;aAAM,IAAI,OAAO,EAAE;YAClB,OAAuB,OAAO,CAAC;SAChC;IACH,CAAC;IAED,MAAM,CAAC,SAAS,CACd,OAAuB,EACvB,YAAoB,EACpB,GAAU,EACV,GAAe,EACf,cAAgC;QAEhC,IAAI,OAAO,EAAE;YACX,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;YAEzD,OAAO,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;SACvD;IACH,CAAC;;AAvCM,0BAAiB,GAAsB;IAC5C,KAAK,EAAE,UAAU,KAAa;QAC5B,OAAO;YACL,OAAO,KAAK,CAAC;QACf,CAAC,CAAC;IACJ,CAAC;IAED,WAAW,EAAE,UAAU,KAAa;QAClC,OAAO,UAAU,YAAoB;YACnC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,YAAY,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC;QAC3D,CAAC,CAAC;IACJ,CAAC;CACF,CAAC;AA8BJ,SAAS,cAAc,CACrB,OAAuB,EACvB,cAAgC;IAEhC,IAAI,OAAO,CAAC,IAAI,IAAI,QAAQ,CAAC,iBAAiB,EAAE;QAC9C,OAAO,QAAQ,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAM,CAAC,CAAC;KACjE;SAAM,IAAI,cAAc,EAAE;QACzB,OAAO,cAAc,CAAC;KACvB;SAAM;QACL,MAAM,IAAI,KAAK,CACb,4BAA4B,OAAO,CAAC,IAAI;kFACoC,CAC7E,CAAC;KACH;AACH,CAAC"}
+23
View File
@@ -0,0 +1,23 @@
import { Child } from './child';
interface ChildPoolOpts {
mainFile?: string;
useWorkerThreads?: boolean;
}
export declare class ChildPool {
retained: {
[key: number]: Child;
};
free: {
[key: string]: Child[];
};
private opts;
constructor({ mainFile, useWorkerThreads, }: ChildPoolOpts);
retain(processFile: string): Promise<Child>;
release(child: Child): void;
remove(child: Child): void;
kill(child: Child, signal?: 'SIGTERM' | 'SIGKILL'): Promise<void>;
clean(): Promise<void>;
getFree(id: string): Child[];
getAllFree(): Child[];
}
export {};
+60
View File
@@ -0,0 +1,60 @@
import * as path from 'path';
import { Child } from './child';
const CHILD_KILL_TIMEOUT = 30000;
export class ChildPool {
constructor({ mainFile = path.join(process.cwd(), 'dist/cjs/classes/main.js'), useWorkerThreads, }) {
this.retained = {};
this.free = {};
this.opts = { mainFile, useWorkerThreads };
}
async retain(processFile) {
let child = this.getFree(processFile).pop();
if (child) {
this.retained[child.pid] = child;
return child;
}
child = new Child(this.opts.mainFile, processFile, {
useWorkerThreads: this.opts.useWorkerThreads,
});
child.on('exit', this.remove.bind(this, child));
try {
await child.init();
this.retained[child.pid] = child;
return child;
}
catch (err) {
console.error(err);
this.release(child);
throw err;
}
}
release(child) {
delete this.retained[child.pid];
this.getFree(child.processFile).push(child);
}
remove(child) {
delete this.retained[child.pid];
const free = this.getFree(child.processFile);
const childIndex = free.indexOf(child);
if (childIndex > -1) {
free.splice(childIndex, 1);
}
}
async kill(child, signal = 'SIGKILL') {
this.remove(child);
return child.kill(signal, CHILD_KILL_TIMEOUT);
}
async clean() {
const children = Object.values(this.retained).concat(this.getAllFree());
this.retained = {};
this.free = {};
await Promise.all(children.map(c => this.kill(c, 'SIGTERM')));
}
getFree(id) {
return (this.free[id] = this.free[id] || []);
}
getAllFree() {
return Object.values(this.free).reduce((first, second) => first.concat(second), []);
}
}
//# sourceMappingURL=child-pool.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"child-pool.js","sourceRoot":"","sources":["../../../src/classes/child-pool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAEhC,MAAM,kBAAkB,GAAG,KAAM,CAAC;AAOlC,MAAM,OAAO,SAAS;IAKpB,YAAY,EACV,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,0BAA0B,CAAC,EAC/D,gBAAgB,GACF;QAPhB,aAAQ,GAA6B,EAAE,CAAC;QACxC,SAAI,GAA+B,EAAE,CAAC;QAOpC,IAAI,CAAC,IAAI,GAAG,EAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,WAAmB;QAC9B,IAAI,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,GAAG,EAAE,CAAC;QAE5C,IAAI,KAAK,EAAE;YACT,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;YACjC,OAAO,KAAK,CAAC;SACd;QAED,KAAK,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE;YACjD,gBAAgB,EAAE,IAAI,CAAC,IAAI,CAAC,gBAAgB;SAC7C,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;QAEhD,IAAI;YACF,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;YAEjC,OAAO,KAAK,CAAC;SACd;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACnB,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,GAAG,CAAC;SACX;IACH,CAAC;IAED,OAAO,CAAC,KAAY;QAClB,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,CAAC,KAAY;QACjB,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAEhC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QAE7C,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACvC,IAAI,UAAU,GAAG,CAAC,CAAC,EAAE;YACnB,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;SAC5B;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CACR,KAAY,EACZ,SAAgC,SAAS;QAEzC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QACxE,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;QACnB,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC;QAEf,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC;IAChE,CAAC;IAED,OAAO,CAAC,EAAU;QAChB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;IAC/C,CAAC;IAED,UAAU;QACR,OAAO,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,CACpC,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,EACvC,EAAE,CACH,CAAC;IACJ,CAAC;CACF"}
+36
View File
@@ -0,0 +1,36 @@
import { JobJson, SandboxedJob } from '../interfaces';
declare enum ChildStatus {
Idle = 0,
Started = 1,
Terminating = 2,
Errored = 3
}
/**
* ChildProcessor
*
* This class acts as the interface between a child process and it parent process
* so that jobs can be processed in different processes.
*
*/
export declare class ChildProcessor {
private send;
status?: ChildStatus;
processor: any;
currentJobPromise: Promise<unknown> | undefined;
constructor(send: (msg: any) => Promise<void>);
init(processorFile: string): Promise<void>;
start(jobJson: JobJson, token?: string): Promise<void>;
stop(): Promise<void>;
waitForCurrentJobAndExit(): Promise<void>;
/**
* Enhance the given job argument with some functions
* that can be called from the sandboxed job processor.
*
* Note, the `job` argument is a JSON deserialized message
* from the main node process to this forked child process,
* the functions on the original job object are not in tact.
* The wrapped job adds back some of those original functions.
*/
protected wrapJob(job: JobJson, send: (msg: any) => Promise<void>): SandboxedJob;
}
export {};
+148
View File
@@ -0,0 +1,148 @@
import { ParentCommand } from '../enums';
import { errorToJSON } from '../utils';
var ChildStatus;
(function (ChildStatus) {
ChildStatus[ChildStatus["Idle"] = 0] = "Idle";
ChildStatus[ChildStatus["Started"] = 1] = "Started";
ChildStatus[ChildStatus["Terminating"] = 2] = "Terminating";
ChildStatus[ChildStatus["Errored"] = 3] = "Errored";
})(ChildStatus || (ChildStatus = {}));
/**
* ChildProcessor
*
* This class acts as the interface between a child process and it parent process
* so that jobs can be processed in different processes.
*
*/
export class ChildProcessor {
constructor(send) {
this.send = send;
}
async init(processorFile) {
let processor;
try {
const { default: processorFn } = await import(processorFile);
processor = processorFn;
if (processor.default) {
// support es2015 module.
processor = processor.default;
}
if (typeof processor !== 'function') {
throw new Error('No function is exported in processor file');
}
}
catch (err) {
this.status = ChildStatus.Errored;
return this.send({
cmd: ParentCommand.InitFailed,
err: errorToJSON(err),
});
}
const origProcessor = processor;
processor = function (job, token) {
try {
return Promise.resolve(origProcessor(job, token));
}
catch (err) {
return Promise.reject(err);
}
};
this.processor = processor;
this.status = ChildStatus.Idle;
await this.send({
cmd: ParentCommand.InitCompleted,
});
}
async start(jobJson, token) {
if (this.status !== ChildStatus.Idle) {
return this.send({
cmd: ParentCommand.Error,
err: errorToJSON(new Error('cannot start a not idling child process')),
});
}
this.status = ChildStatus.Started;
this.currentJobPromise = (async () => {
try {
const job = this.wrapJob(jobJson, this.send);
const result = await this.processor(job, token);
await this.send({
cmd: ParentCommand.Completed,
value: typeof result === 'undefined' ? null : result,
});
}
catch (err) {
await this.send({
cmd: ParentCommand.Failed,
value: errorToJSON(!err.message ? new Error(err) : err),
});
}
finally {
this.status = ChildStatus.Idle;
this.currentJobPromise = undefined;
}
})();
}
async stop() { }
async waitForCurrentJobAndExit() {
this.status = ChildStatus.Terminating;
try {
await this.currentJobPromise;
}
finally {
process.exit(process.exitCode || 0);
}
}
/**
* Enhance the given job argument with some functions
* that can be called from the sandboxed job processor.
*
* Note, the `job` argument is a JSON deserialized message
* from the main node process to this forked child process,
* the functions on the original job object are not in tact.
* The wrapped job adds back some of those original functions.
*/
wrapJob(job, send) {
return Object.assign(Object.assign({}, job), { data: JSON.parse(job.data || '{}'), opts: job.opts, returnValue: JSON.parse(job.returnvalue || '{}'),
/*
* Emulate the real job `updateProgress` function, should works as `progress` function.
*/
async updateProgress(progress) {
// Locally store reference to new progress value
// so that we can return it from this process synchronously.
this.progress = progress;
// Send message to update job progress.
await send({
cmd: ParentCommand.Progress,
value: progress,
});
},
/*
* Emulate the real job `log` function.
*/
log: async (row) => {
send({
cmd: ParentCommand.Log,
value: row,
});
},
/*
* Emulate the real job `moveToDelayed` function.
*/
moveToDelayed: async (timestamp, token) => {
send({
cmd: ParentCommand.MoveToDelayed,
value: { timestamp, token },
});
},
/*
* Emulate the real job `updateData` function.
*/
updateData: async (data) => {
send({
cmd: ParentCommand.Update,
value: data,
});
} });
}
}
//# sourceMappingURL=child-processor.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"child-processor.js","sourceRoot":"","sources":["../../../src/classes/child-processor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAEvC,IAAK,WAKJ;AALD,WAAK,WAAW;IACd,6CAAI,CAAA;IACJ,mDAAO,CAAA;IACP,2DAAW,CAAA;IACX,mDAAO,CAAA;AACT,CAAC,EALI,WAAW,KAAX,WAAW,QAKf;AAED;;;;;;GAMG;AACH,MAAM,OAAO,cAAc;IAKzB,YAAoB,IAAiC;QAAjC,SAAI,GAAJ,IAAI,CAA6B;IAAG,CAAC;IAElD,KAAK,CAAC,IAAI,CAAC,aAAqB;QACrC,IAAI,SAAS,CAAC;QACd,IAAI;YACF,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;YAC7D,SAAS,GAAG,WAAW,CAAC;YAExB,IAAI,SAAS,CAAC,OAAO,EAAE;gBACrB,yBAAyB;gBACzB,SAAS,GAAG,SAAS,CAAC,OAAO,CAAC;aAC/B;YAED,IAAI,OAAO,SAAS,KAAK,UAAU,EAAE;gBACnC,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;aAC9D;SACF;QAAC,OAAO,GAAG,EAAE;YACZ,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC;YAClC,OAAO,IAAI,CAAC,IAAI,CAAC;gBACf,GAAG,EAAE,aAAa,CAAC,UAAU;gBAC7B,GAAG,EAAE,WAAW,CAAC,GAAG,CAAC;aACtB,CAAC,CAAC;SACJ;QAED,MAAM,aAAa,GAAG,SAAS,CAAC;QAChC,SAAS,GAAG,UAAU,GAAiB,EAAE,KAAc;YACrD,IAAI;gBACF,OAAO,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;aACnD;YAAC,OAAO,GAAG,EAAE;gBACZ,OAAO,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;aAC5B;QACH,CAAC,CAAC;QAEF,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC;QAC/B,MAAM,IAAI,CAAC,IAAI,CAAC;YACd,GAAG,EAAE,aAAa,CAAC,aAAa;SACjC,CAAC,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,KAAK,CAAC,OAAgB,EAAE,KAAc;QACjD,IAAI,IAAI,CAAC,MAAM,KAAK,WAAW,CAAC,IAAI,EAAE;YACpC,OAAO,IAAI,CAAC,IAAI,CAAC;gBACf,GAAG,EAAE,aAAa,CAAC,KAAK;gBACxB,GAAG,EAAE,WAAW,CAAC,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;aACvE,CAAC,CAAC;SACJ;QACD,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC;QAClC,IAAI,CAAC,iBAAiB,GAAG,CAAC,KAAK,IAAI,EAAE;YACnC,IAAI;gBACF,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC7C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;gBAChD,MAAM,IAAI,CAAC,IAAI,CAAC;oBACd,GAAG,EAAE,aAAa,CAAC,SAAS;oBAC5B,KAAK,EAAE,OAAO,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM;iBACrD,CAAC,CAAC;aACJ;YAAC,OAAO,GAAG,EAAE;gBACZ,MAAM,IAAI,CAAC,IAAI,CAAC;oBACd,GAAG,EAAE,aAAa,CAAC,MAAM;oBACzB,KAAK,EAAE,WAAW,CAAC,CAAS,GAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,KAAK,CAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;iBACtE,CAAC,CAAC;aACJ;oBAAS;gBACR,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC;gBAC/B,IAAI,CAAC,iBAAiB,GAAG,SAAS,CAAC;aACpC;QACH,CAAC,CAAC,EAAE,CAAC;IACP,CAAC;IAEM,KAAK,CAAC,IAAI,KAAmB,CAAC;IAErC,KAAK,CAAC,wBAAwB;QAC5B,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,WAAW,CAAC;QACtC,IAAI;YACF,MAAM,IAAI,CAAC,iBAAiB,CAAC;SAC9B;gBAAS;YACR,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;SACrC;IACH,CAAC;IAED;;;;;;;;OAQG;IACO,OAAO,CACf,GAAY,EACZ,IAAiC;QAEjC,uCACK,GAAG,KACN,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,IAAI,IAAI,CAAC,EAClC,IAAI,EAAE,GAAG,CAAC,IAAI,EACd,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,IAAI,IAAI,CAAC;YAChD;;eAEG;YACH,KAAK,CAAC,cAAc,CAAC,QAAyB;gBAC5C,gDAAgD;gBAChD,4DAA4D;gBAC5D,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;gBACzB,uCAAuC;gBACvC,MAAM,IAAI,CAAC;oBACT,GAAG,EAAE,aAAa,CAAC,QAAQ;oBAC3B,KAAK,EAAE,QAAQ;iBAChB,CAAC,CAAC;YACL,CAAC;YACD;;eAEG;YACH,GAAG,EAAE,KAAK,EAAE,GAAQ,EAAE,EAAE;gBACtB,IAAI,CAAC;oBACH,GAAG,EAAE,aAAa,CAAC,GAAG;oBACtB,KAAK,EAAE,GAAG;iBACX,CAAC,CAAC;YACL,CAAC;YACD;;eAEG;YACH,aAAa,EAAE,KAAK,EAAE,SAAiB,EAAE,KAAc,EAAE,EAAE;gBACzD,IAAI,CAAC;oBACH,GAAG,EAAE,aAAa,CAAC,aAAa;oBAChC,KAAK,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE;iBAC5B,CAAC,CAAC;YACL,CAAC;YACD;;eAEG;YACH,UAAU,EAAE,KAAK,EAAE,IAAS,EAAE,EAAE;gBAC9B,IAAI,CAAC;oBACH,GAAG,EAAE,aAAa,CAAC,MAAM;oBACzB,KAAK,EAAE,IAAI;iBACZ,CAAC,CAAC;YACL,CAAC,IACD;IACJ,CAAC;CACF"}
+36
View File
@@ -0,0 +1,36 @@
/// <reference types="node" />
/// <reference types="node" />
/// <reference types="node" />
import { ChildProcess } from 'child_process';
import { Worker } from 'worker_threads';
import { EventEmitter } from 'events';
/**
* Child class
*
* This class is used to create a child process or worker thread, and allows using
* isolated processes or threads for processing jobs.
*
*/
export declare class Child extends EventEmitter {
private mainFile;
processFile: string;
private opts;
childProcess: ChildProcess;
worker: Worker;
private _exitCode;
private _signalCode;
private _killed;
constructor(mainFile: string, processFile: string, opts?: {
useWorkerThreads: boolean;
});
get pid(): number;
get exitCode(): number;
get signalCode(): number;
get killed(): boolean;
init(): Promise<void>;
send(msg: any): Promise<void>;
private killProcess;
kill(signal?: 'SIGTERM' | 'SIGKILL', timeoutMs?: number): Promise<void>;
private initChild;
hasProcessExited(): boolean;
}
+212
View File
@@ -0,0 +1,212 @@
import { fork } from 'child_process';
import { Worker } from 'worker_threads';
import { createServer } from 'net';
import { ChildCommand, ParentCommand } from '../enums';
import { EventEmitter } from 'events';
/**
* @see https://nodejs.org/api/process.html#process_exit_codes
*/
const exitCodesErrors = {
1: 'Uncaught Fatal Exception',
2: 'Unused',
3: 'Internal JavaScript Parse Error',
4: 'Internal JavaScript Evaluation Failure',
5: 'Fatal Error',
6: 'Non-function Internal Exception Handler',
7: 'Internal Exception Handler Run-Time Failure',
8: 'Unused',
9: 'Invalid Argument',
10: 'Internal JavaScript Run-Time Failure',
12: 'Invalid Debug Argument',
13: 'Unfinished Top-Level Await',
};
/**
* Child class
*
* This class is used to create a child process or worker thread, and allows using
* isolated processes or threads for processing jobs.
*
*/
export class Child extends EventEmitter {
constructor(mainFile, processFile, opts = {
useWorkerThreads: false,
}) {
super();
this.mainFile = mainFile;
this.processFile = processFile;
this.opts = opts;
this._exitCode = null;
this._signalCode = null;
this._killed = false;
}
get pid() {
if (this.childProcess) {
return this.childProcess.pid;
}
else if (this.worker) {
return this.worker.threadId;
}
else {
throw new Error('No child process or worker thread');
}
}
get exitCode() {
return this._exitCode;
}
get signalCode() {
return this._signalCode;
}
get killed() {
if (this.childProcess) {
return this.childProcess.killed;
}
return this._killed;
}
async init() {
const execArgv = await convertExecArgv(process.execArgv);
let parent;
if (this.opts.useWorkerThreads) {
this.worker = parent = new Worker(this.mainFile, {
execArgv,
stdin: true,
stdout: true,
stderr: true,
});
}
else {
this.childProcess = parent = fork(this.mainFile, [], {
execArgv,
stdio: 'pipe',
});
}
parent.on('exit', (exitCode, signalCode) => {
this._exitCode = exitCode;
// Coerce to null if undefined for backwards compatibility
signalCode = typeof signalCode === 'undefined' ? null : signalCode;
this._signalCode = signalCode;
this._killed = true;
this.emit('exit', exitCode, signalCode);
// Clean all listeners, we do not expect any more events after "exit"
parent.removeAllListeners();
this.removeAllListeners();
});
parent.on('error', (...args) => this.emit('error', ...args));
parent.on('message', (...args) => this.emit('message', ...args));
parent.on('close', (...args) => this.emit('close', ...args));
parent.stdout.pipe(process.stdout);
parent.stderr.pipe(process.stderr);
await this.initChild();
}
async send(msg) {
return new Promise((resolve, reject) => {
if (this.childProcess) {
this.childProcess.send(msg, (err) => {
if (err) {
reject(err);
}
else {
resolve();
}
});
}
else if (this.worker) {
resolve(this.worker.postMessage(msg));
}
else {
resolve();
}
});
}
killProcess(signal = 'SIGKILL') {
if (this.childProcess) {
this.childProcess.kill(signal);
}
else if (this.worker) {
this.worker.terminate();
}
}
async kill(signal = 'SIGKILL', timeoutMs) {
if (this.hasProcessExited()) {
return;
}
const onExit = onExitOnce(this.childProcess || this.worker);
this.killProcess(signal);
if (timeoutMs !== undefined && (timeoutMs === 0 || isFinite(timeoutMs))) {
const timeoutHandle = setTimeout(() => {
if (!this.hasProcessExited()) {
this.killProcess('SIGKILL');
}
}, timeoutMs);
await onExit;
clearTimeout(timeoutHandle);
}
await onExit;
}
async initChild() {
const onComplete = new Promise((resolve, reject) => {
const onMessageHandler = (msg) => {
if (msg.cmd === ParentCommand.InitCompleted) {
resolve();
}
else if (msg.cmd === ParentCommand.InitFailed) {
const err = new Error();
err.stack = msg.err.stack;
err.message = msg.err.message;
reject(err);
}
this.off('message', onMessageHandler);
this.off('close', onCloseHandler);
};
const onCloseHandler = (code, signal) => {
if (code > 128) {
code -= 128;
}
const msg = exitCodesErrors[code] || `Unknown exit code ${code}`;
reject(new Error(`Error initializing child: ${msg} and signal ${signal}`));
this.off('message', onMessageHandler);
this.off('close', onCloseHandler);
};
this.on('message', onMessageHandler);
this.on('close', onCloseHandler);
});
await this.send({
cmd: ChildCommand.Init,
value: this.processFile,
});
await onComplete;
}
hasProcessExited() {
return !!(this.exitCode !== null || this.signalCode);
}
}
function onExitOnce(child) {
return new Promise(resolve => {
child.once('exit', () => resolve());
});
}
const getFreePort = async () => {
return new Promise(resolve => {
const server = createServer();
server.listen(0, () => {
const { port } = server.address();
server.close(() => resolve(port));
});
});
};
const convertExecArgv = async (execArgv) => {
const standard = [];
const convertedArgs = [];
for (let i = 0; i < execArgv.length; i++) {
const arg = execArgv[i];
if (arg.indexOf('--inspect') === -1) {
standard.push(arg);
}
else {
const argName = arg.split('=')[0];
const port = await getFreePort();
convertedArgs.push(`${argName}=${port}`);
}
}
return standard.concat(convertedArgs);
};
//# sourceMappingURL=child.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"child.js","sourceRoot":"","sources":["../../../src/classes/child.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,IAAI,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AACxC,OAAO,EAAe,YAAY,EAAE,MAAM,KAAK,CAAC;AAChD,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAEtC;;GAEG;AACH,MAAM,eAAe,GAAgC;IACnD,CAAC,EAAE,0BAA0B;IAC7B,CAAC,EAAE,QAAQ;IACX,CAAC,EAAE,iCAAiC;IACpC,CAAC,EAAE,wCAAwC;IAC3C,CAAC,EAAE,aAAa;IAChB,CAAC,EAAE,yCAAyC;IAC5C,CAAC,EAAE,6CAA6C;IAChD,CAAC,EAAE,QAAQ;IACX,CAAC,EAAE,kBAAkB;IACrB,EAAE,EAAE,sCAAsC;IAC1C,EAAE,EAAE,wBAAwB;IAC5B,EAAE,EAAE,4BAA4B;CACjC,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,OAAO,KAAM,SAAQ,YAAY;IAQrC,YACU,QAAgB,EACjB,WAAmB,EAClB,OAAO;QACb,gBAAgB,EAAE,KAAK;KACxB;QAED,KAAK,EAAE,CAAC;QANA,aAAQ,GAAR,QAAQ,CAAQ;QACjB,gBAAW,GAAX,WAAW,CAAQ;QAClB,SAAI,GAAJ,IAAI,CAEX;QATK,cAAS,GAAW,IAAI,CAAC;QACzB,gBAAW,GAAW,IAAI,CAAC;QAC3B,YAAO,GAAY,KAAK,CAAC;IAUjC,CAAC;IAED,IAAI,GAAG;QACL,IAAI,IAAI,CAAC,YAAY,EAAE;YACrB,OAAO,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC;SAC9B;aAAM,IAAI,IAAI,CAAC,MAAM,EAAE;YACtB,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;SAC7B;aAAM;YACL,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;SACtD;IACH,CAAC;IAED,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED,IAAI,MAAM;QACR,IAAI,IAAI,CAAC,YAAY,EAAE;YACrB,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;SACjC;QACD,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,QAAQ,GAAG,MAAM,eAAe,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAEzD,IAAI,MAA6B,CAAC;QAElC,IAAI,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE;YAC9B,IAAI,CAAC,MAAM,GAAG,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE;gBAC/C,QAAQ;gBACR,KAAK,EAAE,IAAI;gBACX,MAAM,EAAE,IAAI;gBACZ,MAAM,EAAE,IAAI;aACb,CAAC,CAAC;SACJ;aAAM;YACL,IAAI,CAAC,YAAY,GAAG,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE;gBACnD,QAAQ;gBACR,KAAK,EAAE,MAAM;aACd,CAAC,CAAC;SACJ;QAED,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,QAAgB,EAAE,UAAmB,EAAE,EAAE;YAC1D,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;YAE1B,0DAA0D;YAC1D,UAAU,GAAG,OAAO,UAAU,KAAK,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC;YACnE,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;YAE9B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;YAEpB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;YAExC,qEAAqE;YACrE,MAAM,CAAC,kBAAkB,EAAE,CAAC;YAC5B,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC5B,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC;QAC7D,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAG,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC;QACjE,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC;QAE7D,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACnC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAEnC,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,GAAQ;QACjB,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC3C,IAAI,IAAI,CAAC,YAAY,EAAE;gBACrB,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,GAAiB,EAAE,EAAE;oBAChD,IAAI,GAAG,EAAE;wBACP,MAAM,CAAC,GAAG,CAAC,CAAC;qBACb;yBAAM;wBACL,OAAO,EAAE,CAAC;qBACX;gBACH,CAAC,CAAC,CAAC;aACJ;iBAAM,IAAI,IAAI,CAAC,MAAM,EAAE;gBACtB,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC;aACvC;iBAAM;gBACL,OAAO,EAAE,CAAC;aACX;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,WAAW,CAAC,SAAgC,SAAS;QAC3D,IAAI,IAAI,CAAC,YAAY,EAAE;YACrB,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;SAChC;aAAM,IAAI,IAAI,CAAC,MAAM,EAAE;YACtB,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;SACzB;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CACR,SAAgC,SAAS,EACzC,SAAkB;QAElB,IAAI,IAAI,CAAC,gBAAgB,EAAE,EAAE;YAC3B,OAAO;SACR;QAED,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC;QAC5D,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QAEzB,IAAI,SAAS,KAAK,SAAS,IAAI,CAAC,SAAS,KAAK,CAAC,IAAI,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE;YACvE,MAAM,aAAa,GAAG,UAAU,CAAC,GAAG,EAAE;gBACpC,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE;oBAC5B,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;iBAC7B;YACH,CAAC,EAAE,SAAS,CAAC,CAAC;YACd,MAAM,MAAM,CAAC;YACb,YAAY,CAAC,aAAa,CAAC,CAAC;SAC7B;QACD,MAAM,MAAM,CAAC;IACf,CAAC;IAEO,KAAK,CAAC,SAAS;QACrB,MAAM,UAAU,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACvD,MAAM,gBAAgB,GAAG,CAAC,GAAQ,EAAE,EAAE;gBACpC,IAAI,GAAG,CAAC,GAAG,KAAK,aAAa,CAAC,aAAa,EAAE;oBAC3C,OAAO,EAAE,CAAC;iBACX;qBAAM,IAAI,GAAG,CAAC,GAAG,KAAK,aAAa,CAAC,UAAU,EAAE;oBAC/C,MAAM,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC;oBACxB,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC;oBAC1B,GAAG,CAAC,OAAO,GAAG,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC;oBAC9B,MAAM,CAAC,GAAG,CAAC,CAAC;iBACb;gBACD,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAC;gBACtC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;YACpC,CAAC,CAAC;YAEF,MAAM,cAAc,GAAG,CAAC,IAAY,EAAE,MAAc,EAAE,EAAE;gBACtD,IAAI,IAAI,GAAG,GAAG,EAAE;oBACd,IAAI,IAAI,GAAG,CAAC;iBACb;gBACD,MAAM,GAAG,GAAG,eAAe,CAAC,IAAI,CAAC,IAAI,qBAAqB,IAAI,EAAE,CAAC;gBACjE,MAAM,CACJ,IAAI,KAAK,CAAC,6BAA6B,GAAG,eAAe,MAAM,EAAE,CAAC,CACnE,CAAC;gBACF,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAC;gBACtC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;YACpC,CAAC,CALine truncated
+10
View File
@@ -0,0 +1,10 @@
/**
* DelayedError
*
* Error to be thrown when job is moved to delayed state
* from job in active state.
*
*/
export declare class DelayedError extends Error {
constructor(message?: string);
}
+15
View File
@@ -0,0 +1,15 @@
/**
* DelayedError
*
* Error to be thrown when job is moved to delayed state
* from job in active state.
*
*/
export class DelayedError extends Error {
constructor(message) {
super(message);
this.name = this.constructor.name;
Object.setPrototypeOf(this, new.target.prototype);
}
}
//# sourceMappingURL=delayed-error.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"delayed-error.js","sourceRoot":"","sources":["../../../../src/classes/errors/delayed-error.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,MAAM,OAAO,YAAa,SAAQ,KAAK;IACrC,YAAY,OAAgB;QAC1B,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;QAClC,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACpD,CAAC;CACF"}
+4
View File
@@ -0,0 +1,4 @@
export * from './delayed-error';
export * from './unrecoverable-error';
export * from './rate-limit-error';
export * from './waiting-children-error';
+5
View File
@@ -0,0 +1,5 @@
export * from './delayed-error';
export * from './unrecoverable-error';
export * from './rate-limit-error';
export * from './waiting-children-error';
//# sourceMappingURL=index.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/classes/errors/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC;AAChC,cAAc,uBAAuB,CAAC;AACtC,cAAc,oBAAoB,CAAC;AACnC,cAAc,0BAA0B,CAAC"}
+10
View File
@@ -0,0 +1,10 @@
export declare const RATE_LIMIT_ERROR = "bullmq:rateLimitExceeded";
/**
* RateLimitError
*
* Error to be thrown when queue reaches a rate limit.
*
*/
export declare class RateLimitError extends Error {
constructor(message?: string);
}
+15
View File
@@ -0,0 +1,15 @@
export const RATE_LIMIT_ERROR = 'bullmq:rateLimitExceeded';
/**
* RateLimitError
*
* Error to be thrown when queue reaches a rate limit.
*
*/
export class RateLimitError extends Error {
constructor(message = RATE_LIMIT_ERROR) {
super(message);
this.name = this.constructor.name;
Object.setPrototypeOf(this, new.target.prototype);
}
}
//# sourceMappingURL=rate-limit-error.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"rate-limit-error.js","sourceRoot":"","sources":["../../../../src/classes/errors/rate-limit-error.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,gBAAgB,GAAG,0BAA0B,CAAC;AAE3D;;;;;GAKG;AACH,MAAM,OAAO,cAAe,SAAQ,KAAK;IACvC,YAAY,UAAkB,gBAAgB;QAC5C,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;QAClC,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACpD,CAAC;CACF"}
+10
View File
@@ -0,0 +1,10 @@
/**
* UnrecoverableError
*
* Error to move a job to failed even if the attemptsMade
* are lower than the expected limit.
*
*/
export declare class UnrecoverableError extends Error {
constructor(message?: string);
}
+15
View File
@@ -0,0 +1,15 @@
/**
* UnrecoverableError
*
* Error to move a job to failed even if the attemptsMade
* are lower than the expected limit.
*
*/
export class UnrecoverableError extends Error {
constructor(message) {
super(message);
this.name = this.constructor.name;
Object.setPrototypeOf(this, new.target.prototype);
}
}
//# sourceMappingURL=unrecoverable-error.js.map
@@ -0,0 +1 @@
{"version":3,"file":"unrecoverable-error.js","sourceRoot":"","sources":["../../../../src/classes/errors/unrecoverable-error.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,MAAM,OAAO,kBAAmB,SAAQ,KAAK;IAC3C,YAAY,OAAgB;QAC1B,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;QAClC,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACpD,CAAC;CACF"}
@@ -0,0 +1,10 @@
/**
* WaitingChildrenError
*
* Error to be thrown when job is moved to waiting-children state
* from job in active state.
*
*/
export declare class WaitingChildrenError extends Error {
constructor(message?: string);
}
+15
View File
@@ -0,0 +1,15 @@
/**
* WaitingChildrenError
*
* Error to be thrown when job is moved to waiting-children state
* from job in active state.
*
*/
export class WaitingChildrenError extends Error {
constructor(message) {
super(message);
this.name = this.constructor.name;
Object.setPrototypeOf(this, new.target.prototype);
}
}
//# sourceMappingURL=waiting-children-error.js.map
@@ -0,0 +1 @@
{"version":3,"file":"waiting-children-error.js","sourceRoot":"","sources":["../../../../src/classes/errors/waiting-children-error.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IAC7C,YAAY,OAAgB;QAC1B,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;QAClC,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACpD,CAAC;CACF"}
+175
View File
@@ -0,0 +1,175 @@
/// <reference types="node" />
import { EventEmitter } from 'events';
import { ChainableCommander } from 'ioredis';
import { FlowJob, FlowQueuesOpts, FlowOpts, IoredisListener, QueueBaseOptions, RedisClient } from '../interfaces';
import { Job } from './job';
import { KeysMap, QueueKeys } from './queue-keys';
import { RedisConnection } from './redis-connection';
export interface AddNodeOpts {
multi: ChainableCommander;
node: FlowJob;
parent?: {
parentOpts: {
id: string;
queue: string;
};
parentDependenciesKey: string;
};
/**
* Queues options that will be applied in each node depending on queue name presence.
*/
queuesOpts?: FlowQueuesOpts;
}
export interface AddChildrenOpts {
multi: ChainableCommander;
nodes: FlowJob[];
parent: {
parentOpts: {
id: string;
queue: string;
};
parentDependenciesKey: string;
};
queuesOpts?: FlowQueuesOpts;
}
export interface NodeOpts {
/**
* Root job queue name.
*/
queueName: string;
/**
* Prefix included in job key.
*/
prefix?: string;
/**
* Root job id.
*/
id: string;
/**
* Maximum depth or levels to visit in the tree.
*/
depth?: number;
/**
* Maximum quantity of children per type (processed, unprocessed).
*/
maxChildren?: number;
}
export interface JobNode {
job: Job;
children?: JobNode[];
}
export interface FlowProducerListener extends IoredisListener {
/**
* Listen to 'error' event.
*
* This event is triggered when an error is throw.
*/
error: (failedReason: Error) => void;
}
/**
* This class allows to add jobs with dependencies between them in such
* a way that it is possible to build complex flows.
* Note: A flow is a tree-like structure of jobs that depend on each other.
* Whenever the children of a given parent are completed, the parent
* will be processed, being able to access the children's result data.
* All Jobs can be in different queues, either children or parents,
*/
export declare class FlowProducer extends EventEmitter {
opts: QueueBaseOptions;
toKey: (name: string, type: string) => string;
keys: KeysMap;
closing: Promise<void> | undefined;
queueKeys: QueueKeys;
protected connection: RedisConnection;
constructor(opts?: QueueBaseOptions, Connection?: typeof RedisConnection);
emit<U extends keyof FlowProducerListener>(event: U, ...args: Parameters<FlowProducerListener[U]>): boolean;
off<U extends keyof FlowProducerListener>(eventName: U, listener: FlowProducerListener[U]): this;
on<U extends keyof FlowProducerListener>(event: U, listener: FlowProducerListener[U]): this;
once<U extends keyof FlowProducerListener>(event: U, listener: FlowProducerListener[U]): this;
/**
* Returns a promise that resolves to a redis client. Normally used only by subclasses.
*/
get client(): Promise<RedisClient>;
/**
* Helper to easily extend Job class calls.
*/
protected get Job(): typeof Job;
waitUntilReady(): Promise<RedisClient>;
/**
* Adds a flow.
*
* This call would be atomic, either it fails and no jobs will
* be added to the queues, or it succeeds and all jobs will be added.
*
* @param flow - an object with a tree-like structure where children jobs
* will be processed before their parents.
* @param opts - options that will be applied to the flow object.
*/
add(flow: FlowJob, opts?: FlowOpts): Promise<JobNode>;
/**
* Get a flow.
*
* @param opts - an object with options for getting a JobNode.
*/
getFlow(opts: NodeOpts): Promise<JobNode>;
/**
* Adds multiple flows.
*
* A flow is a tree-like structure of jobs that depend on each other.
* Whenever the children of a given parent are completed, the parent
* will be processed, being able to access the children's result data.
*
* All Jobs can be in different queues, either children or parents,
* however this call would be atomic, either it fails and no jobs will
* be added to the queues, or it succeeds and all jobs will be added.
*
* @param flows - an array of objects with a tree-like structure where children jobs
* will be processed before their parents.
*/
addBulk(flows: FlowJob[]): Promise<JobNode[]>;
/**
* Add a node (job) of a flow to the queue. This method will recursively
* add all its children as well. Note that a given job can potentially be
* a parent and a child job at the same time depending on where it is located
* in the tree hierarchy.
*
* @param multi - ioredis ChainableCommander
* @param node - the node representing a job to be added to some queue
* @param parent - parent data sent to children to create the "links" to their parent
* @returns
*/
protected addNode({ multi, node, parent, queuesOpts }: AddNodeOpts): JobNode;
/**
* Adds nodes (jobs) of multiple flows to the queue. This method will recursively
* add all its children as well. Note that a given job can potentially be
* a parent and a child job at the same time depending on where it is located
* in the tree hierarchy.
*
* @param multi - ioredis ChainableCommander
* @param nodes - the nodes representing jobs to be added to some queue
* @returns
*/
protected addNodes(multi: ChainableCommander, nodes: FlowJob[]): JobNode[];
private getNode;
private addChildren;
private getChildren;
/**
* Helper factory method that creates a queue-like object
* required to create jobs in any queue.
*
* @param node -
* @param queueKeys -
* @returns
*/
private queueFromNode;
/**
*
* Closes the connection and returns a promise that resolves when the connection is closed.
*/
close(): Promise<void>;
/**
*
* Force disconnects a connection.
*/
disconnect(): Promise<void>;
}
+305
View File
@@ -0,0 +1,305 @@
import { EventEmitter } from 'events';
import { get } from 'lodash';
import { v4 } from 'uuid';
import { getParentKey, isRedisInstance } from '../utils';
import { Job } from './job';
import { QueueKeys } from './queue-keys';
import { RedisConnection } from './redis-connection';
/**
* This class allows to add jobs with dependencies between them in such
* a way that it is possible to build complex flows.
* Note: A flow is a tree-like structure of jobs that depend on each other.
* Whenever the children of a given parent are completed, the parent
* will be processed, being able to access the children's result data.
* All Jobs can be in different queues, either children or parents,
*/
export class FlowProducer extends EventEmitter {
constructor(opts = {}, Connection = RedisConnection) {
super();
this.opts = opts;
this.opts = Object.assign({ prefix: 'bull' }, opts);
if (!opts.connection) {
console.warn([
'BullMQ: DEPRECATION WARNING! Optional instantiation of Queue, Worker, QueueEvents and FlowProducer',
'without providing explicitly a connection or connection options is deprecated. This behaviour will',
'be removed in the next major release',
].join(' '));
}
this.connection = new Connection(opts.connection, isRedisInstance(opts === null || opts === void 0 ? void 0 : opts.connection), false, opts.skipVersionCheck);
this.connection.on('error', (error) => this.emit('error', error));
this.connection.on('close', () => {
if (!this.closing) {
this.emit('ioredis:close');
}
});
this.queueKeys = new QueueKeys(opts.prefix);
}
emit(event, ...args) {
return super.emit(event, ...args);
}
off(eventName, listener) {
super.off(eventName, listener);
return this;
}
on(event, listener) {
super.on(event, listener);
return this;
}
once(event, listener) {
super.once(event, listener);
return this;
}
/**
* Returns a promise that resolves to a redis client. Normally used only by subclasses.
*/
get client() {
return this.connection.client;
}
/**
* Helper to easily extend Job class calls.
*/
get Job() {
return Job;
}
waitUntilReady() {
return this.client;
}
/**
* Adds a flow.
*
* This call would be atomic, either it fails and no jobs will
* be added to the queues, or it succeeds and all jobs will be added.
*
* @param flow - an object with a tree-like structure where children jobs
* will be processed before their parents.
* @param opts - options that will be applied to the flow object.
*/
async add(flow, opts) {
var _a;
if (this.closing) {
return;
}
const client = await this.connection.client;
const multi = client.multi();
const parentOpts = (_a = flow === null || flow === void 0 ? void 0 : flow.opts) === null || _a === void 0 ? void 0 : _a.parent;
const parentKey = getParentKey(parentOpts);
const parentDependenciesKey = parentKey
? `${parentKey}:dependencies`
: undefined;
const jobsTree = this.addNode({
multi,
node: flow,
queuesOpts: opts === null || opts === void 0 ? void 0 : opts.queuesOptions,
parent: {
parentOpts,
parentDependenciesKey,
},
});
await multi.exec();
return jobsTree;
}
/**
* Get a flow.
*
* @param opts - an object with options for getting a JobNode.
*/
async getFlow(opts) {
if (this.closing) {
return;
}
const client = await this.connection.client;
const updatedOpts = Object.assign({
depth: 10,
maxChildren: 20,
}, opts);
const jobsTree = this.getNode(client, updatedOpts);
return jobsTree;
}
/**
* Adds multiple flows.
*
* A flow is a tree-like structure of jobs that depend on each other.
* Whenever the children of a given parent are completed, the parent
* will be processed, being able to access the children's result data.
*
* All Jobs can be in different queues, either children or parents,
* however this call would be atomic, either it fails and no jobs will
* be added to the queues, or it succeeds and all jobs will be added.
*
* @param flows - an array of objects with a tree-like structure where children jobs
* will be processed before their parents.
*/
async addBulk(flows) {
if (this.closing) {
return;
}
const client = await this.connection.client;
const multi = client.multi();
const jobsTrees = this.addNodes(multi, flows);
await multi.exec();
return jobsTrees;
}
/**
* Add a node (job) of a flow to the queue. This method will recursively
* add all its children as well. Note that a given job can potentially be
* a parent and a child job at the same time depending on where it is located
* in the tree hierarchy.
*
* @param multi - ioredis ChainableCommander
* @param node - the node representing a job to be added to some queue
* @param parent - parent data sent to children to create the "links" to their parent
* @returns
*/
addNode({ multi, node, parent, queuesOpts }) {
var _a;
const prefix = node.prefix || this.opts.prefix;
const queue = this.queueFromNode(node, new QueueKeys(prefix), prefix);
const queueOpts = queuesOpts && queuesOpts[node.queueName];
const jobsOpts = get(queueOpts, 'defaultJobOptions');
const jobId = ((_a = node.opts) === null || _a === void 0 ? void 0 : _a.jobId) || v4();
const job = new this.Job(queue, node.name, node.data, Object.assign(Object.assign(Object.assign({}, (jobsOpts ? jobsOpts : {})), node.opts), { parent: parent === null || parent === void 0 ? void 0 : parent.parentOpts }), jobId);
const parentKey = getParentKey(parent === null || parent === void 0 ? void 0 : parent.parentOpts);
if (node.children && node.children.length > 0) {
// Create parent job, will be a job in status "waiting-children".
const parentId = jobId;
const queueKeysParent = new QueueKeys(node.prefix || this.opts.prefix);
const waitChildrenKey = queueKeysParent.toKey(node.queueName, 'waiting-children');
job.addJob(multi, {
parentDependenciesKey: parent === null || parent === void 0 ? void 0 : parent.parentDependenciesKey,
waitChildrenKey,
parentKey,
});
const parentDependenciesKey = `${queueKeysParent.toKey(node.queueName, parentId)}:dependencies`;
const children = this.addChildren({
multi,
nodes: node.children,
parent: {
parentOpts: {
id: parentId,
queue: queueKeysParent.getQueueQualifiedName(node.queueName),
},
parentDependenciesKey,
},
queuesOpts,
});
return { job, children };
}
else {
job.addJob(multi, {
parentDependenciesKey: parent === null || parent === void 0 ? void 0 : parent.parentDependenciesKey,
parentKey,
});
return { job };
}
}
/**
* Adds nodes (jobs) of multiple flows to the queue. This method will recursively
* add all its children as well. Note that a given job can potentially be
* a parent and a child job at the same time depending on where it is located
* in the tree hierarchy.
*
* @param multi - ioredis ChainableCommander
* @param nodes - the nodes representing jobs to be added to some queue
* @returns
*/
addNodes(multi, nodes) {
return nodes.map(node => {
var _a;
const parentOpts = (_a = node === null || node === void 0 ? void 0 : node.opts) === null || _a === void 0 ? void 0 : _a.parent;
const parentKey = getParentKey(parentOpts);
const parentDependenciesKey = parentKey
? `${parentKey}:dependencies`
: undefined;
return this.addNode({
multi,
node,
parent: {
parentOpts,
parentDependenciesKey,
},
});
});
}
async getNode(client, node) {
const queue = this.queueFromNode(node, new QueueKeys(node.prefix), node.prefix);
const job = await this.Job.fromId(queue, node.id);
if (job) {
const { processed = {}, unprocessed = [] } = await job.getDependencies({
processed: {
count: node.maxChildren,
},
unprocessed: {
count: node.maxChildren,
},
});
const processedKeys = Object.keys(processed);
const childrenCount = processedKeys.length + unprocessed.length;
const newDepth = node.depth - 1;
if (childrenCount > 0 && newDepth) {
const children = await this.getChildren(client, [...processedKeys, ...unprocessed], newDepth, node.maxChildren);
return { job, children };
}
else {
return { job };
}
}
}
addChildren({ multi, nodes, parent, queuesOpts }) {
return nodes.map(node => this.addNode({ multi, node, parent, queuesOpts }));
}
getChildren(client, childrenKeys, depth, maxChildren) {
const getChild = (key) => {
const [prefix, queueName, id] = key.split(':');
return this.getNode(client, {
id,
queueName,
prefix,
depth,
maxChildren,
});
};
return Promise.all([...childrenKeys.map(getChild)]);
}
/**
* Helper factory method that creates a queue-like object
* required to create jobs in any queue.
*
* @param node -
* @param queueKeys -
* @returns
*/
queueFromNode(node, queueKeys, prefix) {
return {
client: this.connection.client,
name: node.queueName,
keys: queueKeys.getKeys(node.queueName),
toKey: (type) => queueKeys.toKey(node.queueName, type),
opts: { prefix },
qualifiedName: queueKeys.getQueueQualifiedName(node.queueName),
closing: this.closing,
waitUntilReady: async () => this.connection.client,
removeListener: this.removeListener.bind(this),
emit: this.emit.bind(this),
on: this.on.bind(this),
redisVersion: this.connection.redisVersion,
};
}
/**
*
* Closes the connection and returns a promise that resolves when the connection is closed.
*/
async close() {
if (!this.closing) {
this.closing = this.connection.close();
}
await this.closing;
}
/**
*
* Force disconnects a connection.
*/
disconnect() {
return this.connection.disconnect();
}
}
//# sourceMappingURL=flow-producer.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"flow-producer.js","sourceRoot":"","sources":["../../../src/classes/flow-producer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAC;AAE7B,OAAO,EAAE,EAAE,EAAE,MAAM,MAAM,CAAC;AAS1B,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AACzD,OAAO,EAAE,GAAG,EAAE,MAAM,OAAO,CAAC;AAC5B,OAAO,EAAW,SAAS,EAAE,MAAM,cAAc,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAoErD;;;;;;;GAOG;AACH,MAAM,OAAO,YAAa,SAAQ,YAAY;IAQ5C,YACS,OAAyB,EAAE,EAClC,aAAqC,eAAe;QAEpD,KAAK,EAAE,CAAC;QAHD,SAAI,GAAJ,IAAI,CAAuB;QAKlC,IAAI,CAAC,IAAI,mBACP,MAAM,EAAE,MAAM,IACX,IAAI,CACR,CAAC;QAEF,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE;YACpB,OAAO,CAAC,IAAI,CACV;gBACE,oGAAoG;gBACpG,oGAAoG;gBACpG,sCAAsC;aACvC,CAAC,IAAI,CAAC,GAAG,CAAC,CACZ,CAAC;SACH;QAED,IAAI,CAAC,UAAU,GAAG,IAAI,UAAU,CAC9B,IAAI,CAAC,UAAU,EACf,eAAe,CAAC,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,UAAU,CAAC,EACjC,KAAK,EACL,IAAI,CAAC,gBAAgB,CACtB,CAAC;QAEF,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAY,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;QACzE,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YAC/B,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;gBACjB,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;aAC5B;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,SAAS,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC9C,CAAC;IAED,IAAI,CACF,KAAQ,EACR,GAAG,IAAyC;QAE5C,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,CAAC;IACpC,CAAC;IAED,GAAG,CACD,SAAY,EACZ,QAAiC;QAEjC,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAC/B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,EAAE,CACA,KAAQ,EACR,QAAiC;QAEjC,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAC1B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CACF,KAAQ,EACR,QAAiC;QAEjC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAC5B,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,IAAc,GAAG;QACf,OAAO,GAAG,CAAC;IACb,CAAC;IAED,cAAc;QACZ,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,GAAG,CAAC,IAAa,EAAE,IAAe;;QACtC,IAAI,IAAI,CAAC,OAAO,EAAE;YAChB,OAAO;SACR;QACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;QAC5C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC;QAE7B,MAAM,UAAU,GAAG,MAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,IAAI,0CAAE,MAAM,CAAC;QACtC,MAAM,SAAS,GAAG,YAAY,CAAC,UAAU,CAAC,CAAC;QAC3C,MAAM,qBAAqB,GAAG,SAAS;YACrC,CAAC,CAAC,GAAG,SAAS,eAAe;YAC7B,CAAC,CAAC,SAAS,CAAC;QAEd,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC;YAC5B,KAAK;YACL,IAAI,EAAE,IAAI;YACV,UAAU,EAAE,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,aAAa;YAC/B,MAAM,EAAE;gBACN,UAAU;gBACV,qBAAqB;aACtB;SACF,CAAC,CAAC;QAEH,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QAEnB,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO,CAAC,IAAc;QAC1B,IAAI,IAAI,CAAC,OAAO,EAAE;YAChB,OAAO;SACR;QACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;QAE5C,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAC/B;YACE,KAAK,EAAE,EAAE;YACT,WAAW,EAAE,EAAE;SAChB,EACD,IAAI,CACL,CAAC;QACF,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QAEnD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,OAAO,CAAC,KAAgB;QAC5B,IAAI,IAAI,CAAC,OAAO,EAAE;YAChB,OAAO;SACR;QACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;QAC5C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC;QAE7B,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAE9C,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QAEnB,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;;;;;;;;;OAUG;IACO,OAAO,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAe;;QAChE,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;QAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,IAAI,SAAS,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;QACtE,MAAM,SAAS,GAAG,UAAU,IAAI,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAE3D,MAAM,QAAQ,GAAG,GAAG,CAAC,SAAS,EAAE,mBAAmB,CAAC,CAAC;QACrD,MAAM,KAAK,GAAG,CAAA,MAAA,IAAI,CAAC,IAAI,0CAAE,KAAK,KAAI,EAAE,EAAE,CAAC;QAEvC,MAAM,GAAG,GAAG,IAAI,IAAI,CAAC,GAAG,CACtB,KAAK,EACL,IAAI,CAAC,IAAI,EACT,IAAI,CAAC,IAAI,gDAEJ,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,GAC1B,IAAI,CAAC,IAAI,KACZ,MAAM,EAAE,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,UAAU,KAE5B,KAAK,CACN,CAAC;QAEF,MAAM,SAAS,GAAG,YAAY,CAAC,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,UAAU,CAAC,CAAC;QAEnD,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE;YAC7C,iEAAiE;YACjE,MAAM,QAAQ,GAAG,KAAK,CAAC;YACvB,MAAM,eAAe,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvE,MAAM,eAAe,GAAG,eAAe,CAAC,KAAK,CAC3C,IAAI,CAAC,SAAS,EACd,kBAAkB,CACnB,CAAC;YAEF,GAAG,CAAC,MAAM,CAAS,KAAiB,EAAE;gBACpC,qBAAqB,EAAE,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,qBAAqB;gBACpD,eAAe;gBACf,SAAS;aACV,CAAC,CAAC;YAEH,MAAM,qBAAqB,GAAG,GAAG,eAAe,CAAC,KAAK,CACpD,IAAI,CAAC,SAAS,EACd,QAAQ,CACT,eAAe,CAAC;YAEjB,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC;gBAChC,KAAK;gBACL,KAAK,EAAE,IAAI,CAAC,QAAQ;gBACpB,MAAM,EAAE;oBACN,UAAU,EAAE;wBACV,EAAE,EAAE,QAAQ;wBACZ,KAAK,EAAE,eAAe,CAAC,qBAAqB,CAAC,IAAI,CAAC,SAAS,CAAC;qBAC7D;oBACD,qBAAqB;iBACtB;gBACD,UAAU;aACX,CAAC,CAAC;YAEH,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC;SAC1B;aAAM;YACL,GAAG,CAAC,MAAM,CAAS,KAAiB,EAAE;gBACpC,qBAAqB,EAAE,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,qBAAqB;gBACpD,SAAS;aACV,CAAC,CAAC;YAEH,OAAO,EAAE,GAAG,EAAE,CAAC;SAChB;IALine truncated
+17
View File
@@ -0,0 +1,17 @@
export * from './async-fifo-queue';
export * from './backoffs';
export * from './child-pool';
export * from './child-processor';
export * from './errors';
export * from './flow-producer';
export * from './job';
export * from './queue-base';
export * from './queue-events';
export * from './queue-getters';
export * from './queue-keys';
export * from './queue';
export * from './redis-connection';
export * from './repeat';
export * from './sandbox';
export * from './scripts';
export * from './worker';
+20
View File
@@ -0,0 +1,20 @@
export * from './async-fifo-queue';
export * from './backoffs';
export * from './child-pool';
export * from './child-processor';
export * from './errors';
export * from './flow-producer';
export * from './job';
// export * from './main'; this file must not be exported
// export * from './main-worker'; this file must not be exported
export * from './queue-base';
export * from './queue-events';
export * from './queue-getters';
export * from './queue-keys';
export * from './queue';
export * from './redis-connection';
export * from './repeat';
export * from './sandbox';
export * from './scripts';
export * from './worker';
//# sourceMappingURL=index.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/classes/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC;AAC7B,cAAc,mBAAmB,CAAC;AAClC,cAAc,UAAU,CAAC;AACzB,cAAc,iBAAiB,CAAC;AAChC,cAAc,OAAO,CAAC;AACtB,yDAAyD;AACzD,gEAAgE;AAChE,cAAc,cAAc,CAAC;AAC7B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC;AAChC,cAAc,cAAc,CAAC;AAC7B,cAAc,SAAS,CAAC;AACxB,cAAc,oBAAoB,CAAC;AACnC,cAAc,UAAU,CAAC;AACzB,cAAc,WAAW,CAAC;AAC1B,cAAc,WAAW,CAAC;AAC1B,cAAc,UAAU,CAAC"}
+378
View File
@@ -0,0 +1,378 @@
import { ChainableCommander } from 'ioredis';
import { BulkJobOptions, DependenciesOpts, JobJson, JobJsonRaw, MinimalJob, MoveToWaitingChildrenOpts, ParentKeys, ParentOpts, RedisClient } from '../interfaces';
import { FinishedStatus, JobsOptions, JobState, JobJsonSandbox, MinimalQueue } from '../types';
import { Scripts } from './scripts';
import type { QueueEvents } from './queue-events';
export declare const PRIORITY_LIMIT: number;
/**
* Job
*
* This class represents a Job in the queue. Normally job are implicitly created when
* you add a job to the queue with methods such as Queue.addJob( ... )
*
* A Job instance is also passed to the Worker's process function.
*
* @class Job
*/
export declare class Job<DataType = any, ReturnType = any, NameType extends string = string> implements MinimalJob<DataType, ReturnType, NameType> {
protected queue: MinimalQueue;
/**
* The name of the Job
*/
name: NameType;
/**
* The payload for this job.
*/
data: DataType;
/**
* The options object for this job.
*/
opts: JobsOptions;
id?: string;
/**
* It includes the prefix, the namespace separator :, and queue name.
* @see https://www.gnu.org/software/gawk/manual/html_node/Qualified-Names.html
*/
readonly queueQualifiedName: string;
/**
* The progress a job has performed so far.
* @defaultValue 0
*/
progress: number | object;
/**
* The value returned by the processor when processing this job.
* @defaultValue null
*/
returnvalue: ReturnType;
/**
* Stacktrace for the error (for failed jobs).
* @defaultValue null
*/
stacktrace: string[];
/**
* An amount of milliseconds to wait until this job can be processed.
* @defaultValue 0
*/
delay: number;
/**
* Timestamp when the job was created (unless overridden with job options).
*/
timestamp: number;
/**
* Number of attempts after the job has failed.
* @defaultValue 0
*/
attemptsMade: number;
/**
* Reason for failing.
*/
failedReason: string;
/**
* Timestamp for when the job finished (completed or failed).
*/
finishedOn?: number;
/**
* Timestamp for when the job was processed.
*/
processedOn?: number;
/**
* Fully qualified key (including the queue prefix) pointing to the parent of this job.
*/
parentKey?: string;
/**
* Object that contains parentId (id) and parent queueKey.
*/
parent?: ParentKeys;
/**
* Base repeat job key.
*/
repeatJobKey?: string;
/**
* The token used for locking this job.
*/
token?: string;
protected toKey: (type: string) => string;
protected discarded: boolean;
protected scripts: Scripts;
constructor(queue: MinimalQueue,
/**
* The name of the Job
*/
name: NameType,
/**
* The payload for this job.
*/
data: DataType,
/**
* The options object for this job.
*/
opts?: JobsOptions, id?: string);
/**
* Creates a new job and adds it to the queue.
*
* @param queue - the queue where to add the job.
* @param name - the name of the job.
* @param data - the payload of the job.
* @param opts - the options bag for this job.
* @returns
*/
static create<T = any, R = any, N extends string = string>(queue: MinimalQueue, name: N, data: T, opts?: JobsOptions): Promise<Job<T, R, N>>;
/**
* Creates a bulk of jobs and adds them atomically to the given queue.
*
* @param queue -the queue were to add the jobs.
* @param jobs - an array of jobs to be added to the queue.
* @returns
*/
static createBulk<T = any, R = any, N extends string = string>(queue: MinimalQueue, jobs: {
name: N;
data: T;
opts?: BulkJobOptions;
}[]): Promise<Job<T, R, N>[]>;
/**
* Instantiates a Job from a JobJsonRaw object (coming from a deserialized JSON object)
*
* @param queue - the queue where the job belongs to.
* @param json - the plain object containing the job.
* @param jobId - an optional job id (overrides the id coming from the JSON object)
* @returns
*/
static fromJSON<T = any, R = any, N extends string = string>(queue: MinimalQueue, json: JobJsonRaw, jobId?: string): Job<T, R, N>;
private static optsFromJSON;
/**
* Fetches a Job from the queue given the passed job id.
*
* @param queue - the queue where the job belongs to.
* @param jobId - the job id.
* @returns
*/
static fromId<T = any, R = any, N extends string = string>(queue: MinimalQueue, jobId: string): Promise<Job<T, R, N> | undefined>;
/**
* addJobLog
*
* @param queue Queue instance
* @param jobId Job id
* @param logRow Log row
* @param keepLogs optional maximum number of logs to keep
*
* @returns The total number of log entries for this job so far.
*/
static addJobLog(queue: MinimalQueue, jobId: string, logRow: string, keepLogs?: number): Promise<number>;
toJSON(): Omit<this, "toJSON" | "scripts" | "changeDelay" | "changePriority" | "extendLock" | "getState" | "moveToDelayed" | "moveToWaitingChildren" | "promote" | "updateData" | "updateProgress" | "discard" | "addJob" | "prefix" | "queue" | "asJSON" | "asJSONSandbox" | "log" | "clearLogs" | "remove" | "moveToCompleted" | "moveToFailed" | "isCompleted" | "isFailed" | "isDelayed" | "isWaitingChildren" | "isActive" | "isWaiting" | "queueName" | "getChildrenValues" | "getDependencies" | "getDependenciesCount" | "waitUntilFinished" | "retry">;
/**
* Prepares a job to be serialized for storage in Redis.
* @returns
*/
asJSON(): JobJson;
private optsAsJSON;
/**
* Prepares a job to be passed to Sandbox.
* @returns
*/
asJSONSandbox(): JobJsonSandbox;
/**
* Updates a job's data
*
* @param data - the data that will replace the current jobs data.
*/
updateData(data: DataType): Promise<void>;
/**
* Updates a job's progress
*
* @param progress - number or object to be saved as progress.
*/
updateProgress(progress: number | object): Promise<void>;
/**
* Logs one row of log data.
*
* @param logRow - string with log data to be logged.
* @returns The total number of log entries for this job so far.
*/
log(logRow: string): Promise<number>;
/**
* Clears job's logs
*
* @param keepLogs - the amount of log entries to preserve
*/
clearLogs(keepLogs?: number): Promise<void>;
/**
* Completely remove the job from the queue.
* Note, this call will throw an exception if the job
* is being processed when the call is performed.
*
* @param opts - Options to remove a job
*/
remove({ removeChildren }?: {
removeChildren?: boolean;
}): Promise<void>;
/**
* Extend the lock for this job.
*
* @param token - unique token for the lock
* @param duration - lock duration in milliseconds
*/
extendLock(token: string, duration: number): Promise<number>;
/**
* Moves a job to the completed queue.
* Returned job to be used with Queue.prototype.nextJobFromJobData.
*
* @param returnValue - The jobs success message.
* @param token - Worker token used to acquire completed job.
* @param fetchNext - True when wanting to fetch the next job.
* @returns Returns the jobData of the next job in the waiting queue.
*/
moveToCompleted(returnValue: ReturnType, token: string, fetchNext?: boolean): Promise<any[]>;
/**
* Moves a job to the failed queue.
*
* @param err - the jobs error message.
* @param token - token to check job is locked by current worker
* @param fetchNext - true when wanting to fetch the next job
* @returns void
*/
moveToFailed<E extends Error>(err: E, token: string, fetchNext?: boolean): Promise<void>;
/**
* @returns true if the job has completed.
*/
isCompleted(): Promise<boolean>;
/**
* @returns true if the job has failed.
*/
isFailed(): Promise<boolean>;
/**
* @returns true if the job is delayed.
*/
isDelayed(): Promise<boolean>;
/**
* @returns true if the job is waiting for children.
*/
isWaitingChildren(): Promise<boolean>;
/**
* @returns true of the job is active.
*/
isActive(): Promise<boolean>;
/**
* @returns true if the job is waiting.
*/
isWaiting(): Promise<boolean>;
/**
* @returns the queue name this job belongs to.
*/
get queueName(): string;
/**
* @returns the prefix that is used.
*/
get prefix(): string;
/**
* Get current state.
*
* @returns Returns one of these values:
* 'completed', 'failed', 'delayed', 'active', 'waiting', 'waiting-children', 'unknown'.
*/
getState(): Promise<JobState | 'unknown'>;
/**
* Change delay of a delayed job.
*
* @param delay - milliseconds to be added to current time.
* @returns void
*/
changeDelay(delay: number): Promise<void>;
/**
* Change job priority.
*
* @returns void
*/
changePriority(opts: {
priority?: number;
lifo?: boolean;
}): Promise<void>;
/**
* Get this jobs children result values if any.
*
* @returns Object mapping children job keys with their values.
*/
getChildrenValues<CT = any>(): Promise<{
[jobKey: string]: CT;
}>;
/**
* Get children job keys if this job is a parent and has children.
* @remarks
* Count options before Redis v7.2 works as expected with any quantity of entries
* on processed/unprocessed dependencies, since v7.2 you must consider that count
* won't have any effect until processed/unprocessed dependencies have a length
* greater than 127
* @see https://redis.io/docs/management/optimization/memory-optimization/#redis--72
* @returns dependencies separated by processed and unprocessed.
*/
getDependencies(opts?: DependenciesOpts): Promise<{
nextProcessedCursor?: number;
processed?: Record<string, any>;
nextUnprocessedCursor?: number;
unprocessed?: string[];
}>;
/**
* Get children job counts if this job is a parent and has children.
*
* @returns dependencies count separated by processed and unprocessed.
*/
getDependenciesCount(opts?: {
processed?: boolean;
unprocessed?: boolean;
}): Promise<{
processed?: number;
unprocessed?: number;
}>;
/**
* Returns a promise the resolves when the job has completed (containing the return value of the job),
* or rejects when the job has failed (containing the failedReason).
*
* @param queueEvents - Instance of QueueEvents.
* @param ttl - Time in milliseconds to wait for job to finish before timing out.
*/
waitUntilFinished(queueEvents: QueueEvents, ttl?: number): Promise<ReturnType>;
/**
* Moves the job to the delay set.
*
* @param timestamp - timestamp where the job should be moved back to "wait"
* @param token - token to check job is locked by current worker
* @returns
*/
moveToDelayed(timestamp: number, token?: string): Promise<void>;
/**
* Moves the job to the waiting-children set.
*
* @param token - Token to check job is locked by current worker
* @param opts - The options bag for moving a job to waiting-children.
* @returns true if the job was moved
*/
moveToWaitingChildren(token: string, opts?: MoveToWaitingChildrenOpts): Promise<boolean>;
/**
* Promotes a delayed job so that it starts to be processed as soon as possible.
*/
promote(): Promise<void>;
/**
* Attempts to retry the job. Only a job that has failed or completed can be retried.
*
* @param state - completed / failed
* @returns If resolved and return code is 1, then the queue emits a waiting event
* otherwise the operation was not a success and throw the corresponding error. If the promise
* rejects, it indicates that the script failed to execute
*/
retry(state?: FinishedStatus): Promise<void>;
/**
* Marks a job to not be retried if it fails (even if attempts has been configured)
*/
discard(): void;
private isInZSet;
private isInList;
/**
* Adds the job to Redis.
*
* @param client -
* @param parentOpts -
* @returns
*/
addJob(client: RedisClient, parentOpts?: ParentOpts): Promise<string>;
protected validateOptions(jobData: JobJson): void;
protected saveStacktrace(multi: ChainableCommander, err: Error): void;
}
+787
View File
@@ -0,0 +1,787 @@
import { __rest } from "tslib";
import { invert } from 'lodash';
import { debuglog } from 'util';
import { errorObject, isEmpty, getParentKey, lengthInUtf8Bytes, parseObjectValues, tryCatch, } from '../utils';
import { Backoffs } from './backoffs';
import { Scripts } from './scripts';
import { UnrecoverableError } from './errors/unrecoverable-error';
const logger = debuglog('bull');
const optsDecodeMap = {
fpof: 'failParentOnFailure',
kl: 'keepLogs',
rdof: 'removeDependencyOnFailure',
};
const optsEncodeMap = invert(optsDecodeMap);
export const PRIORITY_LIMIT = 2 ** 21;
/**
* Job
*
* This class represents a Job in the queue. Normally job are implicitly created when
* you add a job to the queue with methods such as Queue.addJob( ... )
*
* A Job instance is also passed to the Worker's process function.
*
* @class Job
*/
export class Job {
constructor(queue,
/**
* The name of the Job
*/
name,
/**
* The payload for this job.
*/
data,
/**
* The options object for this job.
*/
opts = {}, id) {
this.queue = queue;
this.name = name;
this.data = data;
this.opts = opts;
this.id = id;
/**
* The progress a job has performed so far.
* @defaultValue 0
*/
this.progress = 0;
/**
* The value returned by the processor when processing this job.
* @defaultValue null
*/
this.returnvalue = null;
/**
* Stacktrace for the error (for failed jobs).
* @defaultValue null
*/
this.stacktrace = null;
/**
* Number of attempts after the job has failed.
* @defaultValue 0
*/
this.attemptsMade = 0;
const _a = this.opts, { repeatJobKey } = _a, restOpts = __rest(_a, ["repeatJobKey"]);
this.opts = Object.assign({
attempts: 0,
delay: 0,
}, restOpts);
this.delay = this.opts.delay;
this.repeatJobKey = repeatJobKey;
this.timestamp = opts.timestamp ? opts.timestamp : Date.now();
this.opts.backoff = Backoffs.normalize(opts.backoff);
this.parentKey = getParentKey(opts.parent);
this.parent = opts.parent
? { id: opts.parent.id, queueKey: opts.parent.queue }
: undefined;
this.toKey = queue.toKey.bind(queue);
this.scripts = new Scripts(queue);
this.queueQualifiedName = queue.qualifiedName;
}
/**
* Creates a new job and adds it to the queue.
*
* @param queue - the queue where to add the job.
* @param name - the name of the job.
* @param data - the payload of the job.
* @param opts - the options bag for this job.
* @returns
*/
static async create(queue, name, data, opts) {
const client = await queue.client;
const job = new this(queue, name, data, opts, opts && opts.jobId);
job.id = await job.addJob(client, {
parentKey: job.parentKey,
parentDependenciesKey: job.parentKey
? `${job.parentKey}:dependencies`
: '',
});
return job;
}
/**
* Creates a bulk of jobs and adds them atomically to the given queue.
*
* @param queue -the queue were to add the jobs.
* @param jobs - an array of jobs to be added to the queue.
* @returns
*/
static async createBulk(queue, jobs) {
const client = await queue.client;
const jobInstances = jobs.map(job => { var _a; return new this(queue, job.name, job.data, job.opts, (_a = job.opts) === null || _a === void 0 ? void 0 : _a.jobId); });
const multi = client.multi();
for (const job of jobInstances) {
job.addJob(multi, {
parentKey: job.parentKey,
parentDependenciesKey: job.parentKey
? `${job.parentKey}:dependencies`
: '',
});
}
const results = (await multi.exec());
for (let index = 0; index < results.length; ++index) {
const [err, id] = results[index];
if (err) {
throw err;
}
jobInstances[index].id = id;
}
return jobInstances;
}
/**
* Instantiates a Job from a JobJsonRaw object (coming from a deserialized JSON object)
*
* @param queue - the queue where the job belongs to.
* @param json - the plain object containing the job.
* @param jobId - an optional job id (overrides the id coming from the JSON object)
* @returns
*/
static fromJSON(queue, json, jobId) {
const data = JSON.parse(json.data || '{}');
const opts = Job.optsFromJSON(json.opts);
const job = new this(queue, json.name, data, opts, json.id || jobId);
job.progress = JSON.parse(json.progress || '0');
job.delay = parseInt(json.delay);
job.timestamp = parseInt(json.timestamp);
if (json.finishedOn) {
job.finishedOn = parseInt(json.finishedOn);
}
if (json.processedOn) {
job.processedOn = parseInt(json.processedOn);
}
if (json.rjk) {
job.repeatJobKey = json.rjk;
}
job.failedReason = json.failedReason;
job.attemptsMade = parseInt(json.attemptsMade || '0');
job.stacktrace = getTraces(json.stacktrace);
if (typeof json.returnvalue === 'string') {
job.returnvalue = getReturnValue(json.returnvalue);
}
if (json.parentKey) {
job.parentKey = json.parentKey;
}
if (json.parent) {
job.parent = JSON.parse(json.parent);
}
return job;
}
static optsFromJSON(rawOpts) {
const opts = JSON.parse(rawOpts || '{}');
const optionEntries = Object.entries(opts);
const options = {};
for (const item of optionEntries) {
const [attributeName, value] = item;
if (optsDecodeMap[attributeName]) {
options[optsDecodeMap[attributeName]] =
value;
}
else {
options[attributeName] = value;
}
}
return options;
}
/**
* Fetches a Job from the queue given the passed job id.
*
* @param queue - the queue where the job belongs to.
* @param jobId - the job id.
* @returns
*/
static async fromId(queue, jobId) {
// jobId can be undefined if moveJob returns undefined
if (jobId) {
const client = await queue.client;
const jobData = await client.hgetall(queue.toKey(jobId));
return isEmpty(jobData)
? undefined
: this.fromJSON(queue, jobData, jobId);
}
}
/**
* addJobLog
*
* @param queue Queue instance
* @param jobId Job id
* @param logRow Log row
* @param keepLogs optional maximum number of logs to keep
*
* @returns The total number of log entries for this job so far.
*/
static async addJobLog(queue, jobId, logRow, keepLogs) {
const client = await queue.client;
const logsKey = queue.toKey(jobId) + ':logs';
const multi = client.multi();
multi.rpush(logsKey, logRow);
if (keepLogs) {
multi.ltrim(logsKey, -keepLogs, -1);
}
const result = (await multi.exec());
return keepLogs ? Math.min(keepLogs, result[0][1]) : result[0][1];
}
toJSON() {
const _a = this, { queue, scripts } = _a, withoutQueueAndScripts = __rest(_a, ["queue", "scripts"]);
return withoutQueueAndScripts;
}
/**
* Prepares a job to be serialized for storage in Redis.
* @returns
*/
asJSON() {
return {
id: this.id,
name: this.name,
data: JSON.stringify(typeof this.data === 'undefined' ? {} : this.data),
opts: this.optsAsJSON(this.opts),
parent: this.parent ? Object.assign({}, this.parent) : undefined,
parentKey: this.parentKey,
progress: this.progress,
attemptsMade: this.attemptsMade,
finishedOn: this.finishedOn,
processedOn: this.processedOn,
timestamp: this.timestamp,
failedReason: JSON.stringify(this.failedReason),
stacktrace: JSON.stringify(this.stacktrace),
repeatJobKey: this.repeatJobKey,
returnvalue: JSON.stringify(this.returnvalue),
};
}
optsAsJSON(opts = {}) {
const optionEntries = Object.entries(opts);
const options = {};
for (const item of optionEntries) {
const [attributeName, value] = item;
if (optsEncodeMap[attributeName]) {
options[optsEncodeMap[attributeName]] =
value;
}
else {
options[attributeName] = value;
}
}
return options;
}
/**
* Prepares a job to be passed to Sandbox.
* @returns
*/
asJSONSandbox() {
return Object.assign(Object.assign({}, this.asJSON()), { queueName: this.queueName, prefix: this.prefix });
}
/**
* Updates a job's data
*
* @param data - the data that will replace the current jobs data.
*/
updateData(data) {
this.data = data;
return this.scripts.updateData(this, data);
}
/**
* Updates a job's progress
*
* @param progress - number or object to be saved as progress.
*/
async updateProgress(progress) {
this.progress = progress;
await this.scripts.updateProgress(this.id, progress);
this.queue.emit('progress', this, progress);
}
/**
* Logs one row of log data.
*
* @param logRow - string with log data to be logged.
* @returns The total number of log entries for this job so far.
*/
async log(logRow) {
return Job.addJobLog(this.queue, this.id, logRow, this.opts.keepLogs);
}
/**
* Clears job's logs
*
* @param keepLogs - the amount of log entries to preserve
*/
async clearLogs(keepLogs) {
const client = await this.queue.client;
const logsKey = this.toKey(this.id) + ':logs';
if (keepLogs) {
await client.ltrim(logsKey, -keepLogs, -1);
}
else {
await client.del(logsKey);
}
}
/**
* Completely remove the job from the queue.
* Note, this call will throw an exception if the job
* is being processed when the call is performed.
*
* @param opts - Options to remove a job
*/
async remove({ removeChildren = true } = {}) {
await this.queue.waitUntilReady();
const queue = this.queue;
const job = this;
const removed = await this.scripts.remove(job.id, removeChildren);
if (removed) {
queue.emit('removed', job);
}
else {
throw new Error(`Job ${this.id} could not be removed because it is locked by another worker`);
}
}
/**
* Extend the lock for this job.
*
* @param token - unique token for the lock
* @param duration - lock duration in milliseconds
*/
extendLock(token, duration) {
return this.scripts.extendLock(this.id, token, duration);
}
/**
* Moves a job to the completed queue.
* Returned job to be used with Queue.prototype.nextJobFromJobData.
*
* @param returnValue - The jobs success message.
* @param token - Worker token used to acquire completed job.
* @param fetchNext - True when wanting to fetch the next job.
* @returns Returns the jobData of the next job in the waiting queue.
*/
async moveToCompleted(returnValue, token, fetchNext = true) {
await this.queue.waitUntilReady();
this.returnvalue = returnValue || void 0;
const stringifiedReturnValue = tryCatch(JSON.stringify, JSON, [
returnValue,
]);
if (stringifiedReturnValue === errorObject) {
throw errorObject.value;
}
const args = this.scripts.moveToCompletedArgs(this, stringifiedReturnValue, this.opts.removeOnComplete, token, fetchNext);
const result = await this.scripts.moveToFinished(this.id, args);
this.finishedOn = args[14];
return result;
}
/**
* Moves a job to the failed queue.
*
* @param err - the jobs error message.
* @param token - token to check job is locked by current worker
* @param fetchNext - true when wanting to fetch the next job
* @returns void
*/
async moveToFailed(err, token, fetchNext = false) {
const client = await this.queue.client;
const message = err === null || err === void 0 ? void 0 : err.message;
const queue = this.queue;
this.failedReason = message;
let command;
const multi = client.multi();
this.saveStacktrace(multi, err);
//
// Check if an automatic retry should be performed
//
let moveToFailed = false;
let finishedOn, delay;
if (this.attemptsMade < this.opts.attempts &&
!this.discarded &&
!(err instanceof UnrecoverableError || err.name == 'UnrecoverableError')) {
const opts = queue.opts;
// Check if backoff is needed
delay = await Backoffs.calculate(this.opts.backoff, this.attemptsMade, err, this, opts.settings && opts.settings.backoffStrategy);
if (delay === -1) {
moveToFailed = true;
}
else if (delay) {
const args = this.scripts.moveToDelayedArgs(this.id, Date.now() + delay, token, delay);
this.scripts.execCommand(multi, 'moveToDelayed', args);
command = 'delayed';
}
else {
// Retry immediately
this.scripts.execCommand(multi, 'retryJob', this.scripts.retryJobArgs(this.id, this.opts.lifo, token));
command = 'retryJob';
}
}
else {
// If not, move to failed
moveToFailed = true;
}
if (moveToFailed) {
const args = this.scripts.moveToFailedArgs(this, message, this.opts.removeOnFail, token, fetchNext);
this.scripts.execCommand(multi, 'moveToFinished', args);
finishedOn = args[14];
command = 'failed';
}
const results = await multi.exec();
const anyError = results.find(result => result[0]);
if (anyError) {
throw new Error(`Error "moveToFailed" with command ${command}: ${anyError}`);
}
const code = results[results.length - 1][1];
if (code < 0) {
throw this.scripts.finishedErrors(code, this.id, command, 'active');
}
if (finishedOn && typeof finishedOn === 'number') {
this.finishedOn = finishedOn;
}
if (delay && typeof delay === 'number') {
this.delay = delay;
}
}
/**
* @returns true if the job has completed.
*/
isCompleted() {
return this.isInZSet('completed');
}
/**
* @returns true if the job has failed.
*/
isFailed() {
return this.isInZSet('failed');
}
/**
* @returns true if the job is delayed.
*/
isDelayed() {
return this.isInZSet('delayed');
}
/**
* @returns true if the job is waiting for children.
*/
isWaitingChildren() {
return this.isInZSet('waiting-children');
}
/**
* @returns true of the job is active.
*/
isActive() {
return this.isInList('active');
}
/**
* @returns true if the job is waiting.
*/
async isWaiting() {
return (await this.isInList('wait')) || (await this.isInList('paused'));
}
/**
* @returns the queue name this job belongs to.
*/
get queueName() {
return this.queue.name;
}
/**
* @returns the prefix that is used.
*/
get prefix() {
return this.queue.opts.prefix;
}
/**
* Get current state.
*
* @returns Returns one of these values:
* 'completed', 'failed', 'delayed', 'active', 'waiting', 'waiting-children', 'unknown'.
*/
getState() {
return this.scripts.getState(this.id);
}
/**
* Change delay of a delayed job.
*
* @param delay - milliseconds to be added to current time.
* @returns void
*/
async changeDelay(delay) {
await this.scripts.changeDelay(this.id, delay);
this.delay = delay;
}
/**
* Change job priority.
*
* @returns void
*/
async changePriority(opts) {
await this.scripts.changePriority(this.id, opts.priority, opts.lifo);
}
/**
* Get this jobs children result values if any.
*
* @returns Object mapping children job keys with their values.
*/
async getChildrenValues() {
const client = await this.queue.client;
const result = (await client.hgetall(this.toKey(`${this.id}:processed`)));
if (result) {
return parseObjectValues(result);
}
}
/**
* Get children job keys if this job is a parent and has children.
* @remarks
* Count options before Redis v7.2 works as expected with any quantity of entries
* on processed/unprocessed dependencies, since v7.2 you must consider that count
* won't have any effect until processed/unprocessed dependencies have a length
* greater than 127
* @see https://redis.io/docs/management/optimization/memory-optimization/#redis--72
* @returns dependencies separated by processed and unprocessed.
*/
async getDependencies(opts = {}) {
const client = await this.queue.client;
const multi = client.multi();
if (!opts.processed && !opts.unprocessed) {
multi.hgetall(this.toKey(`${this.id}:processed`));
multi.smembers(this.toKey(`${this.id}:dependencies`));
const [[err1, processed], [err2, unprocessed]] = (await multi.exec());
const transformedProcessed = parseObjectValues(processed);
return { processed: transformedProcessed, unprocessed };
}
else {
const defaultOpts = {
cursor: 0,
count: 20,
};
if (opts.processed) {
const processedOpts = Object.assign(Object.assign({}, defaultOpts), opts.processed);
multi.hscan(this.toKey(`${this.id}:processed`), processedOpts.cursor, 'COUNT', processedOpts.count);
}
if (opts.unprocessed) {
const unprocessedOpts = Object.assign(Object.assign({}, defaultOpts), opts.unprocessed);
multi.sscan(this.toKey(`${this.id}:dependencies`), unprocessedOpts.cursor, 'COUNT', unprocessedOpts.count);
}
const [result1, result2] = (await multi.exec());
const [processedCursor, processed = []] = opts.processed
? result1[1]
: [];
const [unprocessedCursor, unprocessed = []] = opts.unprocessed
? opts.processed
? result2[1]
: result1[1]
: [];
const transformedProcessed = {};
for (let index = 0; index < processed.length; ++index) {
if (index % 2) {
transformedProcessed[processed[index - 1]] = JSON.parse(processed[index]);
}
}
return Object.assign(Object.assign({}, (processedCursor
? {
processed: transformedProcessed,
nextProcessedCursor: Number(processedCursor),
}
: {})), (unprocessedCursor
? { unprocessed, nextUnprocessedCursor: Number(unprocessedCursor) }
: {}));
}
}
/**
* Get children job counts if this job is a parent and has children.
*
* @returns dependencies count separated by processed and unprocessed.
*/
async getDependenciesCount(opts = {}) {
const client = await this.queue.client;
const multi = client.multi();
const updatedOpts = !opts.processed && !opts.unprocessed
? { processed: true, unprocessed: true }
: opts;
if (updatedOpts.processed) {
multi.hlen(this.toKey(`${this.id}:processed`));
}
if (updatedOpts.unprocessed) {
multi.scard(this.toKey(`${this.id}:dependencies`));
}
const [[err1, result1] = [], [err2, result2] = []] = (await multi.exec());
const processed = updatedOpts.processed ? result1 : undefined;
const unprocessed = updatedOpts.unprocessed
? updatedOpts.processed
? result2
: result1
: undefined;
return Object.assign(Object.assign({}, (updatedOpts.processed
? {
processed,
}
: {})), (updatedOpts.unprocessed ? { unprocessed } : {}));
}
/**
* Returns a promise the resolves when the job has completed (containing the return value of the job),
* or rejects when the job has failed (containing the failedReason).
*
* @param queueEvents - Instance of QueueEvents.
* @param ttl - Time in milliseconds to wait for job to finish before timing out.
*/
async waitUntilFinished(queueEvents, ttl) {
await this.queue.waitUntilReady();
const jobId = this.id;
return new Promise(async (resolve, reject) => {
let timeout;
if (ttl) {
timeout = setTimeout(() => onFailed(
/* eslint-disable max-len */
`Job wait ${this.name} timed out before finishing, no finish notification arrived after ${ttl}ms (id=${jobId})`), ttl);
}
function onCompleted(args) {
removeListeners();
resolve(args.returnvalue);
}
function onFailed(args) {
removeListeners();
reject(new Error(args.failedReason || args));
}
const completedEvent = `completed:${jobId}`;
const failedEvent = `failed:${jobId}`;
queueEvents.on(completedEvent, onCompleted);
queueEvents.on(failedEvent, onFailed);
this.queue.on('closing', onFailed);
const removeListeners = () => {
clearInterval(timeout);
queueEvents.removeListener(completedEvent, onCompleted);
queueEvents.removeListener(failedEvent, onFailed);
this.queue.removeListener('closing', onFailed);
};
// Poll once right now to see if the job has already finished. The job may have been completed before we were able
// to register the event handlers on the QueueEvents, so we check here to make sure we're not waiting for an event
// that has already happened. We block checking the job until the queue events object is actually listening to
// Redis so there's no chance that it will miss events.
await queueEvents.waitUntilReady();
const [status, result] = (await this.scripts.isFinished(jobId, true));
const finished = status != 0;
if (finished) {
if (status == -1 || status == 2) {
onFailed({ failedReason: result });
}
else {
onCompleted({ returnvalue: getReturnValue(result) });
}
}
});
}
/**
* Moves the job to the delay set.
*
* @param timestamp - timestamp where the job should be moved back to "wait"
* @param token - token to check job is locked by current worker
* @returns
*/
moveToDelayed(timestamp, token) {
const delay = timestamp - Date.now();
return this.scripts.moveToDelayed(this.id, timestamp, delay > 0 ? delay : 0, token);
}
/**
* Moves the job to the waiting-children set.
*
* @param token - Token to check job is locked by current worker
* @param opts - The options bag for moving a job to waiting-children.
* @returns true if the job was moved
*/
moveToWaitingChildren(token, opts = {}) {
return this.scripts.moveToWaitingChildren(this.id, token, opts);
}
/**
* Promotes a delayed job so that it starts to be processed as soon as possible.
*/
async promote() {
const jobId = this.id;
await this.scripts.promote(jobId);
this.delay = 0;
}
/**
* Attempts to retry the job. Only a job that has failed or completed can be retried.
*
* @param state - completed / failed
* @returns If resolved and return code is 1, then the queue emits a waiting event
* otherwise the operation was not a success and throw the corresponding error. If the promise
* rejects, it indicates that the script failed to execute
*/
retry(state = 'failed') {
this.failedReason = null;
this.finishedOn = null;
this.processedOn = null;
this.returnvalue = null;
return this.scripts.reprocessJob(this, state);
}
/**
* Marks a job to not be retried if it fails (even if attempts has been configured)
*/
discard() {
this.discarded = true;
}
async isInZSet(set) {
const client = await this.queue.client;
const score = await client.zscore(this.queue.toKey(set), this.id);
return score !== null;
}
async isInList(list) {
return this.scripts.isJobInList(this.queue.toKey(list), this.id);
}
/**
* Adds the job to Redis.
*
* @param client -
* @param parentOpts -
* @returns
*/
addJob(client, parentOpts) {
const jobData = this.asJSON();
this.validateOptions(jobData);
return this.scripts.addJob(client, jobData, jobData.opts, this.id, parentOpts);
}
validateOptions(jobData) {
var _a;
const exceedLimit = this.opts.sizeLimit &&
lengthInUtf8Bytes(jobData.data) > this.opts.sizeLimit;
if (exceedLimit) {
throw new Error(`The size of job ${this.name} exceeds the limit ${this.opts.sizeLimit} bytes`);
}
if (this.opts.delay && this.opts.repeat && !((_a = this.opts.repeat) === null || _a === void 0 ? void 0 : _a.count)) {
throw new Error(`Delay and repeat options could not be used together`);
}
if (this.opts.removeDependencyOnFailure && this.opts.failParentOnFailure) {
throw new Error(`RemoveDependencyOnFailure and failParentOnFailure options can not be used together`);
}
if (`${parseInt(this.id, 10)}` === this.id) {
//TODO: throw an error in next breaking change
console.warn('Custom Ids should not be integers: https://github.com/taskforcesh/bullmq/pull/1569');
}
if (this.opts.priority) {
if (Math.trunc(this.opts.priority) !== this.opts.priority) {
throw new Error(`Priority should not be float`);
}
if (this.opts.priority > PRIORITY_LIMIT) {
throw new Error(`Priority should be between 0 and ${PRIORITY_LIMIT}`);
}
}
}
saveStacktrace(multi, err) {
this.stacktrace = this.stacktrace || [];
if (err === null || err === void 0 ? void 0 : err.stack) {
this.stacktrace.push(err.stack);
if (this.opts.stackTraceLimit) {
this.stacktrace = this.stacktrace.slice(0, this.opts.stackTraceLimit);
}
}
const args = this.scripts.saveStacktraceArgs(this.id, JSON.stringify(this.stacktrace), err === null || err === void 0 ? void 0 : err.message);
this.scripts.execCommand(multi, 'saveStacktrace', args);
}
}
function getTraces(stacktrace) {
const traces = tryCatch(JSON.parse, JSON, [stacktrace]);
if (traces === errorObject || !(traces instanceof Array)) {
return [];
}
else {
return traces;
}
}
function getReturnValue(_value) {
const value = tryCatch(JSON.parse, JSON, [_value]);
if (value !== errorObject) {
return value;
}
else {
logger('corrupted returnvalue: ' + _value, value);
}
}
//# sourceMappingURL=job.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"job.js","sourceRoot":"","sources":["../../../src/classes/job.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAsBhC,OAAO,EACL,WAAW,EACX,OAAO,EACP,YAAY,EACZ,iBAAiB,EACjB,iBAAiB,EACjB,QAAQ,GACT,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAGlE,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;AAEhC,MAAM,aAAa,GAAG;IACpB,IAAI,EAAE,qBAAqB;IAC3B,EAAE,EAAE,UAAU;IACd,IAAI,EAAE,2BAA2B;CAClC,CAAC;AAEF,MAAM,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC,CAAC;AAE5C,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,IAAI,EAAE,CAAC;AAEtC;;;;;;;;;GASG;AACH,MAAM,OAAO,GAAG;IAwFd,YACY,KAAmB;IAC7B;;OAEG;IACI,IAAc;IAErB;;OAEG;IACI,IAAc;IAErB;;OAEG;IACI,OAAoB,EAAE,EACtB,EAAW;QAfR,UAAK,GAAL,KAAK,CAAc;QAItB,SAAI,GAAJ,IAAI,CAAU;QAKd,SAAI,GAAJ,IAAI,CAAU;QAKd,SAAI,GAAJ,IAAI,CAAkB;QACtB,OAAE,GAAF,EAAE,CAAS;QA5FpB;;;WAGG;QACH,aAAQ,GAAoB,CAAC,CAAC;QAE9B;;;WAGG;QACH,gBAAW,GAAe,IAAI,CAAC;QAE/B;;;WAGG;QACH,eAAU,GAAa,IAAI,CAAC;QAa5B;;;WAGG;QACH,iBAAY,GAAG,CAAC,CAAC;QA6Df,MAAM,KAAgC,IAAI,CAAC,IAAI,EAAzC,EAAE,YAAY,OAA2B,EAAtB,QAAQ,cAA3B,gBAA6B,CAAY,CAAC;QAEhD,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,MAAM,CACvB;YACE,QAAQ,EAAE,CAAC;YACX,KAAK,EAAE,CAAC;SACT,EACD,QAAQ,CACT,CAAC;QAEF,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;QAE7B,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QAEjC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;QAE9D,IAAI,CAAC,IAAI,CAAC,OAAO,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAErD,IAAI,CAAC,SAAS,GAAG,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAE3C,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM;YACvB,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE;YACrD,CAAC,CAAC,SAAS,CAAC;QAEd,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrC,IAAI,CAAC,OAAO,GAAG,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC;QAElC,IAAI,CAAC,kBAAkB,GAAG,KAAK,CAAC,aAAa,CAAC;IAChD,CAAC;IAED;;;;;;;;OAQG;IACH,MAAM,CAAC,KAAK,CAAC,MAAM,CACjB,KAAmB,EACnB,IAAO,EACP,IAAO,EACP,IAAkB;QAElB,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC;QAElC,MAAM,GAAG,GAAG,IAAI,IAAI,CAAU,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC;QAE3E,GAAG,CAAC,EAAE,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE;YAChC,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,qBAAqB,EAAE,GAAG,CAAC,SAAS;gBAClC,CAAC,CAAC,GAAG,GAAG,CAAC,SAAS,eAAe;gBACjC,CAAC,CAAC,EAAE;SACP,CAAC,CAAC;QAEH,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;;;;;OAMG;IACH,MAAM,CAAC,KAAK,CAAC,UAAU,CACrB,KAAmB,EACnB,IAIG;QAEH,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC;QAElC,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAC3B,GAAG,CAAC,EAAE,WACJ,OAAA,IAAI,IAAI,CAAU,KAAK,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,MAAA,GAAG,CAAC,IAAI,0CAAE,KAAK,CAAC,CAAA,EAAA,CAC1E,CAAC;QAEF,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC;QAE7B,KAAK,MAAM,GAAG,IAAI,YAAY,EAAE;YAC9B,GAAG,CAAC,MAAM,CAAe,KAAiB,EAAE;gBAC1C,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,qBAAqB,EAAE,GAAG,CAAC,SAAS;oBAClC,CAAC,CAAC,GAAG,GAAG,CAAC,SAAS,eAAe;oBACjC,CAAC,CAAC,EAAE;aACP,CAAC,CAAC;SACJ;QAED,MAAM,OAAO,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,EAAE,CAA6B,CAAC;QACjE,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE;YACnD,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;YACjC,IAAI,GAAG,EAAE;gBACP,MAAM,GAAG,CAAC;aACX;YAED,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC;SAC7B;QAED,OAAO,YAAY,CAAC;IACtB,CAAC;IAED;;;;;;;OAOG;IACH,MAAM,CAAC,QAAQ,CACb,KAAmB,EACnB,IAAgB,EAChB,KAAc;QAEd,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC;QAC3C,MAAM,IAAI,GAAG,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEzC,MAAM,GAAG,GAAG,IAAI,IAAI,CAClB,KAAK,EACL,IAAI,CAAC,IAAS,EACd,IAAI,EACJ,IAAI,EACJ,IAAI,CAAC,EAAE,IAAI,KAAK,CACjB,CAAC;QAEF,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,IAAI,GAAG,CAAC,CAAC;QAEhD,GAAG,CAAC,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAEjC,GAAG,CAAC,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAEzC,IAAI,IAAI,CAAC,UAAU,EAAE;YACnB,GAAG,CAAC,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;SAC5C;QAED,IAAI,IAAI,CAAC,WAAW,EAAE;YACpB,GAAG,CAAC,WAAW,GAAG,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;SAC9C;QAED,IAAI,IAAI,CAAC,GAAG,EAAE;YACZ,GAAG,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC;SAC7B;QAED,GAAG,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QACrC,GAAG,CAAC,YAAY,GAAG,QAAQ,CAAC,IAAI,CAAC,YAAY,IAAI,GAAG,CAAC,CAAC;QAEtD,GAAG,CAAC,UAAU,GAAG,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAE5C,IAAI,OAAO,IAAI,CAAC,WAAW,KAAK,QAAQ,EAAE;YACxC,GAAG,CAAC,WAAW,GAAG,cAAc,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;SACpD;QAED,IAAI,IAAI,CAAC,SAAS,EAAE;YAClB,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;SAChC;QAED,IAAI,IAAI,CAAC,MAAM,EAAE;YACf,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;SACtC;QAED,OAAO,GAAG,CAAC;IACb,CAAC;IAEO,MAAM,CAAC,YAAY,CAAC,OAAgB;QAC1C,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC;QAEzC,MAAM,aAAa,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAExC,CAAC;QAEF,MAAM,OAAO,GAAiC,EAAE,CAAC;QACjD,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE;YAChC,MAAM,CAAC,aAAa,EAAE,KAAK,CAAC,GAAG,IAAI,CAAC;YACpC,IAAK,aAAqCLine truncated
+4
View File
@@ -0,0 +1,4 @@
declare const _default: (send: (msg: any) => Promise<void>, receiver: {
on: (evt: 'message', cb: (msg: any) => void) => void;
}) => void;
export default _default;
+43
View File
@@ -0,0 +1,43 @@
/**
* Wrapper for sandboxing.
*
*/
import { toString } from 'lodash';
import { ChildProcessor } from './child-processor';
import { ParentCommand, ChildCommand } from '../enums';
import { errorToJSON } from '../utils';
export default (send, receiver) => {
const childProcessor = new ChildProcessor(send);
receiver === null || receiver === void 0 ? void 0 : receiver.on('message', async (msg) => {
try {
switch (msg.cmd) {
case ChildCommand.Init:
await childProcessor.init(msg.value);
break;
case ChildCommand.Start:
await childProcessor.start(msg.job, msg === null || msg === void 0 ? void 0 : msg.token);
break;
case ChildCommand.Stop:
break;
}
}
catch (err) {
console.error('Error handling child message');
}
});
process.on('SIGTERM', () => childProcessor.waitForCurrentJobAndExit());
process.on('SIGINT', () => childProcessor.waitForCurrentJobAndExit());
process.on('uncaughtException', async (err) => {
if (!err.message) {
err = new Error(toString(err));
}
await send({
cmd: ParentCommand.Failed,
value: errorToJSON(err),
});
// An uncaughException leaves this process in a potentially undetermined state so
// we must exit
process.exit();
});
};
//# sourceMappingURL=main-base.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"main-base.js","sourceRoot":"","sources":["../../../src/classes/main-base.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AACvD,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAEvC,eAAe,CACb,IAAiC,EACjC,QAAkE,EAClE,EAAE;IACF,MAAM,cAAc,GAAG,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC;IAEhD,QAAQ,aAAR,QAAQ,uBAAR,QAAQ,CAAE,EAAE,CAAC,SAAS,EAAE,KAAK,EAAC,GAAG,EAAC,EAAE;QAClC,IAAI;YACF,QAAQ,GAAG,CAAC,GAAmB,EAAE;gBAC/B,KAAK,YAAY,CAAC,IAAI;oBACpB,MAAM,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;oBACrC,MAAM;gBACR,KAAK,YAAY,CAAC,KAAK;oBACrB,MAAM,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,KAAK,CAAC,CAAC;oBAChD,MAAM;gBACR,KAAK,YAAY,CAAC,IAAI;oBACpB,MAAM;aACT;SACF;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;SAC/C;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,cAAc,CAAC,wBAAwB,EAAE,CAAC,CAAC;IACvE,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,cAAc,CAAC,wBAAwB,EAAE,CAAC,CAAC;IAEtE,OAAO,CAAC,EAAE,CAAC,mBAAmB,EAAE,KAAK,EAAE,GAAU,EAAE,EAAE;QACnD,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE;YAChB,GAAG,GAAG,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;SAChC;QACD,MAAM,IAAI,CAAC;YACT,GAAG,EAAE,aAAa,CAAC,MAAM;YACzB,KAAK,EAAE,WAAW,CAAC,GAAG,CAAC;SACxB,CAAC,CAAC;QAEH,iFAAiF;QACjF,eAAe;QACf,OAAO,CAAC,IAAI,EAAE,CAAC;IACjB,CAAC,CAAC,CAAC;AACL,CAAC,CAAC"}
+1
View File
@@ -0,0 +1 @@
export {};
+8
View File
@@ -0,0 +1,8 @@
/**
* Worker Thread wrapper for sandboxing
*
*/
import { parentPort } from 'worker_threads';
import mainBase from './main-base';
mainBase(async (msg) => parentPort.postMessage(msg), parentPort);
//# sourceMappingURL=main-worker.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"main-worker.js","sourceRoot":"","sources":["../../../src/classes/main-worker.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,QAAQ,MAAM,aAAa,CAAC;AAEnC,QAAQ,CAAC,KAAK,EAAE,GAAQ,EAAE,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,UAAU,CAAC,CAAC"}
+1
View File
@@ -0,0 +1 @@
export {};
+8
View File
@@ -0,0 +1,8 @@
/**
* Child process wrapper for sandboxing.
*
*/
import { childSend } from '../utils';
import mainBase from './main-base';
mainBase((msg) => childSend(process, msg), process);
//# sourceMappingURL=main.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"main.js","sourceRoot":"","sources":["../../../src/classes/main.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,QAAQ,MAAM,aAAa,CAAC;AAEnC,QAAQ,CAAC,CAAC,GAAQ,EAAE,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,CAAC,EAAE,OAAO,CAAC,CAAC"}
+69
View File
@@ -0,0 +1,69 @@
/// <reference types="node" />
import { EventEmitter } from 'events';
import { QueueBaseOptions, RedisClient } from '../interfaces';
import { MinimalQueue } from '../types';
import { RedisConnection } from './redis-connection';
import { Job } from './job';
import { KeysMap } from './queue-keys';
import { Scripts } from './scripts';
/**
* @class QueueBase
* @extends EventEmitter
*
* @description Base class for all classes that need to interact with queues.
* This class is normally not used directly, but extended by the other classes.
*
*/
export declare class QueueBase extends EventEmitter implements MinimalQueue {
readonly name: string;
opts: QueueBaseOptions;
toKey: (type: string) => string;
keys: KeysMap;
closing: Promise<void> | undefined;
protected closed: boolean;
protected scripts: Scripts;
protected connection: RedisConnection;
readonly qualifiedName: string;
/**
*
* @param name - The name of the queue.
* @param opts - Options for the queue.
* @param Connection - An optional "Connection" class used to instantiate a Connection. This is useful for
* testing with mockups and/or extending the Connection class and passing an alternate implementation.
*/
constructor(name: string, opts?: QueueBaseOptions, Connection?: typeof RedisConnection);
/**
* Returns a promise that resolves to a redis client. Normally used only by subclasses.
*/
get client(): Promise<RedisClient>;
/**
* Returns the version of the Redis instance the client is connected to,
*/
get redisVersion(): string;
/**
* Helper to easily extend Job class calls.
*/
protected get Job(): typeof Job;
/**
* Emits an event. Normally used by subclasses to emit events.
*
* @param event - The emitted event.
* @param args -
* @returns
*/
emit(event: string | symbol, ...args: any[]): boolean;
waitUntilReady(): Promise<RedisClient>;
protected base64Name(): string;
protected clientName(suffix?: string): string;
/**
*
* Closes the connection and returns a promise that resolves when the connection is closed.
*/
close(): Promise<void>;
/**
*
* Force disconnects a connection.
*/
disconnect(): Promise<void>;
protected checkConnectionError<T>(fn: () => Promise<T>, delayInMs?: number): Promise<T | undefined>;
}
+137
View File
@@ -0,0 +1,137 @@
import { EventEmitter } from 'events';
import { delay, DELAY_TIME_5, isNotConnectionError, isRedisInstance, } from '../utils';
import { RedisConnection } from './redis-connection';
import { Job } from './job';
import { QueueKeys } from './queue-keys';
import { Scripts } from './scripts';
/**
* @class QueueBase
* @extends EventEmitter
*
* @description Base class for all classes that need to interact with queues.
* This class is normally not used directly, but extended by the other classes.
*
*/
export class QueueBase extends EventEmitter {
/**
*
* @param name - The name of the queue.
* @param opts - Options for the queue.
* @param Connection - An optional "Connection" class used to instantiate a Connection. This is useful for
* testing with mockups and/or extending the Connection class and passing an alternate implementation.
*/
constructor(name, opts = {}, Connection = RedisConnection) {
super();
this.name = name;
this.opts = opts;
this.closed = false;
this.opts = Object.assign({ prefix: 'bull' }, opts);
if (!name) {
throw new Error('Queue name must be provided');
}
if (!opts.connection) {
console.warn([
'BullMQ: DEPRECATION WARNING! Optional instantiation of Queue, Worker, QueueEvents and FlowProducer',
'without providing explicitly a connection or connection options is deprecated. This behaviour will',
'be removed in the next major release',
].join(' '));
}
this.connection = new Connection(opts.connection, isRedisInstance(opts === null || opts === void 0 ? void 0 : opts.connection), opts.blockingConnection, opts.skipVersionCheck);
this.connection.on('error', (error) => this.emit('error', error));
this.connection.on('close', () => {
if (!this.closing) {
this.emit('ioredis:close');
}
});
const queueKeys = new QueueKeys(opts.prefix);
this.qualifiedName = queueKeys.getQueueQualifiedName(name);
this.keys = queueKeys.getKeys(name);
this.toKey = (type) => queueKeys.toKey(name, type);
this.scripts = new Scripts(this);
}
/**
* Returns a promise that resolves to a redis client. Normally used only by subclasses.
*/
get client() {
return this.connection.client;
}
/**
* Returns the version of the Redis instance the client is connected to,
*/
get redisVersion() {
return this.connection.redisVersion;
}
/**
* Helper to easily extend Job class calls.
*/
get Job() {
return Job;
}
/**
* Emits an event. Normally used by subclasses to emit events.
*
* @param event - The emitted event.
* @param args -
* @returns
*/
emit(event, ...args) {
try {
return super.emit(event, ...args);
}
catch (err) {
try {
return super.emit('error', err);
}
catch (err) {
// We give up if the error event also throws an exception.
console.error(err);
return false;
}
}
}
waitUntilReady() {
return this.client;
}
base64Name() {
return Buffer.from(this.name).toString('base64');
}
clientName(suffix = '') {
const queueNameBase64 = this.base64Name();
return `${this.opts.prefix}:${queueNameBase64}${suffix}`;
}
/**
*
* Closes the connection and returns a promise that resolves when the connection is closed.
*/
async close() {
if (!this.closing) {
this.closing = this.connection.close();
}
await this.closing;
this.closed = true;
}
/**
*
* Force disconnects a connection.
*/
disconnect() {
return this.connection.disconnect();
}
async checkConnectionError(fn, delayInMs = DELAY_TIME_5) {
try {
return await fn();
}
catch (error) {
if (isNotConnectionError(error)) {
this.emit('error', error);
}
if (!this.closing && delayInMs) {
await delay(delayInMs);
}
else {
return;
}
}
}
}
//# sourceMappingURL=queue-base.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"queue-base.js","sourceRoot":"","sources":["../../../src/classes/queue-base.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAGtC,OAAO,EACL,KAAK,EACL,YAAY,EACZ,oBAAoB,EACpB,eAAe,GAChB,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,GAAG,EAAE,MAAM,OAAO,CAAC;AAC5B,OAAO,EAAW,SAAS,EAAE,MAAM,cAAc,CAAC;AAClD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC;;;;;;;GAOG;AACH,MAAM,OAAO,SAAU,SAAQ,YAAY;IAUzC;;;;;;OAMG;IACH,YACkB,IAAY,EACrB,OAAyB,EAAE,EAClC,aAAqC,eAAe;QAEpD,KAAK,EAAE,CAAC;QAJQ,SAAI,GAAJ,IAAI,CAAQ;QACrB,SAAI,GAAJ,IAAI,CAAuB;QAd1B,WAAM,GAAY,KAAK,CAAC;QAmBhC,IAAI,CAAC,IAAI,mBACP,MAAM,EAAE,MAAM,IACX,IAAI,CACR,CAAC;QAEF,IAAI,CAAC,IAAI,EAAE;YACT,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;SAChD;QAED,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE;YACpB,OAAO,CAAC,IAAI,CACV;gBACE,oGAAoG;gBACpG,oGAAoG;gBACpG,sCAAsC;aACvC,CAAC,IAAI,CAAC,GAAG,CAAC,CACZ,CAAC;SACH;QAED,IAAI,CAAC,UAAU,GAAG,IAAI,UAAU,CAC9B,IAAI,CAAC,UAAU,EACf,eAAe,CAAC,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,UAAU,CAAC,EACjC,IAAI,CAAC,kBAAkB,EACvB,IAAI,CAAC,gBAAgB,CACtB,CAAC;QAEF,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAY,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;QACzE,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YAC/B,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;gBACjB,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;aAC5B;QACH,CAAC,CAAC,CAAC;QAEH,MAAM,SAAS,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC7C,IAAI,CAAC,aAAa,GAAG,SAAS,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAC3D,IAAI,CAAC,IAAI,GAAG,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,CAAC,KAAK,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC3D,IAAI,CAAC,OAAO,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC;IAED;;OAEG;IACH,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,IAAc,GAAG;QACf,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;;;;;OAMG;IACH,IAAI,CAAC,KAAsB,EAAE,GAAG,IAAW;QACzC,IAAI;YACF,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,CAAC;SACnC;QAAC,OAAO,GAAG,EAAE;YACZ,IAAI;gBACF,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;aACjC;YAAC,OAAO,GAAG,EAAE;gBACZ,0DAA0D;gBAC1D,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACnB,OAAO,KAAK,CAAC;aACd;SACF;IACH,CAAC;IAED,cAAc;QACZ,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAES,UAAU;QAClB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACnD,CAAC;IAES,UAAU,CAAC,MAAM,GAAG,EAAE;QAC9B,MAAM,eAAe,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;QAC1C,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,eAAe,GAAG,MAAM,EAAE,CAAC;IAC3D,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YACjB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;SACxC;QACD,MAAM,IAAI,CAAC,OAAO,CAAC;QACnB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;IACrB,CAAC;IAED;;;OAGG;IACH,UAAU;QACR,OAAO,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,CAAC;IACtC,CAAC;IAES,KAAK,CAAC,oBAAoB,CAClC,EAAoB,EACpB,SAAS,GAAG,YAAY;QAExB,IAAI;YACF,OAAO,MAAM,EAAE,EAAE,CAAC;SACnB;QAAC,OAAO,KAAK,EAAE;YACd,IAAI,oBAAoB,CAAC,KAAc,CAAC,EAAE;gBACxC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAS,KAAK,CAAC,CAAC;aAClC;YAED,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,SAAS,EAAE;gBAC9B,MAAM,KAAK,CAAC,SAAS,CAAC,CAAC;aACxB;iBAAM;gBACL,OAAO;aACR;SACF;IACH,CAAC;CACF"}
+179
View File
@@ -0,0 +1,179 @@
import { IoredisListener, QueueEventsOptions } from '../interfaces';
import { QueueBase } from './queue-base';
import { RedisConnection } from './redis-connection';
export interface QueueEventsListener extends IoredisListener {
/**
* Listen to 'active' event.
*
* This event is triggered when a job enters the 'active' state.
*/
active: (args: {
jobId: string;
prev?: string;
}, id: string) => void;
/**
* Listen to 'added' event.
*
* This event is triggered when a job is created.
*/
added: (args: {
jobId: string;
name: string;
}, id: string) => void;
/**
* Listen to 'cleaned' event.
*
* This event is triggered when a cleaned method is triggered.
*/
cleaned: (args: {
count: string;
}, id: string) => void;
/**
* Listen to 'completed' event.
*
* This event is triggered when a job has successfully completed.
*/
completed: (args: {
jobId: string;
returnvalue: string;
prev?: string;
}, id: string) => void;
/**
* Listen to 'delayed' event.
*
* This event is triggered when a job is delayed.
*/
delayed: (args: {
jobId: string;
delay: number;
}, id: string) => void;
/**
* Listen to 'drained' event.
*
* This event is triggered when the queue has drained the waiting list.
* Note that there could still be delayed jobs waiting their timers to expire
* and this event will still be triggered as long as the waiting list has emptied.
*/
drained: (id: string) => void;
/**
* Listen to 'duplicated' event.
*
* This event is triggered when a job is not created because it already exist.
*/
duplicated: (args: {
jobId: string;
}, id: string) => void;
/**
* Listen to 'error' event.
*
* This event is triggered when an exception is thrown.
*/
error: (args: Error) => void;
/**
* Listen to 'failed' event.
*
* This event is triggered when a job has thrown an exception.
*/
failed: (args: {
jobId: string;
failedReason: string;
prev?: string;
}, id: string) => void;
/**
* Listen to 'paused' event.
*
* This event is triggered when a queue is paused.
*/
paused: (args: {}, id: string) => void;
/**
* Listen to 'progress' event.
*
* This event is triggered when a job updates it progress, i.e. the
* Job##updateProgress() method is called. This is useful to notify
* progress or any other data from within a processor to the rest of the
* world.
*/
progress: (args: {
jobId: string;
data: number | object;
}, id: string) => void;
/**
* Listen to 'removed' event.
*
* This event is triggered when a job has been manually
* removed from the queue.
*/
removed: (args: {
jobId: string;
prev: string;
}, id: string) => void;
/**
* Listen to 'resumed' event.
*
* This event is triggered when a queue is resumed.
*/
resumed: (args: {}, id: string) => void;
/**
* Listen to 'retries-exhausted' event.
*
* This event is triggered when a job has retried the maximum attempts.
*/
'retries-exhausted': (args: {
jobId: string;
attemptsMade: string;
}, id: string) => void;
/**
* Listen to 'stalled' event.
*
* This event is triggered when a job has been moved from 'active' back
* to 'waiting'/'failed' due to the processor not being able to renew
* the lock on the said job.
*/
stalled: (args: {
jobId: string;
}, id: string) => void;
/**
* Listen to 'waiting' event.
*
* This event is triggered when a job enters the 'waiting' state.
*/
waiting: (args: {
jobId: string;
prev?: string;
}, id: string) => void;
/**
* Listen to 'waiting-children' event.
*
* This event is triggered when a job enters the 'waiting-children' state.
*/
'waiting-children': (args: {
jobId: string;
}, id: string) => void;
}
/**
* The QueueEvents class is used for listening to the global events
* emitted by a given queue.
*
* This class requires a dedicated redis connection.
*
*/
export declare class QueueEvents extends QueueBase {
private running;
constructor(name: string, { connection, autorun, ...opts }?: QueueEventsOptions, Connection?: typeof RedisConnection);
emit<U extends keyof QueueEventsListener>(event: U, ...args: Parameters<QueueEventsListener[U]>): boolean;
off<U extends keyof QueueEventsListener>(eventName: U, listener: QueueEventsListener[U]): this;
on<U extends keyof QueueEventsListener>(event: U, listener: QueueEventsListener[U]): this;
once<U extends keyof QueueEventsListener>(event: U, listener: QueueEventsListener[U]): this;
/**
* Manually starts running the event consumming loop. This shall be used if you do not
* use the default "autorun" option on the constructor.
*/
run(): Promise<void>;
private consumeEvents;
/**
* Stops consuming events and close the underlying Redis connection if necessary.
*
* @returns
*/
close(): Promise<void>;
}
+116
View File
@@ -0,0 +1,116 @@
import { __rest } from "tslib";
import { array2obj, clientCommandMessageReg, isRedisInstance, QUEUE_EVENT_SUFFIX, } from '../utils';
import { QueueBase } from './queue-base';
/**
* The QueueEvents class is used for listening to the global events
* emitted by a given queue.
*
* This class requires a dedicated redis connection.
*
*/
export class QueueEvents extends QueueBase {
constructor(name, _a = {}, Connection) {
var { connection, autorun = true } = _a, opts = __rest(_a, ["connection", "autorun"]);
super(name, Object.assign(Object.assign({}, opts), { connection: isRedisInstance(connection)
? connection.duplicate()
: connection, blockingConnection: true }), Connection);
this.running = false;
this.opts = Object.assign({
blockingTimeout: 10000,
}, this.opts);
if (autorun) {
this.run().catch(error => this.emit('error', error));
}
}
emit(event, ...args) {
return super.emit(event, ...args);
}
off(eventName, listener) {
super.off(eventName, listener);
return this;
}
on(event, listener) {
super.on(event, listener);
return this;
}
once(event, listener) {
super.once(event, listener);
return this;
}
/**
* Manually starts running the event consumming loop. This shall be used if you do not
* use the default "autorun" option on the constructor.
*/
async run() {
if (!this.running) {
try {
this.running = true;
const client = await this.client;
try {
await client.client('SETNAME', this.clientName(QUEUE_EVENT_SUFFIX));
}
catch (err) {
if (!clientCommandMessageReg.test(err.message)) {
throw err;
}
}
await this.consumeEvents(client);
}
catch (error) {
this.running = false;
throw error;
}
}
else {
throw new Error('Queue Events is already running.');
}
}
async consumeEvents(client) {
const opts = this.opts;
const key = this.keys.events;
let id = opts.lastEventId || '$';
while (!this.closing) {
// Cast to actual return type, see: https://github.com/DefinitelyTyped/DefinitelyTyped/issues/44301
const data = await this.checkConnectionError(() => client.xread('BLOCK', opts.blockingTimeout, 'STREAMS', key, id));
if (data) {
const stream = data[0];
const events = stream[1];
for (let i = 0; i < events.length; i++) {
id = events[i][0];
const args = array2obj(events[i][1]);
//
// TODO: we may need to have a separate xtream for progress data
// to avoid this hack.
switch (args.event) {
case 'progress':
args.data = JSON.parse(args.data);
break;
case 'completed':
args.returnvalue = JSON.parse(args.returnvalue);
break;
}
const { event } = args, restArgs = __rest(args, ["event"]);
if (event === 'drained') {
this.emit(event, id);
}
else {
this.emit(event, restArgs, id);
this.emit(`${event}:${restArgs.jobId}`, restArgs, id);
}
}
}
}
}
/**
* Stops consuming events and close the underlying Redis connection if necessary.
*
* @returns
*/
close() {
if (!this.closing) {
this.closing = this.disconnect();
}
return this.closing;
}
}
//# sourceMappingURL=queue-events.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"queue-events.js","sourceRoot":"","sources":["../../../src/classes/queue-events.ts"],"names":[],"mappings":";AAMA,OAAO,EACL,SAAS,EACT,uBAAuB,EACvB,eAAe,EACf,kBAAkB,GACnB,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAgJzC;;;;;;GAMG;AACH,MAAM,OAAO,WAAY,SAAQ,SAAS;IAGxC,YACE,IAAY,EACZ,KAA8D,EAAE,EAChE,UAAmC;YADnC,EAAE,UAAU,EAAE,OAAO,GAAG,IAAI,OAAoC,EAA/B,IAAI,cAArC,yBAAuC,CAAF;QAGrC,KAAK,CACH,IAAI,kCAEC,IAAI,KACP,UAAU,EAAE,eAAe,CAAC,UAAU,CAAC;gBACrC,CAAC,CAAe,UAAW,CAAC,SAAS,EAAE;gBACvC,CAAC,CAAC,UAAU,EACd,kBAAkB,EAAE,IAAI,KAE1B,UAAU,CACX,CAAC;QAjBI,YAAO,GAAG,KAAK,CAAC;QAmBtB,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,MAAM,CACvB;YACE,eAAe,EAAE,KAAK;SACvB,EACD,IAAI,CAAC,IAAI,CACV,CAAC;QAEF,IAAI,OAAO,EAAE;YACX,IAAI,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;SACtD;IACH,CAAC;IAED,IAAI,CACF,KAAQ,EACR,GAAG,IAAwC;QAE3C,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,CAAC;IACpC,CAAC;IAED,GAAG,CACD,SAAY,EACZ,QAAgC;QAEhC,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAC/B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,EAAE,CACA,KAAQ,EACR,QAAgC;QAEhC,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAC1B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CACF,KAAQ,EACR,QAAgC;QAEhC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAC5B,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,GAAG;QACP,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YACjB,IAAI;gBACF,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;gBACpB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC;gBAEjC,IAAI;oBACF,MAAM,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,CAAC;iBACrE;gBAAC,OAAO,GAAG,EAAE;oBACZ,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAS,GAAI,CAAC,OAAO,CAAC,EAAE;wBACvD,MAAM,GAAG,CAAC;qBACX;iBACF;gBAED,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;aAClC;YAAC,OAAO,KAAK,EAAE;gBACd,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;gBACrB,MAAM,KAAK,CAAC;aACb;SACF;aAAM;YACL,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;SACrD;IACH,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,MAAmB;QAC7C,MAAM,IAAI,GAAuB,IAAI,CAAC,IAAI,CAAC;QAE3C,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;QAC7B,IAAI,EAAE,GAAG,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;QAEjC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE;YACpB,mGAAmG;YACnG,MAAM,IAAI,GAAkB,MAAM,IAAI,CAAC,oBAAoB,CAAC,GAAG,EAAE,CAC/D,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,eAAgB,EAAE,SAAS,EAAE,GAAG,EAAE,EAAE,CAAC,CACjE,CAAC;YACF,IAAI,IAAI,EAAE;gBACR,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;gBACvB,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;oBACtC,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBAClB,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBAErC,EAAE;oBACF,gEAAgE;oBAChE,sBAAsB;oBACtB,QAAQ,IAAI,CAAC,KAAK,EAAE;wBAClB,KAAK,UAAU;4BACb,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;4BAClC,MAAM;wBACR,KAAK,WAAW;4BACd,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;4BAChD,MAAM;qBACT;oBAED,MAAM,EAAE,KAAK,KAAkB,IAAI,EAAjB,QAAQ,UAAK,IAAI,EAA7B,SAAsB,CAAO,CAAC;oBAEpC,IAAI,KAAK,KAAK,SAAS,EAAE;wBACvB,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;qBACtB;yBAAM;wBACL,IAAI,CAAC,IAAI,CAAC,KAAY,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC;wBACtC,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,IAAI,QAAQ,CAAC,KAAK,EAAS,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC;qBAC9D;iBACF;aACF;SACF;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YACjB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;SAClC;QACD,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;CACF"}
+208
View File
@@ -0,0 +1,208 @@
import { QueueBase } from './queue-base';
import { Job } from './job';
import { JobState, JobType } from '../types';
import { JobJsonRaw, Metrics } from '../interfaces';
/**
*
* @class QueueGetters
* @extends QueueBase
*
* @description Provides different getters for different aspects of a queue.
*/
export declare class QueueGetters<DataType, ResultType, NameType extends string> extends QueueBase {
getJob(jobId: string): Promise<Job<DataType, ResultType, NameType> | undefined>;
private commandByType;
/**
* Helper to easily extend Job class calls.
*/
protected get Job(): typeof Job;
private sanitizeJobTypes;
/**
Returns the number of jobs waiting to be processed. This includes jobs that are
"waiting" or "delayed" or "prioritized" or "waiting-children".
*/
count(): Promise<number>;
/**
* Returns the time to live for a rate limited key in milliseconds.
* @returns -2 if the key does not exist.
* -1 if the key exists but has no associated expire.
* @see {@link https://redis.io/commands/pttl/}
*/
getRateLimitTtl(): Promise<number>;
/**
* Job counts by type
*
* Queue#getJobCountByTypes('completed') => completed count
* Queue#getJobCountByTypes('completed,failed') => completed + failed count
* Queue#getJobCountByTypes('completed', 'failed') => completed + failed count
* Queue#getJobCountByTypes('completed', 'waiting', 'failed') => completed + waiting + failed count
*/
getJobCountByTypes(...types: JobType[]): Promise<number>;
/**
* Returns the job counts for each type specified or every list/set in the queue by default.
*
* @returns An object, key (type) and value (count)
*/
getJobCounts(...types: JobType[]): Promise<{
[index: string]: number;
}>;
/**
* Get current job state.
*
* @returns Returns one of these values:
* 'completed', 'failed', 'delayed', 'active', 'waiting', 'waiting-children', 'unknown'.
*/
getJobState(jobId: string): Promise<JobState | 'unknown'>;
/**
* Returns the number of jobs in completed status.
*/
getCompletedCount(): Promise<number>;
/**
* Returns the number of jobs in failed status.
*/
getFailedCount(): Promise<number>;
/**
* Returns the number of jobs in delayed status.
*/
getDelayedCount(): Promise<number>;
/**
* Returns the number of jobs in active status.
*/
getActiveCount(): Promise<number>;
/**
* Returns the number of jobs in prioritized status.
*/
getPrioritizedCount(): Promise<number>;
/**
* Returns the number of jobs in waiting or paused statuses.
*/
getWaitingCount(): Promise<number>;
/**
* Returns the number of jobs in waiting-children status.
*/
getWaitingChildrenCount(): Promise<number>;
/**
* Returns the jobs that are in the "waiting" status.
* @param start - zero based index from where to start returning jobs.
* @param end - zero based index where to stop returning jobs.
*/
getWaiting(start?: number, end?: number): Promise<Job<DataType, ResultType, NameType>[]>;
/**
* Returns the jobs that are in the "waiting-children" status.
* I.E. parent jobs that have at least one child that has not completed yet.
* @param start - zero based index from where to start returning jobs.
* @param end - zero based index where to stop returning jobs.
*/
getWaitingChildren(start?: number, end?: number): Promise<Job<DataType, ResultType, NameType>[]>;
/**
* Returns the jobs that are in the "active" status.
* @param start - zero based index from where to start returning jobs.
* @param end - zero based index where to stop returning jobs.
*/
getActive(start?: number, end?: number): Promise<Job<DataType, ResultType, NameType>[]>;
/**
* Returns the jobs that are in the "delayed" status.
* @param start - zero based index from where to start returning jobs.
* @param end - zero based index where to stop returning jobs.
*/
getDelayed(start?: number, end?: number): Promise<Job<DataType, ResultType, NameType>[]>;
/**
* Returns the jobs that are in the "prioritized" status.
* @param start - zero based index from where to start returning jobs.
* @param end - zero based index where to stop returning jobs.
*/
getPrioritized(start?: number, end?: number): Promise<Job<DataType, ResultType, NameType>[]>;
/**
* Returns the jobs that are in the "completed" status.
* @param start - zero based index from where to start returning jobs.
* @param end - zero based index where to stop returning jobs.
*/
getCompleted(start?: number, end?: number): Promise<Job<DataType, ResultType, NameType>[]>;
/**
* Returns the jobs that are in the "failed" status.
* @param start - zero based index from where to start returning jobs.
* @param end - zero based index where to stop returning jobs.
*/
getFailed(start?: number, end?: number): Promise<Job<DataType, ResultType, NameType>[]>;
/**
* Returns the qualified job ids and the raw job data (if available) of the
* children jobs of the given parent job.
* It is possible to get either the already processed children, in this case
* an array of qualified job ids and their result values will be returned,
* or the pending children, in this case an array of qualified job ids will
* be returned.
* A qualified job id is a string representing the job id in a given queue,
* for example: "bull:myqueue:jobid".
*
* @param parentId The id of the parent job
* @param type "processed" | "pending"
* @param opts
*
* @returns { items: { id: string, v?: any, err?: string } [], jobs: JobJsonRaw[], total: number}
*/
getDependencies(parentId: string, type: 'processed' | 'pending', start: number, end: number): Promise<{
items: {
id: string;
v?: any;
err?: string;
}[];
jobs: JobJsonRaw[];
total: number;
}>;
getRanges(types: JobType[], start?: number, end?: number, asc?: boolean): Promise<string[]>;
/**
* Returns the jobs that are on the given statuses (note that JobType is synonym for job status)
* @param types - the statuses of the jobs to return.
* @param start - zero based index from where to start returning jobs.
* @param end - zero based index where to stop returning jobs.
* @param asc - if true, the jobs will be returned in ascending order.
*/
getJobs(types?: JobType[] | JobType, start?: number, end?: number, asc?: boolean): Promise<Job<DataType, ResultType, NameType>[]>;
/**
* Returns the logs for a given Job.
* @param jobId - the id of the job to get the logs for.
* @param start - zero based index from where to start returning jobs.
* @param end - zero based index where to stop returning jobs.
* @param asc - if true, the jobs will be returned in ascending order.
*/
getJobLogs(jobId: string, start?: number, end?: number, asc?: boolean): Promise<{
logs: string[];
count: number;
}>;
private baseGetClients;
/**
* Get the worker list related to the queue. i.e. all the known
* workers that are available to process jobs for this queue.
* Note: GCP does not support SETNAME, so this call will not work
*
* @returns - Returns an array with workers info.
*/
getWorkers(): Promise<{
[index: string]: string;
}[]>;
/**
* Get queue events list related to the queue.
* Note: GCP does not support SETNAME, so this call will not work
*
* @returns - Returns an array with queue events info.
*/
getQueueEvents(): Promise<{
[index: string]: string;
}[]>;
/**
* Get queue metrics related to the queue.
*
* This method returns the gathered metrics for the queue.
* The metrics are represented as an array of job counts
* per unit of time (1 minute).
*
* @param start - Start point of the metrics, where 0
* is the newest point to be returned.
* @param end - End point of the metrics, where -1 is the
* oldest point to be returned.
*
* @returns - Returns an object with queue metrics.
*/
getMetrics(type: 'completed' | 'failed', start?: number, end?: number): Promise<Metrics>;
private parseClientList;
}
+402
View File
@@ -0,0 +1,402 @@
/*eslint-env node */
'use strict';
import { QueueBase } from './queue-base';
import { Job } from './job';
import { clientCommandMessageReg, QUEUE_EVENT_SUFFIX, WORKER_SUFFIX, } from '../utils';
/**
*
* @class QueueGetters
* @extends QueueBase
*
* @description Provides different getters for different aspects of a queue.
*/
export class QueueGetters extends QueueBase {
getJob(jobId) {
return this.Job.fromId(this, jobId);
}
commandByType(types, count, callback) {
return types.map((type) => {
type = type === 'waiting' ? 'wait' : type; // alias
const key = this.toKey(type);
switch (type) {
case 'completed':
case 'failed':
case 'delayed':
case 'prioritized':
case 'repeat':
case 'waiting-children':
return callback(key, count ? 'zcard' : 'zrange');
case 'active':
case 'wait':
case 'paused':
return callback(key, count ? 'llen' : 'lrange');
}
});
}
/**
* Helper to easily extend Job class calls.
*/
get Job() {
return Job;
}
sanitizeJobTypes(types) {
const currentTypes = typeof types === 'string' ? [types] : types;
if (Array.isArray(currentTypes) && currentTypes.length > 0) {
const sanitizedTypes = [...currentTypes];
if (sanitizedTypes.indexOf('waiting') !== -1) {
sanitizedTypes.push('paused');
}
return [...new Set(sanitizedTypes)];
}
return [
'active',
'completed',
'delayed',
'failed',
'paused',
'prioritized',
'waiting',
'waiting-children',
];
}
/**
Returns the number of jobs waiting to be processed. This includes jobs that are
"waiting" or "delayed" or "prioritized" or "waiting-children".
*/
async count() {
const count = await this.getJobCountByTypes('waiting', 'paused', 'delayed', 'prioritized', 'waiting-children');
return count;
}
/**
* Returns the time to live for a rate limited key in milliseconds.
* @returns -2 if the key does not exist.
* -1 if the key exists but has no associated expire.
* @see {@link https://redis.io/commands/pttl/}
*/
async getRateLimitTtl() {
const client = await this.client;
return client.pttl(this.keys.limiter);
}
/**
* Job counts by type
*
* Queue#getJobCountByTypes('completed') => completed count
* Queue#getJobCountByTypes('completed,failed') => completed + failed count
* Queue#getJobCountByTypes('completed', 'failed') => completed + failed count
* Queue#getJobCountByTypes('completed', 'waiting', 'failed') => completed + waiting + failed count
*/
async getJobCountByTypes(...types) {
const result = await this.getJobCounts(...types);
return Object.values(result).reduce((sum, count) => sum + count, 0);
}
/**
* Returns the job counts for each type specified or every list/set in the queue by default.
*
* @returns An object, key (type) and value (count)
*/
async getJobCounts(...types) {
const currentTypes = this.sanitizeJobTypes(types);
const responses = await this.scripts.getCounts(currentTypes);
const counts = {};
responses.forEach((res, index) => {
counts[currentTypes[index]] = res || 0;
});
return counts;
}
/**
* Get current job state.
*
* @returns Returns one of these values:
* 'completed', 'failed', 'delayed', 'active', 'waiting', 'waiting-children', 'unknown'.
*/
getJobState(jobId) {
return this.scripts.getState(jobId);
}
/**
* Returns the number of jobs in completed status.
*/
getCompletedCount() {
return this.getJobCountByTypes('completed');
}
/**
* Returns the number of jobs in failed status.
*/
getFailedCount() {
return this.getJobCountByTypes('failed');
}
/**
* Returns the number of jobs in delayed status.
*/
getDelayedCount() {
return this.getJobCountByTypes('delayed');
}
/**
* Returns the number of jobs in active status.
*/
getActiveCount() {
return this.getJobCountByTypes('active');
}
/**
* Returns the number of jobs in prioritized status.
*/
getPrioritizedCount() {
return this.getJobCountByTypes('prioritized');
}
/**
* Returns the number of jobs in waiting or paused statuses.
*/
getWaitingCount() {
return this.getJobCountByTypes('waiting');
}
/**
* Returns the number of jobs in waiting-children status.
*/
getWaitingChildrenCount() {
return this.getJobCountByTypes('waiting-children');
}
/**
* Returns the jobs that are in the "waiting" status.
* @param start - zero based index from where to start returning jobs.
* @param end - zero based index where to stop returning jobs.
*/
getWaiting(start = 0, end = -1) {
return this.getJobs(['waiting'], start, end, true);
}
/**
* Returns the jobs that are in the "waiting-children" status.
* I.E. parent jobs that have at least one child that has not completed yet.
* @param start - zero based index from where to start returning jobs.
* @param end - zero based index where to stop returning jobs.
*/
getWaitingChildren(start = 0, end = -1) {
return this.getJobs(['waiting-children'], start, end, true);
}
/**
* Returns the jobs that are in the "active" status.
* @param start - zero based index from where to start returning jobs.
* @param end - zero based index where to stop returning jobs.
*/
getActive(start = 0, end = -1) {
return this.getJobs(['active'], start, end, true);
}
/**
* Returns the jobs that are in the "delayed" status.
* @param start - zero based index from where to start returning jobs.
* @param end - zero based index where to stop returning jobs.
*/
getDelayed(start = 0, end = -1) {
return this.getJobs(['delayed'], start, end, true);
}
/**
* Returns the jobs that are in the "prioritized" status.
* @param start - zero based index from where to start returning jobs.
* @param end - zero based index where to stop returning jobs.
*/
getPrioritized(start = 0, end = -1) {
return this.getJobs(['prioritized'], start, end, true);
}
/**
* Returns the jobs that are in the "completed" status.
* @param start - zero based index from where to start returning jobs.
* @param end - zero based index where to stop returning jobs.
*/
getCompleted(start = 0, end = -1) {
return this.getJobs(['completed'], start, end, false);
}
/**
* Returns the jobs that are in the "failed" status.
* @param start - zero based index from where to start returning jobs.
* @param end - zero based index where to stop returning jobs.
*/
getFailed(start = 0, end = -1) {
return this.getJobs(['failed'], start, end, false);
}
/**
* Returns the qualified job ids and the raw job data (if available) of the
* children jobs of the given parent job.
* It is possible to get either the already processed children, in this case
* an array of qualified job ids and their result values will be returned,
* or the pending children, in this case an array of qualified job ids will
* be returned.
* A qualified job id is a string representing the job id in a given queue,
* for example: "bull:myqueue:jobid".
*
* @param parentId The id of the parent job
* @param type "processed" | "pending"
* @param opts
*
* @returns { items: { id: string, v?: any, err?: string } [], jobs: JobJsonRaw[], total: number}
*/
async getDependencies(parentId, type, start, end) {
const key = this.toKey(type == 'processed'
? `${parentId}:processed`
: `${parentId}:dependencies`);
const { items, total, jobs } = await this.scripts.paginate(key, {
start,
end,
fetchJobs: true,
});
return {
items,
jobs,
total,
};
}
async getRanges(types, start = 0, end = 1, asc = false) {
const multiCommands = [];
this.commandByType(types, false, (key, command) => {
switch (command) {
case 'lrange':
multiCommands.push('lrange');
break;
case 'zrange':
multiCommands.push('zrange');
break;
}
});
const responses = await this.scripts.getRanges(types, start, end, asc);
let results = [];
responses.forEach((response, index) => {
const result = response || [];
if (asc && multiCommands[index] === 'lrange') {
results = results.concat(result.reverse());
}
else {
results = results.concat(result);
}
});
return [...new Set(results)];
}
/**
* Returns the jobs that are on the given statuses (note that JobType is synonym for job status)
* @param types - the statuses of the jobs to return.
* @param start - zero based index from where to start returning jobs.
* @param end - zero based index where to stop returning jobs.
* @param asc - if true, the jobs will be returned in ascending order.
*/
async getJobs(types, start = 0, end = -1, asc = false) {
const currentTypes = this.sanitizeJobTypes(types);
const jobIds = await this.getRanges(currentTypes, start, end, asc);
return Promise.all(jobIds.map(jobId => this.Job.fromId(this, jobId)));
}
/**
* Returns the logs for a given Job.
* @param jobId - the id of the job to get the logs for.
* @param start - zero based index from where to start returning jobs.
* @param end - zero based index where to stop returning jobs.
* @param asc - if true, the jobs will be returned in ascending order.
*/
async getJobLogs(jobId, start = 0, end = -1, asc = true) {
const client = await this.client;
const multi = client.multi();
const logsKey = this.toKey(jobId + ':logs');
if (asc) {
multi.lrange(logsKey, start, end);
}
else {
multi.lrange(logsKey, -(end + 1), -(start + 1));
}
multi.llen(logsKey);
const result = (await multi.exec());
if (!asc) {
result[0][1].reverse();
}
return {
logs: result[0][1],
count: result[1][1],
};
}
async baseGetClients(suffix) {
const client = await this.client;
const clients = (await client.client('LIST'));
try {
const list = this.parseClientList(clients, suffix);
return list;
}
catch (err) {
if (!clientCommandMessageReg.test(err.message)) {
throw err;
}
return [];
}
}
/**
* Get the worker list related to the queue. i.e. all the known
* workers that are available to process jobs for this queue.
* Note: GCP does not support SETNAME, so this call will not work
*
* @returns - Returns an array with workers info.
*/
getWorkers() {
return this.baseGetClients(WORKER_SUFFIX);
}
/**
* Get queue events list related to the queue.
* Note: GCP does not support SETNAME, so this call will not work
*
* @returns - Returns an array with queue events info.
*/
async getQueueEvents() {
return this.baseGetClients(QUEUE_EVENT_SUFFIX);
}
/**
* Get queue metrics related to the queue.
*
* This method returns the gathered metrics for the queue.
* The metrics are represented as an array of job counts
* per unit of time (1 minute).
*
* @param start - Start point of the metrics, where 0
* is the newest point to be returned.
* @param end - End point of the metrics, where -1 is the
* oldest point to be returned.
*
* @returns - Returns an object with queue metrics.
*/
async getMetrics(type, start = 0, end = -1) {
const client = await this.client;
const metricsKey = this.toKey(`metrics:${type}`);
const dataKey = `${metricsKey}:data`;
const multi = client.multi();
multi.hmget(metricsKey, 'count', 'prevTS', 'prevCount');
multi.lrange(dataKey, start, end);
multi.llen(dataKey);
const [hmget, range, len] = (await multi.exec());
const [err, [count, prevTS, prevCount]] = hmget;
const [err2, data] = range;
const [err3, numPoints] = len;
if (err || err2) {
throw err || err2 || err3;
}
return {
meta: {
count: parseInt(count || '0', 10),
prevTS: parseInt(prevTS || '0', 10),
prevCount: parseInt(prevCount || '0', 10),
},
data,
count: numPoints,
};
}
parseClientList(list, suffix = '') {
const lines = list.split('\n');
const clients = [];
lines.forEach((line) => {
const client = {};
const keyValues = line.split(' ');
keyValues.forEach(function (keyValue) {
const index = keyValue.indexOf('=');
const key = keyValue.substring(0, index);
const value = keyValue.substring(index + 1);
client[key] = value;
});
const name = client['name'];
if (name && name === `${this.clientName()}${suffix ? `${suffix}` : ''}`) {
client['name'] = this.name;
clients.push(client);
}
});
return clients;
}
}
//# sourceMappingURL=queue-getters.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"queue-getters.js","sourceRoot":"","sources":["../../../src/classes/queue-getters.ts"],"names":[],"mappings":"AAAA,oBAAoB;AACpB,YAAY,CAAC;AAEb,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,GAAG,EAAE,MAAM,OAAO,CAAC;AAC5B,OAAO,EACL,uBAAuB,EACvB,kBAAkB,EAClB,aAAa,GACd,MAAM,UAAU,CAAC;AAIlB;;;;;;GAMG;AACH,MAAM,OAAO,YAIX,SAAQ,SAAS;IACjB,MAAM,CACJ,KAAa;QAEb,OAAO,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,CAEjC,CAAC;IACJ,CAAC;IAEO,aAAa,CACnB,KAAgB,EAChB,KAAc,EACd,QAAiD;QAEjD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAY,EAAE,EAAE;YAChC,IAAI,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ;YAEnD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAE7B,QAAQ,IAAI,EAAE;gBACZ,KAAK,WAAW,CAAC;gBACjB,KAAK,QAAQ,CAAC;gBACd,KAAK,SAAS,CAAC;gBACf,KAAK,aAAa,CAAC;gBACnB,KAAK,QAAQ,CAAC;gBACd,KAAK,kBAAkB;oBACrB,OAAO,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;gBACnD,KAAK,QAAQ,CAAC;gBACd,KAAK,MAAM,CAAC;gBACZ,KAAK,QAAQ;oBACX,OAAO,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;aACnD;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,IAAc,GAAG;QACf,OAAO,GAAG,CAAC;IACb,CAAC;IAEO,gBAAgB,CAAC,KAAsC;QAC7D,MAAM,YAAY,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QAEjE,IAAI,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE;YAC1D,MAAM,cAAc,GAAG,CAAC,GAAG,YAAY,CAAC,CAAC;YAEzC,IAAI,cAAc,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE;gBAC5C,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;aAC/B;YAED,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC;SACrC;QAED,OAAO;YACL,QAAQ;YACR,WAAW;YACX,SAAS;YACT,QAAQ;YACR,QAAQ;YACR,aAAa;YACb,SAAS;YACT,kBAAkB;SACnB,CAAC;IACJ,CAAC;IAED;;;MAGE;IACF,KAAK,CAAC,KAAK;QACT,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,kBAAkB,CACzC,SAAS,EACT,QAAQ,EACR,SAAS,EACT,aAAa,EACb,kBAAkB,CACnB,CAAC;QAEF,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,eAAe;QACnB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC;QACjC,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACxC,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,kBAAkB,CAAC,GAAG,KAAgB;QAC1C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,KAAK,CAAC,CAAC;QACjD,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC;IACtE,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,YAAY,CAAC,GAAG,KAAgB;QAGpC,MAAM,YAAY,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAElD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;QAE7D,MAAM,MAAM,GAAgC,EAAE,CAAC;QAC/C,SAAS,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;YAC/B,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;;;OAKG;IACH,WAAW,CAAC,KAAa;QACvB,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,iBAAiB;QACf,OAAO,IAAI,CAAC,kBAAkB,CAAC,WAAW,CAAC,CAAC;IAC9C,CAAC;IAED;;OAEG;IACH,cAAc;QACZ,OAAO,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,eAAe;QACb,OAAO,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACH,cAAc;QACZ,OAAO,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,mBAAmB;QACjB,OAAO,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,CAAC;IAChD,CAAC;IAED;;OAEG;IACH,eAAe;QACb,OAAO,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACH,uBAAuB;QACrB,OAAO,IAAI,CAAC,kBAAkB,CAAC,kBAAkB,CAAC,CAAC;IACrD,CAAC;IAED;;;;OAIG;IACH,UAAU,CACR,KAAK,GAAG,CAAC,EACT,GAAG,GAAG,CAAC,CAAC;QAER,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;IACrD,CAAC;IAED;;;;;OAKG;IACH,kBAAkB,CAChB,KAAK,GAAG,CAAC,EACT,GAAG,GAAG,CAAC,CAAC;QAER,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,kBAAkB,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;IAC9D,CAAC;IAED;;;;OAIG;IACH,SAAS,CACP,KAAK,GAAG,CAAC,EACT,GAAG,GAAG,CAAC,CAAC;QAER,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;IACpD,CAAC;IAED;;;;OAIG;IACH,UAAU,CACR,KAAK,GAAG,CAAC,EACT,GAAG,GAAG,CAAC,CAAC;QAER,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;IACrD,CAAC;IAED;;;;OAIG;IACH,cAAc,CACZ,KAAK,GAAG,CAAC,EACT,GAAG,GAAG,CAAC,CAAC;QAER,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,aAAa,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;IACzD,CAAC;IAED;;;;OAIG;IACH,YAAY,CACV,KAAK,GAAG,CAAC,EACT,GAAG,GAAG,CAAC,CAAC;QAER,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,WAAW,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;IACxD,CAAC;IAED;;;;OAIG;IACH,SAAS,CACP,KAAK,GAAG,CAAC,EACT,GAAG,GAAG,CAAC,CAAC;QAER,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;IACrD,CAAC;IAED;;;;;;;;;;;;;;;OAeG;IACH,KAAK,CAAC,eAAe,CACnB,QAAgB,EAChB,IAA6B,EAC7B,KAAa,EACb,GAAW;QAMX,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CACpB,IAAI,IAAI,WAAW;YACjB,CAAC,CAAC,GAAG,QAAQ,YAAY;YACzB,CAAC,CAAC,GAAG,QAAQ,eAAe,CAC/B,CAAC;QACF,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE;YAC9D,KAAK;YACL,GAAG;YACH,SAAS,EAAE,IAAI;SAChB,CAAC,CAAC;QACH,OAAO;YACL,KAAK;YACL,IAAI;YACJ,KAAK;SACN,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,SAAS,CACb,KAAgB,EAChB,KAAK,GAAG,CAAC,EACT,GAAG,GAAG,CAAC,EACP,GAAG,GAAG,KAAK;QAEX,MAAM,aAALine truncated
+10
View File
@@ -0,0 +1,10 @@
export type KeysMap = {
[index in string]: string;
};
export declare class QueueKeys {
readonly prefix: string;
constructor(prefix?: string);
getKeys(name: string): KeysMap;
toKey(name: string, type: string): string;
getQueueQualifiedName(name: string): string;
}
+37
View File
@@ -0,0 +1,37 @@
export class QueueKeys {
constructor(prefix = 'bull') {
this.prefix = prefix;
}
getKeys(name) {
const keys = {};
[
'',
'active',
'wait',
'waiting-children',
'paused',
'id',
'delayed',
'prioritized',
'stalled-check',
'completed',
'failed',
'stalled',
'repeat',
'limiter',
'meta',
'events',
'pc',
].forEach(key => {
keys[key] = this.toKey(name, key);
});
return keys;
}
toKey(name, type) {
return `${this.getQueueQualifiedName(name)}:${type}`;
}
getQueueQualifiedName(name) {
return `${this.prefix}:${name}`;
}
}
//# sourceMappingURL=queue-keys.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"queue-keys.js","sourceRoot":"","sources":["../../../src/classes/queue-keys.ts"],"names":[],"mappings":"AAEA,MAAM,OAAO,SAAS;IACpB,YAA4B,SAAiB,MAAM;QAAvB,WAAM,GAAN,MAAM,CAAiB;IAAG,CAAC;IAEvD,OAAO,CAAC,IAAY;QAClB,MAAM,IAAI,GAAgC,EAAE,CAAC;QAC7C;YACE,EAAE;YACF,QAAQ;YACR,MAAM;YACN,kBAAkB;YAClB,QAAQ;YACR,IAAI;YACJ,SAAS;YACT,aAAa;YACb,eAAe;YACf,WAAW;YACX,QAAQ;YACR,SAAS;YACT,QAAQ;YACR,SAAS;YACT,MAAM;YACN,QAAQ;YACR,IAAI;SACL,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;YACd,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,IAAY,EAAE,IAAY;QAC9B,OAAO,GAAG,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;IACvD,CAAC;IAED,qBAAqB,CAAC,IAAY;QAChC,OAAO,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC;IAClC,CAAC;CACF"}
+271
View File
@@ -0,0 +1,271 @@
import { BaseJobOptions, BulkJobOptions, IoredisListener, QueueOptions, RepeatableJob, RepeatOptions } from '../interfaces';
import { FinishedStatus, JobsOptions } from '../types';
import { Job } from './job';
import { QueueGetters } from './queue-getters';
import { Repeat } from './repeat';
import { RedisConnection } from './redis-connection';
export interface ObliterateOpts {
/**
* Use force = true to force obliteration even with active jobs in the queue
* @defaultValue false
*/
force?: boolean;
/**
* Use count with the maximum number of deleted keys per iteration
* @defaultValue 1000
*/
count?: number;
}
export interface QueueListener<DataType, ResultType, NameType extends string> extends IoredisListener {
/**
* Listen to 'cleaned' event.
*
* This event is triggered when the queue calls clean method.
*/
cleaned: (jobs: string[], type: string) => void;
/**
* Listen to 'error' event.
*
* This event is triggered when an error is thrown.
*/
error: (err: Error) => void;
/**
* Listen to 'paused' event.
*
* This event is triggered when the queue is paused.
*/
paused: () => void;
/**
* Listen to 'progress' event.
*
* This event is triggered when the job updates its progress.
*/
progress: (job: Job<DataType, ResultType, NameType>, progress: number | object) => void;
/**
* Listen to 'removed' event.
*
* This event is triggered when a job is removed.
*/
removed: (job: Job<DataType, ResultType, NameType>) => void;
/**
* Listen to 'resumed' event.
*
* This event is triggered when the queue is resumed.
*/
resumed: () => void;
/**
* Listen to 'waiting' event.
*
* This event is triggered when the queue creates a new job.
*/
waiting: (job: Job<DataType, ResultType, NameType>) => void;
}
/**
* Queue
*
* This class provides methods to add jobs to a queue and some othe high-level
* administration such as pausing or deleting queues.
*
*/
export declare class Queue<DataType = any, ResultType = any, NameType extends string = string> extends QueueGetters<DataType, ResultType, NameType> {
token: string;
jobsOpts: BaseJobOptions;
opts: QueueOptions;
private _repeat?;
protected libName: string;
constructor(name: string, opts?: QueueOptions, Connection?: typeof RedisConnection);
emit<U extends keyof QueueListener<DataType, ResultType, NameType>>(event: U, ...args: Parameters<QueueListener<DataType, ResultType, NameType>[U]>): boolean;
off<U extends keyof QueueListener<DataType, ResultType, NameType>>(eventName: U, listener: QueueListener<DataType, ResultType, NameType>[U]): this;
on<U extends keyof QueueListener<DataType, ResultType, NameType>>(event: U, listener: QueueListener<DataType, ResultType, NameType>[U]): this;
once<U extends keyof QueueListener<DataType, ResultType, NameType>>(event: U, listener: QueueListener<DataType, ResultType, NameType>[U]): this;
/**
* Returns this instance current default job options.
*/
get defaultJobOptions(): JobsOptions;
get metaValues(): Record<string, string | number>;
/**
* Get library version.
*
* @returns the content of the meta.library field.
*/
getVersion(): Promise<string>;
get repeat(): Promise<Repeat>;
/**
* Adds a new job to the queue.
*
* @param name - Name of the job to be added to the queue,.
* @param data - Arbitrary data to append to the job.
* @param opts - Job options that affects how the job is going to be processed.
*/
add(name: NameType, data: DataType, opts?: JobsOptions): Promise<Job<DataType, ResultType, NameType>>;
/**
* Adds an array of jobs to the queue. This method may be faster than adding
* one job at a time in a sequence.
*
* @param jobs - The array of jobs to add to the queue. Each job is defined by 3
* properties, 'name', 'data' and 'opts'. They follow the same signature as 'Queue.add'.
*/
addBulk(jobs: {
name: NameType;
data: DataType;
opts?: BulkJobOptions;
}[]): Promise<Job<DataType, ResultType, NameType>[]>;
/**
* Pauses the processing of this queue globally.
*
* We use an atomic RENAME operation on the wait queue. Since
* we have blocking calls with BRPOPLPUSH on the wait queue, as long as the queue
* is renamed to 'paused', no new jobs will be processed (the current ones
* will run until finalized).
*
* Adding jobs requires a LUA script to check first if the paused list exist
* and in that case it will add it there instead of the wait list.
*/
pause(): Promise<void>;
/**
* Close the queue instance.
*
*/
close(): Promise<void>;
/**
* Resumes the processing of this queue globally.
*
* The method reverses the pause operation by resuming the processing of the
* queue.
*/
resume(): Promise<void>;
/**
* Returns true if the queue is currently paused.
*/
isPaused(): Promise<boolean>;
/**
* Get all repeatable meta jobs.
*
* @param start - Offset of first job to return.
* @param end - Offset of last job to return.
* @param asc - Determine the order in which jobs are returned based on their
* next execution time.
*/
getRepeatableJobs(start?: number, end?: number, asc?: boolean): Promise<RepeatableJob[]>;
/**
* Removes a repeatable job.
*
* Note: you need to use the exact same repeatOpts when deleting a repeatable job
* than when adding it.
*
* @see removeRepeatableByKey
*
* @param name - job name
* @param repeatOpts -
* @param jobId -
* @returns
*/
removeRepeatable(name: NameType, repeatOpts: RepeatOptions, jobId?: string): Promise<boolean>;
/**
* Removes a repeatable job by its key. Note that the key is the one used
* to store the repeatable job metadata and not one of the job iterations
* themselves. You can use "getRepeatableJobs" in order to get the keys.
*
* @see getRepeatableJobs
*
* @param repeatJobKey - to the repeatable job.
* @returns
*/
removeRepeatableByKey(key: string): Promise<boolean>;
/**
* Removes the given job from the queue as well as all its
* dependencies.
*
* @param jobId - The id of the job to remove
* @param opts - Options to remove a job
* @returns 1 if it managed to remove the job or 0 if the job or
* any of its dependencies were locked.
*/
remove(jobId: string, { removeChildren }?: {
removeChildren?: boolean;
}): Promise<number>;
/**
* Updates the given job's progress.
*
* @param jobId - The id of the job to update
* @param progress - number or object to be saved as progress.
*/
updateJobProgress(jobId: string, progress: number | object): Promise<void>;
/**
* Logs one row of job's log data.
*
* @param jobId - The job id to log against.
* @param logRow - string with log data to be logged.
* @param keepLogs - max number of log entries to keep (0 for unlimited).
*
* @returns The total number of log entries for this job so far.
*/
addJobLog(jobId: string, logRow: string, keepLogs?: number): Promise<number>;
/**
* Drains the queue, i.e., removes all jobs that are waiting
* or delayed, but not active, completed or failed.
*
* @param delayed - Pass true if it should also clean the
* delayed jobs.
*/
drain(delayed?: boolean): Promise<void>;
/**
* Cleans jobs from a queue. Similar to drain but keeps jobs within a certain
* grace period.
*
* @param grace - The grace period
* @param limit - Max number of jobs to clean
* @param type - The type of job to clean
* Possible values are completed, wait, active, paused, delayed, failed. Defaults to completed.
* @returns Id jobs from the deleted records
*/
clean(grace: number, limit: number, type?: 'completed' | 'wait' | 'active' | 'paused' | 'prioritized' | 'delayed' | 'failed'): Promise<string[]>;
/**
* Completely destroys the queue and all of its contents irreversibly.
* This method will the *pause* the queue and requires that there are no
* active jobs. It is possible to bypass this requirement, i.e. not
* having active jobs using the "force" option.
*
* Note: This operation requires to iterate on all the jobs stored in the queue
* and can be slow for very large queues.
*
* @param opts - Obliterate options.
*/
obliterate(opts?: ObliterateOpts): Promise<void>;
/**
* Retry all the failed jobs.
*
* @param opts: { count: number; state: FinishedStatus; timestamp: number}
* - count number to limit how many jobs will be moved to wait status per iteration,
* - state failed by default or completed.
* - timestamp from which timestamp to start moving jobs to wait status, default Date.now().
*
* @returns
*/
retryJobs(opts?: {
count?: number;
state?: FinishedStatus;
timestamp?: number;
}): Promise<void>;
/**
* Promote all the delayed jobs.
*
* @param opts: { count: number }
* - count number to limit how many jobs will be moved to wait status per iteration
*
* @returns
*/
promoteJobs(opts?: {
count?: number;
}): Promise<void>;
/**
* Trim the event stream to an approximately maxLength.
*
* @param maxLength -
*/
trimEvents(maxLength: number): Promise<number>;
/**
* Delete old priority helper key.
*/
removeDeprecatedPriorityKey(): Promise<number>;
}
+340
View File
@@ -0,0 +1,340 @@
import { get } from 'lodash';
import { v4 } from 'uuid';
import { Job } from './job';
import { QueueGetters } from './queue-getters';
import { Repeat } from './repeat';
import { version } from '../version';
/**
* Queue
*
* This class provides methods to add jobs to a queue and some othe high-level
* administration such as pausing or deleting queues.
*
*/
export class Queue extends QueueGetters {
constructor(name, opts, Connection) {
var _a;
super(name, Object.assign({ blockingConnection: false }, opts), Connection);
this.token = v4();
this.libName = 'bullmq';
this.jobsOpts = (_a = get(opts, 'defaultJobOptions')) !== null && _a !== void 0 ? _a : {};
this.waitUntilReady()
.then(client => {
if (!this.closing && !(opts === null || opts === void 0 ? void 0 : opts.skipMetasUpdate)) {
return client.hmset(this.keys.meta, this.metaValues);
}
})
.catch(err => {
// We ignore this error to avoid warnings. The error can still
// be received by listening to event 'error'
});
}
emit(event, ...args) {
return super.emit(event, ...args);
}
off(eventName, listener) {
super.off(eventName, listener);
return this;
}
on(event, listener) {
super.on(event, listener);
return this;
}
once(event, listener) {
super.once(event, listener);
return this;
}
/**
* Returns this instance current default job options.
*/
get defaultJobOptions() {
return Object.assign({}, this.jobsOpts);
}
get metaValues() {
var _a, _b, _c, _d;
return {
'opts.maxLenEvents': (_d = (_c = (_b = (_a = this.opts) === null || _a === void 0 ? void 0 : _a.streams) === null || _b === void 0 ? void 0 : _b.events) === null || _c === void 0 ? void 0 : _c.maxLen) !== null && _d !== void 0 ? _d : 10000,
version: `${this.libName}:${version}`,
};
}
/**
* Get library version.
*
* @returns the content of the meta.library field.
*/
async getVersion() {
const client = await this.client;
return await client.hget(this.keys.meta, 'version');
}
get repeat() {
return new Promise(async (resolve) => {
if (!this._repeat) {
this._repeat = new Repeat(this.name, Object.assign(Object.assign({}, this.opts), { connection: await this.client }));
this._repeat.on('error', e => this.emit.bind(this, e));
}
resolve(this._repeat);
});
}
/**
* Adds a new job to the queue.
*
* @param name - Name of the job to be added to the queue,.
* @param data - Arbitrary data to append to the job.
* @param opts - Job options that affects how the job is going to be processed.
*/
async add(name, data, opts) {
if (opts && opts.repeat) {
return (await this.repeat).addNextRepeatableJob(name, data, Object.assign(Object.assign({}, this.jobsOpts), opts), true);
}
else {
const jobId = opts === null || opts === void 0 ? void 0 : opts.jobId;
if (jobId == '0' || (jobId === null || jobId === void 0 ? void 0 : jobId.startsWith('0:'))) {
throw new Error("JobId cannot be '0' or start with 0:");
}
const job = await this.Job.create(this, name, data, Object.assign(Object.assign(Object.assign({}, this.jobsOpts), opts), { jobId }));
this.emit('waiting', job);
return job;
}
}
/**
* Adds an array of jobs to the queue. This method may be faster than adding
* one job at a time in a sequence.
*
* @param jobs - The array of jobs to add to the queue. Each job is defined by 3
* properties, 'name', 'data' and 'opts'. They follow the same signature as 'Queue.add'.
*/
addBulk(jobs) {
return this.Job.createBulk(this, jobs.map(job => {
var _a;
return ({
name: job.name,
data: job.data,
opts: Object.assign(Object.assign(Object.assign({}, this.jobsOpts), job.opts), { jobId: (_a = job.opts) === null || _a === void 0 ? void 0 : _a.jobId }),
});
}));
}
/**
* Pauses the processing of this queue globally.
*
* We use an atomic RENAME operation on the wait queue. Since
* we have blocking calls with BRPOPLPUSH on the wait queue, as long as the queue
* is renamed to 'paused', no new jobs will be processed (the current ones
* will run until finalized).
*
* Adding jobs requires a LUA script to check first if the paused list exist
* and in that case it will add it there instead of the wait list.
*/
async pause() {
await this.scripts.pause(true);
this.emit('paused');
}
/**
* Close the queue instance.
*
*/
async close() {
if (!this.closing) {
if (this._repeat) {
await this._repeat.close();
}
}
return super.close();
}
/**
* Resumes the processing of this queue globally.
*
* The method reverses the pause operation by resuming the processing of the
* queue.
*/
async resume() {
await this.scripts.pause(false);
this.emit('resumed');
}
/**
* Returns true if the queue is currently paused.
*/
async isPaused() {
const client = await this.client;
const pausedKeyExists = await client.hexists(this.keys.meta, 'paused');
return pausedKeyExists === 1;
}
/**
* Get all repeatable meta jobs.
*
* @param start - Offset of first job to return.
* @param end - Offset of last job to return.
* @param asc - Determine the order in which jobs are returned based on their
* next execution time.
*/
async getRepeatableJobs(start, end, asc) {
return (await this.repeat).getRepeatableJobs(start, end, asc);
}
/**
* Removes a repeatable job.
*
* Note: you need to use the exact same repeatOpts when deleting a repeatable job
* than when adding it.
*
* @see removeRepeatableByKey
*
* @param name - job name
* @param repeatOpts -
* @param jobId -
* @returns
*/
async removeRepeatable(name, repeatOpts, jobId) {
const repeat = await this.repeat;
const removed = await repeat.removeRepeatable(name, repeatOpts, jobId);
return !removed;
}
/**
* Removes a repeatable job by its key. Note that the key is the one used
* to store the repeatable job metadata and not one of the job iterations
* themselves. You can use "getRepeatableJobs" in order to get the keys.
*
* @see getRepeatableJobs
*
* @param repeatJobKey - to the repeatable job.
* @returns
*/
async removeRepeatableByKey(key) {
const repeat = await this.repeat;
const removed = await repeat.removeRepeatableByKey(key);
return !removed;
}
/**
* Removes the given job from the queue as well as all its
* dependencies.
*
* @param jobId - The id of the job to remove
* @param opts - Options to remove a job
* @returns 1 if it managed to remove the job or 0 if the job or
* any of its dependencies were locked.
*/
remove(jobId, { removeChildren = true } = {}) {
return this.scripts.remove(jobId, removeChildren);
}
/**
* Updates the given job's progress.
*
* @param jobId - The id of the job to update
* @param progress - number or object to be saved as progress.
*/
async updateJobProgress(jobId, progress) {
return this.scripts.updateProgress(jobId, progress);
}
/**
* Logs one row of job's log data.
*
* @param jobId - The job id to log against.
* @param logRow - string with log data to be logged.
* @param keepLogs - max number of log entries to keep (0 for unlimited).
*
* @returns The total number of log entries for this job so far.
*/
async addJobLog(jobId, logRow, keepLogs) {
return Job.addJobLog(this, jobId, logRow, keepLogs);
}
/**
* Drains the queue, i.e., removes all jobs that are waiting
* or delayed, but not active, completed or failed.
*
* @param delayed - Pass true if it should also clean the
* delayed jobs.
*/
drain(delayed = false) {
return this.scripts.drain(delayed);
}
/**
* Cleans jobs from a queue. Similar to drain but keeps jobs within a certain
* grace period.
*
* @param grace - The grace period
* @param limit - Max number of jobs to clean
* @param type - The type of job to clean
* Possible values are completed, wait, active, paused, delayed, failed. Defaults to completed.
* @returns Id jobs from the deleted records
*/
async clean(grace, limit, type = 'completed') {
const maxCount = limit || Infinity;
const maxCountPerCall = Math.min(10000, maxCount);
const timestamp = Date.now() - grace;
let deletedCount = 0;
const deletedJobsIds = [];
while (deletedCount < maxCount) {
const jobsIds = await this.scripts.cleanJobsInSet(type, timestamp, maxCountPerCall);
this.emit('cleaned', jobsIds, type);
deletedCount += jobsIds.length;
deletedJobsIds.push(...jobsIds);
if (jobsIds.length < maxCountPerCall) {
break;
}
}
return deletedJobsIds;
}
/**
* Completely destroys the queue and all of its contents irreversibly.
* This method will the *pause* the queue and requires that there are no
* active jobs. It is possible to bypass this requirement, i.e. not
* having active jobs using the "force" option.
*
* Note: This operation requires to iterate on all the jobs stored in the queue
* and can be slow for very large queues.
*
* @param opts - Obliterate options.
*/
async obliterate(opts) {
await this.pause();
let cursor = 0;
do {
cursor = await this.scripts.obliterate(Object.assign({ force: false, count: 1000 }, opts));
} while (cursor);
}
/**
* Retry all the failed jobs.
*
* @param opts: { count: number; state: FinishedStatus; timestamp: number}
* - count number to limit how many jobs will be moved to wait status per iteration,
* - state failed by default or completed.
* - timestamp from which timestamp to start moving jobs to wait status, default Date.now().
*
* @returns
*/
async retryJobs(opts = {}) {
let cursor = 0;
do {
cursor = await this.scripts.retryJobs(opts.state, opts.count, opts.timestamp);
} while (cursor);
}
/**
* Promote all the delayed jobs.
*
* @param opts: { count: number }
* - count number to limit how many jobs will be moved to wait status per iteration
*
* @returns
*/
async promoteJobs(opts = {}) {
let cursor = 0;
do {
cursor = await this.scripts.promoteJobs(opts.count);
} while (cursor);
}
/**
* Trim the event stream to an approximately maxLength.
*
* @param maxLength -
*/
async trimEvents(maxLength) {
const client = await this.client;
return client.xtrim(this.keys.events, 'MAXLEN', '~', maxLength);
}
/**
* Delete old priority helper key.
*/
async removeDeprecatedPriorityKey() {
const client = await this.client;
return client.del(this.toKey('priority'));
}
}
//# sourceMappingURL=queue.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"queue.js","sourceRoot":"","sources":["../../../src/classes/queue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAC;AAC7B,OAAO,EAAE,EAAE,EAAE,MAAM,MAAM,CAAC;AAU1B,OAAO,EAAE,GAAG,EAAE,MAAM,OAAO,CAAC;AAC5B,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAElC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAsErC;;;;;;GAMG;AACH,MAAM,OAAO,KAIX,SAAQ,YAA4C;IAQpD,YACE,IAAY,EACZ,IAAmB,EACnB,UAAmC;;QAEnC,KAAK,CACH,IAAI,kBAEF,kBAAkB,EAAE,KAAK,IACtB,IAAI,GAET,UAAU,CACX,CAAC;QAnBJ,UAAK,GAAG,EAAE,EAAE,CAAC;QAKH,YAAO,GAAG,QAAQ,CAAC;QAgB3B,IAAI,CAAC,QAAQ,GAAG,MAAA,GAAG,CAAC,IAAI,EAAE,mBAAmB,CAAC,mCAAI,EAAE,CAAC;QAErD,IAAI,CAAC,cAAc,EAAE;aAClB,IAAI,CAAC,MAAM,CAAC,EAAE;YACb,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,CAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,eAAe,CAAA,EAAE;gBAC3C,OAAO,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;aACtD;QACH,CAAC,CAAC;aACD,KAAK,CAAC,GAAG,CAAC,EAAE;YACX,8DAA8D;YAC9D,4CAA4C;QAC9C,CAAC,CAAC,CAAC;IACP,CAAC;IAED,IAAI,CACF,KAAQ,EACR,GAAG,IAAkE;QAErE,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,CAAC;IACpC,CAAC;IAED,GAAG,CACD,SAAY,EACZ,QAA0D;QAE1D,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAC/B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,EAAE,CACA,KAAQ,EACR,QAA0D;QAE1D,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAC1B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CACF,KAAQ,EACR,QAA0D;QAE1D,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAC5B,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,IAAI,iBAAiB;QACnB,yBAAY,IAAI,CAAC,QAAQ,EAAG;IAC9B,CAAC;IAED,IAAI,UAAU;;QACZ,OAAO;YACL,mBAAmB,EAAE,MAAA,MAAA,MAAA,MAAA,IAAI,CAAC,IAAI,0CAAE,OAAO,0CAAE,MAAM,0CAAE,MAAM,mCAAI,KAAK;YAChE,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,IAAI,OAAO,EAAE;SACtC,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,UAAU;QACd,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC;QACjC,OAAO,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IACtD,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,OAAO,CAAS,KAAK,EAAC,OAAO,EAAC,EAAE;YACzC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;gBACjB,IAAI,CAAC,OAAO,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,kCAC9B,IAAI,CAAC,IAAI,KACZ,UAAU,EAAE,MAAM,IAAI,CAAC,MAAM,IAC7B,CAAC;gBACH,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;aACxD;YACD,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,GAAG,CACP,IAAc,EACd,IAAc,EACd,IAAkB;QAElB,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,EAAE;YACvB,OAAO,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,oBAAoB,CAI7C,IAAI,EAAE,IAAI,kCAAO,IAAI,CAAC,QAAQ,GAAK,IAAI,GAAI,IAAI,CAAC,CAAC;SACpD;aAAM;YACL,MAAM,KAAK,GAAG,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,KAAK,CAAC;YAE1B,IAAI,KAAK,IAAI,GAAG,KAAI,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,UAAU,CAAC,IAAI,CAAC,CAAA,EAAE;gBAC3C,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;aACzD;YAED,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,MAAM,CAC/B,IAAoB,EACpB,IAAI,EACJ,IAAI,gDAEC,IAAI,CAAC,QAAQ,GACb,IAAI,KACP,KAAK,IAER,CAAC;YACF,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;YAC1B,OAAO,GAAG,CAAC;SACZ;IACH,CAAC;IAED;;;;;;OAMG;IACH,OAAO,CACL,IAAiE;QAEjE,OAAO,IAAI,CAAC,GAAG,CAAC,UAAU,CACxB,IAAoB,EACpB,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;;YAAC,OAAA,CAAC;gBACf,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,IAAI,gDACC,IAAI,CAAC,QAAQ,GACb,GAAG,CAAC,IAAI,KACX,KAAK,EAAE,MAAA,GAAG,CAAC,IAAI,0CAAE,KAAK,GACvB;aACF,CAAC,CAAA;SAAA,CAAC,CACJ,CAAC;IACJ,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC/B,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACtB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YACjB,IAAI,IAAI,CAAC,OAAO,EAAE;gBAChB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;aAC5B;SACF;QACD,OAAO,KAAK,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IACD;;;;;OAKG;IACH,KAAK,CAAC,MAAM;QACV,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ;QACZ,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC;QACjC,MAAM,eAAe,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACvE,OAAO,eAAe,KAAK,CAAC,CAAC;IAC/B,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,iBAAiB,CACrB,KAAc,EACd,GAAY,EACZ,GAAa;QAEb,OAAO,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;IAChE,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,gBAAgB,CACpB,IAAc,EACd,UAAyB,EACzB,KAAc;QAEd,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC;QACjC,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,IAAI,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;QAEvE,OAAO,CAAC,OAAO,CAAC;IAClB,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,qBAAqB,CAAC,GAAW;QACrC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC;QACjC,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC;QAExD,OAAO,CAAC,OAAO,CAAC;IAClB,CAAC;IAED;;;;;;;;OAQG;IACH,MAAM,CAAC,KAAa,EAAE,EAAE,cAAc,GAAG,IAAI,EAAE,GAAG,EAAE;QAClD,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;IACpD,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,iBAAiB,CACrB,KAAa,EACb,QAAyB;QAEzB,OAAO,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IACtD,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,SAAS,CACb,KAAa,EACb,MAAc,EACd,QAAiB;QAEjLine truncated
+43
View File
@@ -0,0 +1,43 @@
/// <reference types="node" />
import { EventEmitter } from 'events';
import { ConnectionOptions, RedisClient } from '../interfaces';
interface RedisCapabilities {
canDoubleTimeout: boolean;
}
export interface RawCommand {
content: string;
name: string;
keys: number;
}
export declare class RedisConnection extends EventEmitter {
private readonly shared;
private readonly blocking;
static minimumVersion: string;
static recommendedMinimumVersion: string;
closing: boolean;
capabilities: RedisCapabilities;
protected _client: RedisClient;
private readonly opts;
private readonly initializing;
private version;
private skipVersionCheck;
private handleClientError;
private handleClientClose;
private handleClientReady;
constructor(opts?: ConnectionOptions, shared?: boolean, blocking?: boolean, skipVersionCheck?: boolean);
private checkBlockingOptions;
/**
* Waits for a redis client to be ready.
* @param redis - client
*/
static waitUntilReady(client: RedisClient): Promise<void>;
get client(): Promise<RedisClient>;
protected loadCommands(version?: string, providedScripts?: Record<string, RawCommand>): void;
private init;
disconnect(wait?: boolean): Promise<void>;
reconnect(): Promise<void>;
close(): Promise<void>;
private getRedisVersion;
get redisVersion(): string;
}
export {};
+229
View File
@@ -0,0 +1,229 @@
import { EventEmitter } from 'events';
import { default as IORedis } from 'ioredis';
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
import { CONNECTION_CLOSED_ERROR_MSG } from 'ioredis/built/utils';
import { decreaseMaxListeners, increaseMaxListeners, isNotConnectionError, isRedisCluster, isRedisInstance, isRedisVersionLowerThan, } from '../utils';
import { version } from '../version';
import * as scripts from '../scripts';
const overrideMessage = [
'BullMQ: WARNING! Your redis options maxRetriesPerRequest must be null',
'and will be overridden by BullMQ.',
].join(' ');
const deprecationMessage = [
'BullMQ: DEPRECATION WARNING! Your redis options maxRetriesPerRequest must be null.',
'On the next versions having this settings will throw an exception',
].join(' ');
export class RedisConnection extends EventEmitter {
constructor(opts, shared = false, blocking = true, skipVersionCheck = false) {
super();
this.shared = shared;
this.blocking = blocking;
this.capabilities = {
canDoubleTimeout: false,
};
if (!isRedisInstance(opts)) {
this.checkBlockingOptions(overrideMessage, opts);
this.opts = Object.assign({ port: 6379, host: '127.0.0.1', retryStrategy: function (times) {
return Math.max(Math.min(Math.exp(times), 20000), 1000);
} }, opts);
if (this.blocking) {
this.opts.maxRetriesPerRequest = null;
}
}
else {
this._client = opts;
// Test if the redis instance is using keyPrefix
// and if so, throw an error.
if (this._client.options.keyPrefix) {
throw new Error('BullMQ: ioredis does not support ioredis prefixes, use the prefix option instead.');
}
if (isRedisCluster(this._client)) {
this.opts = this._client.options.redisOptions;
}
else {
this.opts = this._client.options;
}
this.checkBlockingOptions(deprecationMessage, this.opts);
}
this.skipVersionCheck =
skipVersionCheck || !!(this.opts && this.opts.skipVersionCheck);
this.handleClientError = (err) => {
this.emit('error', err);
};
this.handleClientClose = () => {
this.emit('close');
};
this.handleClientReady = () => {
this.emit('ready');
};
this.initializing = this.init();
this.initializing.catch(err => this.emit('error', err));
}
checkBlockingOptions(msg, options) {
if (this.blocking && options && options.maxRetriesPerRequest) {
console.error(msg);
}
}
/**
* Waits for a redis client to be ready.
* @param redis - client
*/
static async waitUntilReady(client) {
if (client.status === 'ready') {
return;
}
if (client.status === 'wait') {
return client.connect();
}
if (client.status === 'end') {
throw new Error(CONNECTION_CLOSED_ERROR_MSG);
}
let handleReady;
let handleEnd;
let handleError;
try {
await new Promise((resolve, reject) => {
let lastError;
handleError = (err) => {
lastError = err;
};
handleReady = () => {
resolve();
};
handleEnd = () => {
reject(lastError || new Error(CONNECTION_CLOSED_ERROR_MSG));
};
increaseMaxListeners(client, 3);
client.once('ready', handleReady);
client.on('end', handleEnd);
client.once('error', handleError);
});
}
finally {
client.removeListener('end', handleEnd);
client.removeListener('error', handleError);
client.removeListener('ready', handleReady);
decreaseMaxListeners(client, 3);
}
}
get client() {
return this.initializing;
}
loadCommands(version, providedScripts) {
const finalScripts = providedScripts || scripts;
for (const property in finalScripts) {
// Only define the command if not already defined
const commandName = `${finalScripts[property].name}:${version}`;
if (!this._client[commandName]) {
this._client.defineCommand(commandName, {
numberOfKeys: finalScripts[property].keys,
lua: finalScripts[property].content,
});
}
}
}
async init() {
if (!this._client) {
this._client = new IORedis(this.opts);
}
increaseMaxListeners(this._client, 3);
this._client.on('error', this.handleClientError);
// ioredis treats connection errors as a different event ('close')
this._client.on('close', this.handleClientClose);
this._client.on('ready', this.handleClientReady);
await RedisConnection.waitUntilReady(this._client);
this.loadCommands(version);
this.version = await this.getRedisVersion();
if (this.skipVersionCheck !== true && !this.closing) {
if (isRedisVersionLowerThan(this.version, RedisConnection.minimumVersion)) {
throw new Error(`Redis version needs to be greater or equal than ${RedisConnection.minimumVersion} Current: ${this.version}`);
}
if (isRedisVersionLowerThan(this.version, RedisConnection.recommendedMinimumVersion)) {
console.warn(`It is highly recommended to use a minimum Redis version of ${RedisConnection.recommendedMinimumVersion}
Current: ${this.version}`);
}
}
this.capabilities = {
canDoubleTimeout: !isRedisVersionLowerThan(this.version, '6.0.0'),
};
return this._client;
}
async disconnect(wait = true) {
const client = await this.client;
if (client.status !== 'end') {
let _resolve, _reject;
if (!wait) {
return client.disconnect();
}
const disconnecting = new Promise((resolve, reject) => {
increaseMaxListeners(client, 2);
client.once('end', resolve);
client.once('error', reject);
_resolve = resolve;
_reject = reject;
});
client.disconnect();
try {
await disconnecting;
}
finally {
decreaseMaxListeners(client, 2);
client.removeListener('end', _resolve);
client.removeListener('error', _reject);
}
}
}
async reconnect() {
const client = await this.client;
return client.connect();
}
async close() {
if (!this.closing) {
this.closing = true;
try {
await this.initializing;
if (!this.shared) {
await this._client.quit();
}
}
catch (error) {
if (isNotConnectionError(error)) {
throw error;
}
}
finally {
this._client.off('error', this.handleClientError);
this._client.off('close', this.handleClientClose);
this._client.off('ready', this.handleClientReady);
decreaseMaxListeners(this._client, 3);
this.removeAllListeners();
}
}
}
async getRedisVersion() {
const doc = await this._client.info();
const redisPrefix = 'redis_version:';
const maxMemoryPolicyPrefix = 'maxmemory_policy:';
const lines = doc.split('\r\n');
let redisVersion;
for (let i = 0; i < lines.length; i++) {
if (lines[i].indexOf(maxMemoryPolicyPrefix) === 0) {
const maxMemoryPolicy = lines[i].substr(maxMemoryPolicyPrefix.length);
if (maxMemoryPolicy !== 'noeviction') {
console.warn(`IMPORTANT! Eviction policy is ${maxMemoryPolicy}. It should be "noeviction"`);
}
}
if (lines[i].indexOf(redisPrefix) === 0) {
redisVersion = lines[i].substr(redisPrefix.length);
}
}
return redisVersion;
}
get redisVersion() {
return this.version;
}
}
RedisConnection.minimumVersion = '5.0.0';
RedisConnection.recommendedMinimumVersion = '6.2.0';
//# sourceMappingURL=redis-connection.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"redis-connection.js","sourceRoot":"","sources":["../../../src/classes/redis-connection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,EAAE,OAAO,IAAI,OAAO,EAAE,MAAM,SAAS,CAAC;AAC7C,6DAA6D;AAC7D,aAAa;AACb,OAAO,EAAE,2BAA2B,EAAE,MAAM,qBAAqB,CAAC;AAElE,OAAO,EACL,oBAAoB,EACpB,oBAAoB,EACpB,oBAAoB,EACpB,cAAc,EACd,eAAe,EACf,uBAAuB,GACxB,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,KAAK,OAAO,MAAM,YAAY,CAAC;AAEtC,MAAM,eAAe,GAAG;IACtB,uEAAuE;IACvE,mCAAmC;CACpC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAEZ,MAAM,kBAAkB,GAAG;IACzB,oFAAoF;IACpF,mEAAmE;CACpE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAYZ,MAAM,OAAO,eAAgB,SAAQ,YAAY;IAoB/C,YACE,IAAwB,EACP,SAAkB,KAAK,EACvB,WAAW,IAAI,EAChC,gBAAgB,GAAG,KAAK;QAExB,KAAK,EAAE,CAAC;QAJS,WAAM,GAAN,MAAM,CAAiB;QACvB,aAAQ,GAAR,QAAQ,CAAO;QAlBlC,iBAAY,GAAsB;YAChC,gBAAgB,EAAE,KAAK;SACxB,CAAC;QAqBA,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,EAAE;YAC1B,IAAI,CAAC,oBAAoB,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;YAEjD,IAAI,CAAC,IAAI,mBACP,IAAI,EAAE,IAAI,EACV,IAAI,EAAE,WAAW,EACjB,aAAa,EAAE,UAAU,KAAa;oBACpC,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,EAAE,IAAI,CAAC,CAAC;gBAC1D,CAAC,IACE,IAAI,CACR,CAAC;YAEF,IAAI,IAAI,CAAC,QAAQ,EAAE;gBACjB,IAAI,CAAC,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC;aACvC;SACF;aAAM;YACL,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;YAEpB,gDAAgD;YAChD,6BAA6B;YAC7B,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE;gBAClC,MAAM,IAAI,KAAK,CACb,mFAAmF,CACpF,CAAC;aACH;YAED,IAAI,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;gBAChC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC;aAC/C;iBAAM;gBACL,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;aAClC;YAED,IAAI,CAAC,oBAAoB,CAAC,kBAAkB,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;SAC1D;QAED,IAAI,CAAC,gBAAgB;YACnB,gBAAgB,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAClE,IAAI,CAAC,iBAAiB,GAAG,CAAC,GAAU,EAAQ,EAAE;YAC5C,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAC1B,CAAC,CAAC;QAEF,IAAI,CAAC,iBAAiB,GAAG,GAAS,EAAE;YAClC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACrB,CAAC,CAAC;QAEF,IAAI,CAAC,iBAAiB,GAAG,GAAS,EAAE;YAClC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACrB,CAAC,CAAC;QAEF,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;IAC1D,CAAC;IAEO,oBAAoB,CAAC,GAAW,EAAE,OAAsB;QAC9D,IAAI,IAAI,CAAC,QAAQ,IAAI,OAAO,IAAI,OAAO,CAAC,oBAAoB,EAAE;YAC5D,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;SACpB;IACH,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,MAAmB;QAC7C,IAAI,MAAM,CAAC,MAAM,KAAK,OAAO,EAAE;YAC7B,OAAO;SACR;QAED,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,EAAE;YAC5B,OAAO,MAAM,CAAC,OAAO,EAAE,CAAC;SACzB;QAED,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK,EAAE;YAC3B,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;SAC9C;QAED,IAAI,WAAuB,CAAC;QAC5B,IAAI,SAAqB,CAAC;QAC1B,IAAI,WAA+B,CAAC;QACpC,IAAI;YACF,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBAC1C,IAAI,SAAgB,CAAC;gBAErB,WAAW,GAAG,CAAC,GAAU,EAAE,EAAE;oBAC3B,SAAS,GAAG,GAAG,CAAC;gBAClB,CAAC,CAAC;gBAEF,WAAW,GAAG,GAAG,EAAE;oBACjB,OAAO,EAAE,CAAC;gBACZ,CAAC,CAAC;gBAEF,SAAS,GAAG,GAAG,EAAE;oBACf,MAAM,CAAC,SAAS,IAAI,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC,CAAC;gBAC9D,CAAC,CAAC;gBAEF,oBAAoB,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;gBAEhC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;gBAClC,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;gBAC5B,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;YACpC,CAAC,CAAC,CAAC;SACJ;gBAAS;YACR,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;YACxC,MAAM,CAAC,cAAc,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;YAC5C,MAAM,CAAC,cAAc,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;YAE5C,oBAAoB,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;SACjC;IACH,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAES,YAAY,CACpB,OAAgB,EAChB,eAA4C;QAE5C,MAAM,YAAY,GAChB,eAAe,IAAK,OAAsC,CAAC;QAC7D,KAAK,MAAM,QAAQ,IAAI,YAA0C,EAAE;YACjE,iDAAiD;YACjD,MAAM,WAAW,GAAG,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC,IAAI,IAAI,OAAO,EAAE,CAAC;YAEhE,IAAI,CAAO,IAAI,CAAC,OAAQ,CAAC,WAAW,CAAC,EAAE;gBAC/B,IAAI,CAAC,OAAQ,CAAC,aAAa,CAAC,WAAW,EAAE;oBAC7C,YAAY,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC,IAAI;oBACzC,GAAG,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC,OAAO;iBACpC,CAAC,CAAC;aACJ;SACF;IACH,CAAC;IAEO,KAAK,CAAC,IAAI;QAChB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YACjB,IAAI,CAAC,OAAO,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SACvC;QAED,oBAAoB,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAEtC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACjD,kEAAkE;QAClE,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAEjD,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAEjD,MAAM,eAAe,CAAC,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACnD,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;QAE3B,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAC5C,IAAI,IAAI,CAAC,gBAAgB,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YACnD,IACE,uBAAuB,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,cAAc,CAAC,EACrE;gBACA,MAAM,IAAI,KAAK,CACb,mDAAmD,eAAe,CAAC,cAAc,aAAa,IAAI,CAAC,OAAO,EAAE,CAC7G,CAAC;aACH;YAED,IACE,uBAAuB,CACrB,IAAI,CAAC,OAAO,EACZ,eAAe,CAAC,yBAAyB,CAC1C,EACD;gBACA,OAAO,CAAC,IAAI,CACV,8DAA8D,eAAe,CAAC,yBAAyB;sBAC3F,IAAI,CAAC,OAALine truncated
+20
View File
@@ -0,0 +1,20 @@
import { RepeatBaseOptions, RepeatableJob, RepeatOptions } from '../interfaces';
import { JobsOptions } from '../types';
import { Job } from './job';
import { QueueBase } from './queue-base';
import { RedisConnection } from './redis-connection';
export declare class Repeat extends QueueBase {
private repeatStrategy;
private repeatKeyHashAlgorithm;
constructor(name: string, opts: RepeatBaseOptions, Connection?: typeof RedisConnection);
addNextRepeatableJob<T = any, R = any, N extends string = string>(name: N, data: T, opts: JobsOptions, skipCheckExists?: boolean): Promise<Job<T, R, N> | undefined>;
private createNextJob;
removeRepeatable(name: string, repeat: RepeatOptions, jobId?: string): Promise<number>;
removeRepeatableByKey(repeatJobKey: string): Promise<number>;
private keyToData;
getRepeatableJobs(start?: number, end?: number, asc?: boolean): Promise<RepeatableJob[]>;
getRepeatableCount(): Promise<number>;
private hash;
private getRepeatJobId;
}
export declare const getNextMillis: (millis: number, opts: RepeatOptions) => number | undefined;
+146
View File
@@ -0,0 +1,146 @@
import { __rest } from "tslib";
import { parseExpression } from 'cron-parser';
import { createHash } from 'crypto';
import { QueueBase } from './queue-base';
export class Repeat extends QueueBase {
constructor(name, opts, Connection) {
super(name, opts, Connection);
this.repeatStrategy =
(opts.settings && opts.settings.repeatStrategy) || getNextMillis;
this.repeatKeyHashAlgorithm =
(opts.settings && opts.settings.repeatKeyHashAlgorithm) || 'md5';
}
async addNextRepeatableJob(name, data, opts, skipCheckExists) {
var _a;
// HACK: This is a temporary fix to enable easy migration from bullmq <3.0.0
// to >= 3.0.0. It should be removed when moving to 4.x.
const repeatOpts = Object.assign({}, opts.repeat);
(_a = repeatOpts.pattern) !== null && _a !== void 0 ? _a : (repeatOpts.pattern = repeatOpts.cron);
delete repeatOpts.cron;
const prevMillis = opts.prevMillis || 0;
const currentCount = repeatOpts.count ? repeatOpts.count + 1 : 1;
if (typeof repeatOpts.limit !== 'undefined' &&
currentCount > repeatOpts.limit) {
return;
}
let now = Date.now();
if (!(typeof repeatOpts.endDate === undefined) &&
now > new Date(repeatOpts.endDate).getTime()) {
return;
}
now = prevMillis < now ? now : prevMillis;
const nextMillis = await this.repeatStrategy(now, repeatOpts, name);
const pattern = repeatOpts.pattern;
const hasImmediately = Boolean((repeatOpts.every || pattern) && repeatOpts.immediately);
const offset = hasImmediately ? now - nextMillis : undefined;
if (nextMillis) {
// We store the undecorated opts.jobId into the repeat options
if (!prevMillis && opts.jobId) {
repeatOpts.jobId = opts.jobId;
}
const repeatJobKey = getRepeatKey(name, repeatOpts);
let repeatableExists = true;
if (!skipCheckExists) {
// Check that the repeatable job hasn't been removed
// TODO: a lua script would be better here
const client = await this.client;
repeatableExists = !!(await client.zscore(this.keys.repeat, repeatJobKey));
}
const { immediately } = repeatOpts, filteredRepeatOpts = __rest(repeatOpts, ["immediately"]);
// The job could have been deleted since this check
if (repeatableExists) {
return this.createNextJob(name, nextMillis, repeatJobKey, Object.assign(Object.assign({}, opts), { repeat: Object.assign({ offset }, filteredRepeatOpts) }), data, currentCount, hasImmediately);
}
}
}
async createNextJob(name, nextMillis, repeatJobKey, opts, data, currentCount, hasImmediately) {
const client = await this.client;
//
// Generate unique job id for this iteration.
//
const jobId = this.getRepeatJobId(name, nextMillis, this.hash(repeatJobKey), opts.repeat.jobId);
const now = Date.now();
const delay = nextMillis + (opts.repeat.offset ? opts.repeat.offset : 0) - now;
const mergedOpts = Object.assign(Object.assign({}, opts), { jobId, delay: delay < 0 || hasImmediately ? 0 : delay, timestamp: now, prevMillis: nextMillis, repeatJobKey });
mergedOpts.repeat = Object.assign(Object.assign({}, opts.repeat), { count: currentCount });
await client.zadd(this.keys.repeat, nextMillis.toString(), repeatJobKey);
return this.Job.create(this, name, data, mergedOpts);
}
async removeRepeatable(name, repeat, jobId) {
const repeatJobKey = getRepeatKey(name, Object.assign(Object.assign({}, repeat), { jobId }));
const repeatJobId = this.getRepeatJobId(name, '', this.hash(repeatJobKey), jobId || repeat.jobId);
return this.scripts.removeRepeatable(repeatJobId, repeatJobKey);
}
async removeRepeatableByKey(repeatJobKey) {
const data = this.keyToData(repeatJobKey);
const repeatJobId = this.getRepeatJobId(data.name, '', this.hash(repeatJobKey), data.id);
return this.scripts.removeRepeatable(repeatJobId, repeatJobKey);
}
keyToData(key, next) {
const data = key.split(':');
const pattern = data.slice(4).join(':') || null;
return {
key,
name: data[0],
id: data[1] || null,
endDate: parseInt(data[2]) || null,
tz: data[3] || null,
pattern,
next,
};
}
async getRepeatableJobs(start = 0, end = -1, asc = false) {
const client = await this.client;
const key = this.keys.repeat;
const result = asc
? await client.zrange(key, start, end, 'WITHSCORES')
: await client.zrevrange(key, start, end, 'WITHSCORES');
const jobs = [];
for (let i = 0; i < result.length; i += 2) {
jobs.push(this.keyToData(result[i], parseInt(result[i + 1])));
}
return jobs;
}
async getRepeatableCount() {
const client = await this.client;
return client.zcard(this.toKey('repeat'));
}
hash(str) {
return createHash(this.repeatKeyHashAlgorithm).update(str).digest('hex');
}
getRepeatJobId(name, nextMillis, namespace, jobId) {
const checksum = this.hash(`${name}${jobId || ''}${namespace}`);
return `repeat:${checksum}:${nextMillis}`;
// return `repeat:${jobId || ''}:${name}:${namespace}:${nextMillis}`;
//return `repeat:${name}:${namespace}:${nextMillis}`;
}
}
function getRepeatKey(name, repeat) {
const endDate = repeat.endDate ? new Date(repeat.endDate).getTime() : '';
const tz = repeat.tz || '';
const pattern = repeat.pattern;
const suffix = (pattern ? pattern : String(repeat.every)) || '';
const jobId = repeat.jobId ? repeat.jobId : '';
return `${name}:${jobId}:${endDate}:${tz}:${suffix}`;
}
export const getNextMillis = (millis, opts) => {
const pattern = opts.pattern;
if (pattern && opts.every) {
throw new Error('Both .pattern and .every options are defined for this repeatable job');
}
if (opts.every) {
return (Math.floor(millis / opts.every) * opts.every +
(opts.immediately ? 0 : opts.every));
}
const currentDate = opts.startDate && new Date(opts.startDate) > new Date(millis)
? new Date(opts.startDate)
: new Date(millis);
const interval = parseExpression(pattern, Object.assign(Object.assign({}, opts), { currentDate }));
try {
return interval.next().getTime();
}
catch (e) {
// Ignore error
}
};
//# sourceMappingURL=repeat.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"repeat.js","sourceRoot":"","sources":["../../../src/classes/repeat.ts"],"names":[],"mappings":";AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAIpC,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAGzC,MAAM,OAAO,MAAO,SAAQ,SAAS;IAInC,YACE,IAAY,EACZ,IAAuB,EACvB,UAAmC;QAEnC,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;QAE9B,IAAI,CAAC,cAAc;YACjB,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,aAAa,CAAC;QAEnE,IAAI,CAAC,sBAAsB;YACzB,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,CAAC,sBAAsB,CAAC,IAAI,KAAK,CAAC;IACrE,CAAC;IAED,KAAK,CAAC,oBAAoB,CACxB,IAAO,EACP,IAAO,EACP,IAAiB,EACjB,eAAyB;;QAEzB,4EAA4E;QAC5E,wDAAwD;QACxD,MAAM,UAAU,qBAA2C,IAAI,CAAC,MAAM,CAAE,CAAC;QACzE,MAAA,UAAU,CAAC,OAAO,oCAAlB,UAAU,CAAC,OAAO,GAAK,UAAU,CAAC,IAAI,EAAC;QACvC,OAAO,UAAU,CAAC,IAAI,CAAC;QAEvB,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QACxC,MAAM,YAAY,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAEjE,IACE,OAAO,UAAU,CAAC,KAAK,KAAK,WAAW;YACvC,YAAY,GAAG,UAAU,CAAC,KAAK,EAC/B;YACA,OAAO;SACR;QAED,IAAI,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAErB,IACE,CAAC,CAAC,OAAO,UAAU,CAAC,OAAO,KAAK,SAAS,CAAC;YAC1C,GAAG,GAAG,IAAI,IAAI,CAAC,UAAU,CAAC,OAAQ,CAAC,CAAC,OAAO,EAAE,EAC7C;YACA,OAAO;SACR;QAED,GAAG,GAAG,UAAU,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC;QAE1C,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC;QACpE,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC;QAEnC,MAAM,cAAc,GAAG,OAAO,CAC5B,CAAC,UAAU,CAAC,KAAK,IAAI,OAAO,CAAC,IAAI,UAAU,CAAC,WAAW,CACxD,CAAC;QACF,MAAM,MAAM,GAAG,cAAc,CAAC,CAAC,CAAC,GAAG,GAAG,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC;QAC7D,IAAI,UAAU,EAAE;YACd,8DAA8D;YAC9D,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,KAAK,EAAE;gBAC7B,UAAU,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;aAC/B;YAED,MAAM,YAAY,GAAG,YAAY,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YAEpD,IAAI,gBAAgB,GAAG,IAAI,CAAC;YAE5B,IAAI,CAAC,eAAe,EAAE;gBACpB,oDAAoD;gBACpD,0CAA0C;gBAC1C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC;gBACjC,gBAAgB,GAAG,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,MAAM,CACvC,IAAI,CAAC,IAAI,CAAC,MAAM,EAChB,YAAY,CACb,CAAC,CAAC;aACJ;YACD,MAAM,EAAE,WAAW,KAA4B,UAAU,EAAjC,kBAAkB,UAAK,UAAU,EAAnD,eAAsC,CAAa,CAAC;YAE1D,mDAAmD;YACnD,IAAI,gBAAgB,EAAE;gBACpB,OAAO,IAAI,CAAC,aAAa,CACvB,IAAI,EACJ,UAAU,EACV,YAAY,kCACP,IAAI,KAAE,MAAM,kBAAI,MAAM,IAAK,kBAAkB,MAClD,IAAI,EACJ,YAAY,EACZ,cAAc,CACf,CAAC;aACH;SACF;IACH,CAAC;IAEO,KAAK,CAAC,aAAa,CACzB,IAAO,EACP,UAAkB,EAClB,YAAoB,EACpB,IAAiB,EACjB,IAAO,EACP,YAAoB,EACpB,cAAuB;QAEvB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC;QAEjC,EAAE;QACF,6CAA6C;QAC7C,EAAE;QACF,MAAM,KAAK,GAAG,IAAI,CAAC,cAAc,CAC/B,IAAI,EACJ,UAAU,EACV,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,EACvB,IAAI,CAAC,MAAM,CAAC,KAAK,CAClB,CAAC;QACF,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,KAAK,GACT,UAAU,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC;QAEnE,MAAM,UAAU,mCACX,IAAI,KACP,KAAK,EACL,KAAK,EAAE,KAAK,GAAG,CAAC,IAAI,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,EAC9C,SAAS,EAAE,GAAG,EACd,UAAU,EAAE,UAAU,EACtB,YAAY,GACb,CAAC;QAEF,UAAU,CAAC,MAAM,mCAAQ,IAAI,CAAC,MAAM,KAAE,KAAK,EAAE,YAAY,GAAE,CAAC;QAE5D,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,UAAU,CAAC,QAAQ,EAAE,EAAE,YAAY,CAAC,CAAC;QAEzE,OAAO,IAAI,CAAC,GAAG,CAAC,MAAM,CAAU,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;IAChE,CAAC;IAED,KAAK,CAAC,gBAAgB,CACpB,IAAY,EACZ,MAAqB,EACrB,KAAc;QAEd,MAAM,YAAY,GAAG,YAAY,CAAC,IAAI,kCAAO,MAAM,KAAE,KAAK,IAAG,CAAC;QAC9D,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CACrC,IAAI,EACJ,EAAE,EACF,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,EACvB,KAAK,IAAI,MAAM,CAAC,KAAK,CACtB,CAAC;QAEF,OAAO,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;IAClE,CAAC;IAED,KAAK,CAAC,qBAAqB,CAAC,YAAoB;QAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;QAE1C,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CACrC,IAAI,CAAC,IAAI,EACT,EAAE,EACF,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,EACvB,IAAI,CAAC,EAAE,CACR,CAAC;QAEF,OAAO,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;IAClE,CAAC;IAEO,SAAS,CAAC,GAAW,EAAE,IAAa;QAC1C,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC;QAEhD,OAAO;YACL,GAAG;YACH,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;YACb,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI;YACnB,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI;YAClC,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI;YACnB,OAAO;YACP,IAAI;SACL,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,iBAAiB,CACrB,KAAK,GAAG,CAAC,EACT,GAAG,GAAG,CAAC,CAAC,EACR,GAAG,GAAG,KAAK;QAEX,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC;QAEjC,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;QAC7B,MAAM,MAAM,GAAG,GAAG;YAChB,CAAC,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,YAAY,CAAC;YACpD,CAAC,CAAC,MAAM,MAAM,CAAC,SAAS,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,YAAY,CAAC,CAAC;QAE1D,MAAM,IAAI,GAAG,EAAE,CAAC;QAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE;YACzC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;SAC/D;QACD,OAAO,IAAI,CAAC;IACd,CAALine truncated
+4
View File
@@ -0,0 +1,4 @@
import { ChildPool } from './child-pool';
import { Job } from './job';
declare const sandbox: <T, R, N extends string>(processFile: any, childPool: ChildPool) => (job: Job<T, R, N>, token?: string) => Promise<R>;
export default sandbox;
+63
View File
@@ -0,0 +1,63 @@
import { ChildCommand, ParentCommand } from '../enums';
const sandbox = (processFile, childPool) => {
return async function process(job, token) {
const child = await childPool.retain(processFile);
let msgHandler;
let exitHandler;
await child.send({
cmd: ChildCommand.Start,
job: job.asJSONSandbox(),
token,
});
const done = new Promise((resolve, reject) => {
msgHandler = async (msg) => {
var _a, _b;
switch (msg.cmd) {
case ParentCommand.Completed:
resolve(msg.value);
break;
case ParentCommand.Failed:
case ParentCommand.Error: {
const err = new Error();
Object.assign(err, msg.value);
reject(err);
break;
}
case ParentCommand.Progress:
await job.updateProgress(msg.value);
break;
case ParentCommand.Log:
await job.log(msg.value);
break;
case ParentCommand.MoveToDelayed:
await job.moveToDelayed((_a = msg.value) === null || _a === void 0 ? void 0 : _a.timestamp, (_b = msg.value) === null || _b === void 0 ? void 0 : _b.token);
break;
case ParentCommand.Update:
await job.updateData(msg.value);
break;
}
};
exitHandler = (exitCode, signal) => {
reject(new Error('Unexpected exit code: ' + exitCode + ' signal: ' + signal));
};
child.on('message', msgHandler);
child.on('exit', exitHandler);
});
try {
await done;
return done;
}
finally {
child.off('message', msgHandler);
child.off('exit', exitHandler);
if (child.exitCode !== null || /SIG.*/.test(`${child.signalCode}`)) {
childPool.remove(child);
}
else {
childPool.release(child);
}
}
};
};
export default sandbox;
//# sourceMappingURL=sandbox.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"sandbox.js","sourceRoot":"","sources":["../../../src/classes/sandbox.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAKvD,MAAM,OAAO,GAAG,CACd,WAAgB,EAChB,SAAoB,EACpB,EAAE;IACF,OAAO,KAAK,UAAU,OAAO,CAAC,GAAiB,EAAE,KAAc;QAC7D,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QAClD,IAAI,UAAe,CAAC;QACpB,IAAI,WAAgB,CAAC;QAErB,MAAM,KAAK,CAAC,IAAI,CAAC;YACf,GAAG,EAAE,YAAY,CAAC,KAAK;YACvB,GAAG,EAAE,GAAG,CAAC,aAAa,EAAE;YACxB,KAAK;SACN,CAAC,CAAC;QAEH,MAAM,IAAI,GAAe,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACvD,UAAU,GAAG,KAAK,EAAE,GAAiB,EAAE,EAAE;;gBACvC,QAAQ,GAAG,CAAC,GAAG,EAAE;oBACf,KAAK,aAAa,CAAC,SAAS;wBAC1B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;wBACnB,MAAM;oBACR,KAAK,aAAa,CAAC,MAAM,CAAC;oBAC1B,KAAK,aAAa,CAAC,KAAK,CAAC,CAAC;wBACxB,MAAM,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC;wBACxB,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;wBAC9B,MAAM,CAAC,GAAG,CAAC,CAAC;wBACZ,MAAM;qBACP;oBACD,KAAK,aAAa,CAAC,QAAQ;wBACzB,MAAM,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;wBACpC,MAAM;oBACR,KAAK,aAAa,CAAC,GAAG;wBACpB,MAAM,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;wBACzB,MAAM;oBACR,KAAK,aAAa,CAAC,aAAa;wBAC9B,MAAM,GAAG,CAAC,aAAa,CAAC,MAAA,GAAG,CAAC,KAAK,0CAAE,SAAS,EAAE,MAAA,GAAG,CAAC,KAAK,0CAAE,KAAK,CAAC,CAAC;wBAChE,MAAM;oBACR,KAAK,aAAa,CAAC,MAAM;wBACvB,MAAM,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;wBAChC,MAAM;iBACT;YACH,CAAC,CAAC;YAEF,WAAW,GAAG,CAAC,QAAa,EAAE,MAAW,EAAE,EAAE;gBAC3C,MAAM,CACJ,IAAI,KAAK,CAAC,wBAAwB,GAAG,QAAQ,GAAG,WAAW,GAAG,MAAM,CAAC,CACtE,CAAC;YACJ,CAAC,CAAC;YAEF,KAAK,CAAC,EAAE,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;YAChC,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,IAAI;YACF,MAAM,IAAI,CAAC;YACX,OAAO,IAAI,CAAC;SACb;gBAAS;YACR,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;YACjC,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;YAE/B,IAAI,KAAK,CAAC,QAAQ,KAAK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,UAAU,EAAE,CAAC,EAAE;gBAClE,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;aACzB;iBAAM;gBACL,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;aAC1B;SACF;IACH,CAAC,CAAC;AACJ,CAAC,CAAC;AAEF,eAAe,OAAO,CAAC"}
+131
View File
@@ -0,0 +1,131 @@
/**
* Includes all the scripts needed by the queue and jobs.
*/
/// <reference types="node" />
import { JobJson, JobJsonRaw, MinimalJob, MoveToWaitingChildrenOpts, ParentOpts, RedisClient, KeepJobs } from '../interfaces';
import { JobState, JobType, FinishedStatus, FinishedPropValAttribute, MinimalQueue, RedisJobOptions } from '../types';
import { ChainableCommander } from 'ioredis';
export type JobData = [JobJsonRaw | number, string?];
export declare class Scripts {
protected queue: MinimalQueue;
protected version: string;
moveToFinishedKeys: (string | undefined)[];
constructor(queue: MinimalQueue);
execCommand(client: RedisClient | ChainableCommander, commandName: string, args: any[]): any;
isJobInList(listKey: string, jobId: string): Promise<boolean>;
private addDelayedJob;
private addPrioritizedJob;
private addParentJob;
addJob(client: RedisClient, job: JobJson, opts: RedisJobOptions, jobId: string, parentOpts?: ParentOpts): Promise<string>;
pause(pause: boolean): Promise<void>;
private removeRepeatableArgs;
removeRepeatable(repeatJobId: string, repeatJobKey: string): Promise<number>;
remove(jobId: string, removeChildren: boolean): Promise<number>;
extendLock(jobId: string, token: string, duration: number, client?: RedisClient | ChainableCommander): Promise<number>;
updateData<T = any, R = any, N extends string = string>(job: MinimalJob<T, R, N>, data: T): Promise<void>;
updateProgress<T = any, R = any, N extends string = string>(jobId: string, progress: number | object): Promise<void>;
protected moveToFinishedArgs<T = any, R = any, N extends string = string>(job: MinimalJob<T, R, N>, val: any, propVal: FinishedPropValAttribute, shouldRemove: undefined | boolean | number | KeepJobs, target: FinishedStatus, token: string, timestamp: number, fetchNext?: boolean): (string | number | boolean | Buffer)[];
protected getKeepJobs(shouldRemove: undefined | boolean | number | KeepJobs, workerKeepJobs: undefined | KeepJobs): KeepJobs;
moveToFinished(jobId: string, args: (string | number | boolean | Buffer)[]): Promise<any[]>;
finishedErrors(code: number, jobId: string, command: string, state?: string): Error;
private drainArgs;
drain(delayed: boolean): Promise<void>;
private getRangesArgs;
getRanges(types: JobType[], start?: number, end?: number, asc?: boolean): Promise<[string][]>;
private getCountsArgs;
getCounts(types: JobType[]): Promise<number[]>;
moveToCompletedArgs<T = any, R = any, N extends string = string>(job: MinimalJob<T, R, N>, returnvalue: R, removeOnComplete: boolean | number | KeepJobs, token: string, fetchNext?: boolean): (string | number | boolean | Buffer)[];
moveToFailedArgs<T = any, R = any, N extends string = string>(job: MinimalJob<T, R, N>, failedReason: string, removeOnFailed: boolean | number | KeepJobs, token: string, fetchNext?: boolean): (string | number | boolean | Buffer)[];
isFinished(jobId: string, returnValue?: boolean): Promise<number | [number, string]>;
getState(jobId: string): Promise<JobState | 'unknown'>;
changeDelay(jobId: string, delay: number): Promise<void>;
private changeDelayArgs;
changePriority(jobId: string, priority?: number, lifo?: boolean): Promise<void>;
private changePriorityArgs;
moveToDelayedArgs(jobId: string, timestamp: number, token: string, delay: number): (string | number)[];
saveStacktraceArgs(jobId: string, stacktrace: string, failedReason: string): string[];
moveToWaitingChildrenArgs(jobId: string, token: string, opts?: MoveToWaitingChildrenOpts): string[];
moveToDelayed(jobId: string, timestamp: number, delay: number, token?: string): Promise<void>;
/**
* Move parent job to waiting-children state.
*
* @returns true if job is successfully moved, false if there are pending dependencies.
* @throws JobNotExist
* This exception is thrown if jobId is missing.
* @throws JobLockNotExist
* This exception is thrown if job lock is missing.
* @throws JobNotInState
* This exception is thrown if job is not in active state.
*/
moveToWaitingChildren(jobId: string, token: string, opts?: MoveToWaitingChildrenOpts): Promise<boolean>;
/**
* Remove jobs in a specific state.
*
* @returns Id jobs from the deleted records.
*/
cleanJobsInSet(set: string, timestamp: number, limit?: number): Promise<string[]>;
retryJobArgs(jobId: string, lifo: boolean, token: string): (string | number)[];
protected moveJobsToWaitArgs(state: FinishedStatus | 'delayed', count: number, timestamp: number): (string | number)[];
retryJobs(state?: FinishedStatus, count?: number, timestamp?: number): Promise<number>;
promoteJobs(count?: number): Promise<number>;
/**
* Attempts to reprocess a job
*
* @param job -
* @param state - The expected job state. If the job is not found
* on the provided state, then it's not reprocessed. Supported states: 'failed', 'completed'
*
* @returns Returns a promise that evaluates to a return code:
* 1 means the operation was a success
* 0 means the job does not exist
* -1 means the job is currently locked and can't be retried.
* -2 means the job was not found in the expected set
*/
reprocessJob<T = any, R = any, N extends string = string>(job: MinimalJob<T, R, N>, state: 'failed' | 'completed'): Promise<void>;
moveToActive(client: RedisClient, token: string, jobId?: string): Promise<any[]>;
promote(jobId: string): Promise<void>;
/**
* Looks for unlocked jobs in the active queue.
*
* The job was being worked on, but the worker process died and it failed to renew the lock.
* We call these jobs 'stalled'. This is the most common case. We resolve these by moving them
* back to wait to be re-processed. To prevent jobs from cycling endlessly between active and wait,
* (e.g. if the job handler keeps crashing),
* we limit the number stalled job recoveries to settings.maxStalledCount.
*/
moveStalledJobsToWait(): Promise<[string[], string[]]>;
/**
* Moves a job back from Active to Wait.
* This script is used when a job has been manually rate limited and needs
* to be moved back to wait from active status.
*
* @param client - Redis client
* @param jobId - Job id
* @returns
*/
moveJobFromActiveToWait(jobId: string, token: string): Promise<any>;
obliterate(opts: {
force: boolean;
count: number;
}): Promise<number>;
/**
* Paginate a set or hash keys.
* @param opts
*
*/
paginate(key: string, opts: {
start: number;
end: number;
fetchJobs?: boolean;
}): Promise<{
cursor: string;
items: {
id: string;
v?: any;
err?: string;
}[];
total: number;
jobs?: JobJsonRaw[];
}>;
}
export declare function raw2NextJobData(raw: any[]): any[];
+778
View File
@@ -0,0 +1,778 @@
/**
* Includes all the scripts needed by the queue and jobs.
*/
/*eslint-env node */
'use strict';
import { Packr } from 'msgpackr';
const packer = new Packr({
useRecords: false,
encodeUndefinedAsNil: true,
});
const pack = packer.pack;
import { ErrorCode } from '../enums';
import { array2obj, getParentKey, isRedisVersionLowerThan } from '../utils';
import { version } from '../version';
export class Scripts {
constructor(queue) {
this.queue = queue;
const queueKeys = this.queue.keys;
this.version = version;
this.moveToFinishedKeys = [
queueKeys.wait,
queueKeys.active,
queueKeys.prioritized,
queueKeys.events,
queueKeys.stalled,
queueKeys.limiter,
queueKeys.delayed,
queueKeys.paused,
queueKeys.meta,
queueKeys.pc,
undefined,
undefined,
undefined,
];
}
execCommand(client, commandName, args) {
const commandNameWithVersion = `${commandName}:${this.version}`;
return client[commandNameWithVersion](args);
}
async isJobInList(listKey, jobId) {
const client = await this.queue.client;
let result;
if (isRedisVersionLowerThan(this.queue.redisVersion, '6.0.6')) {
result = await this.execCommand(client, 'isJobInList', [listKey, jobId]);
}
else {
result = await client.lpos(listKey, jobId);
}
return Number.isInteger(result);
}
async addDelayedJob(client, job, encodedOpts, args) {
const queueKeys = this.queue.keys;
const keys = [
queueKeys.wait,
queueKeys.paused,
queueKeys.meta,
queueKeys.id,
queueKeys.delayed,
queueKeys.completed,
queueKeys.events,
];
keys.push(pack(args), job.data, encodedOpts);
return this.execCommand(client, 'addDelayedJob', keys);
}
async addPrioritizedJob(client, job, encodedOpts, args) {
const queueKeys = this.queue.keys;
const keys = [
queueKeys.wait,
queueKeys.paused,
queueKeys.meta,
queueKeys.id,
queueKeys.prioritized,
queueKeys.completed,
queueKeys.events,
queueKeys.pc,
];
keys.push(pack(args), job.data, encodedOpts);
return this.execCommand(client, 'addPrioritizedJob', keys);
}
async addParentJob(client, job, encodedOpts, args) {
const queueKeys = this.queue.keys;
const keys = [
queueKeys.meta,
queueKeys.id,
queueKeys.completed,
queueKeys.events,
];
keys.push(pack(args), job.data, encodedOpts);
return this.execCommand(client, 'addParentJob', keys);
}
async addJob(client, job, opts, jobId, parentOpts = {}) {
const queueKeys = this.queue.keys;
const parent = job.parent
? Object.assign(Object.assign({}, job.parent), { fpof: opts.fpof, rdof: opts.rdof }) : null;
const args = [
queueKeys[''],
typeof jobId !== 'undefined' ? jobId : '',
job.name,
job.timestamp,
job.parentKey || null,
parentOpts.waitChildrenKey || null,
parentOpts.parentDependenciesKey || null,
parent,
job.repeatJobKey,
];
let encodedOpts;
if (opts.repeat) {
const repeat = Object.assign({}, opts.repeat);
if (repeat.startDate) {
repeat.startDate = +new Date(repeat.startDate);
}
if (repeat.endDate) {
repeat.endDate = +new Date(repeat.endDate);
}
encodedOpts = pack(Object.assign(Object.assign({}, opts), { repeat }));
}
else {
encodedOpts = pack(opts);
}
let result;
if (parentOpts.waitChildrenKey) {
result = await this.addParentJob(client, job, encodedOpts, args);
}
else if (opts.delay) {
result = await this.addDelayedJob(client, job, encodedOpts, args);
}
else if (opts.priority) {
result = await this.addPrioritizedJob(client, job, encodedOpts, args);
}
else {
const keys = [
queueKeys.wait,
queueKeys.paused,
queueKeys.meta,
queueKeys.id,
queueKeys.completed,
queueKeys.events,
];
keys.push(pack(args), job.data, encodedOpts);
result = await this.execCommand(client, 'addStandardJob', keys);
}
if (result < 0) {
throw this.finishedErrors(result, parentOpts.parentKey, 'addJob');
}
return result;
}
async pause(pause) {
const client = await this.queue.client;
let src = 'wait', dst = 'paused';
if (!pause) {
src = 'paused';
dst = 'wait';
}
const keys = [src, dst, 'meta', 'prioritized'].map((name) => this.queue.toKey(name));
keys.push(this.queue.keys.events);
return this.execCommand(client, 'pause', keys.concat([pause ? 'paused' : 'resumed']));
}
removeRepeatableArgs(repeatJobId, repeatJobKey) {
const queueKeys = this.queue.keys;
const keys = [queueKeys.repeat, queueKeys.delayed];
const args = [repeatJobId, repeatJobKey, queueKeys['']];
return keys.concat(args);
}
async removeRepeatable(repeatJobId, repeatJobKey) {
const client = await this.queue.client;
const args = this.removeRepeatableArgs(repeatJobId, repeatJobKey);
return this.execCommand(client, 'removeRepeatable', args);
}
async remove(jobId, removeChildren) {
const client = await this.queue.client;
const keys = [''].map(name => this.queue.toKey(name));
return this.execCommand(client, 'removeJob', keys.concat([jobId, removeChildren ? 1 : 0]));
}
async extendLock(jobId, token, duration, client) {
client = client || (await this.queue.client);
const args = [
this.queue.toKey(jobId) + ':lock',
this.queue.keys.stalled,
token,
duration,
jobId,
];
return this.execCommand(client, 'extendLock', args);
}
async updateData(job, data) {
const client = await this.queue.client;
const keys = [this.queue.toKey(job.id)];
const dataJson = JSON.stringify(data);
const result = await this.execCommand(client, 'updateData', keys.concat([dataJson]));
if (result < 0) {
throw this.finishedErrors(result, job.id, 'updateData');
}
}
async updateProgress(jobId, progress) {
const client = await this.queue.client;
const keys = [
this.queue.toKey(jobId),
this.queue.keys.events,
this.queue.keys.meta,
];
const progressJson = JSON.stringify(progress);
const result = await this.execCommand(client, 'updateProgress', keys.concat([jobId, progressJson]));
if (result < 0) {
throw this.finishedErrors(result, jobId, 'updateProgress');
}
}
moveToFinishedArgs(job, val, propVal, shouldRemove, target, token, timestamp, fetchNext = true) {
var _a, _b, _c, _d, _e;
const queueKeys = this.queue.keys;
const opts = this.queue.opts;
const workerKeepJobs = target === 'completed' ? opts.removeOnComplete : opts.removeOnFail;
const metricsKey = this.queue.toKey(`metrics:${target}`);
const keys = this.moveToFinishedKeys;
keys[10] = queueKeys[target];
keys[11] = this.queue.toKey((_a = job.id) !== null && _a !== void 0 ? _a : '');
keys[12] = metricsKey;
const keepJobs = this.getKeepJobs(shouldRemove, workerKeepJobs);
const args = [
job.id,
timestamp,
propVal,
typeof val === 'undefined' ? 'null' : val,
target,
JSON.stringify({ jobId: job.id, val: val }),
!fetchNext || this.queue.closing ? 0 : 1,
queueKeys[''],
pack({
token,
keepJobs,
limiter: opts.limiter,
lockDuration: opts.lockDuration,
attempts: job.opts.attempts,
attemptsMade: job.attemptsMade,
maxMetricsSize: ((_b = opts.metrics) === null || _b === void 0 ? void 0 : _b.maxDataPoints)
? (_c = opts.metrics) === null || _c === void 0 ? void 0 : _c.maxDataPoints
: '',
fpof: !!((_d = job.opts) === null || _d === void 0 ? void 0 : _d.failParentOnFailure),
rdof: !!((_e = job.opts) === null || _e === void 0 ? void 0 : _e.removeDependencyOnFailure),
}),
];
return keys.concat(args);
}
getKeepJobs(shouldRemove, workerKeepJobs) {
if (typeof shouldRemove === 'undefined') {
return workerKeepJobs || { count: shouldRemove ? 0 : -1 };
}
return typeof shouldRemove === 'object'
? shouldRemove
: typeof shouldRemove === 'number'
? { count: shouldRemove }
: { count: shouldRemove ? 0 : -1 };
}
async moveToFinished(jobId, args) {
const client = await this.queue.client;
const result = await this.execCommand(client, 'moveToFinished', args);
if (result < 0) {
throw this.finishedErrors(result, jobId, 'moveToFinished', 'active');
}
else {
if (typeof result !== 'undefined') {
return raw2NextJobData(result);
}
}
}
finishedErrors(code, jobId, command, state) {
switch (code) {
case ErrorCode.JobNotExist:
return new Error(`Missing key for job ${jobId}. ${command}`);
case ErrorCode.JobLockNotExist:
return new Error(`Missing lock for job ${jobId}. ${command}`);
case ErrorCode.JobNotInState:
return new Error(`Job ${jobId} is not in the ${state} state. ${command}`);
case ErrorCode.JobPendingDependencies:
return new Error(`Job ${jobId} has pending dependencies. ${command}`);
case ErrorCode.ParentJobNotExist:
return new Error(`Missing key for parent job ${jobId}. ${command}`);
case ErrorCode.JobLockMismatch:
return new Error(`Lock mismatch for job ${jobId}. Cmd ${command} from ${state}`);
default:
return new Error(`Unknown code ${code} error for ${jobId}. ${command}`);
}
}
drainArgs(delayed) {
const queueKeys = this.queue.keys;
const keys = [
queueKeys.wait,
queueKeys.paused,
delayed ? queueKeys.delayed : '',
queueKeys.prioritized,
];
const args = [queueKeys['']];
return keys.concat(args);
}
async drain(delayed) {
const client = await this.queue.client;
const args = this.drainArgs(delayed);
return this.execCommand(client, 'drain', args);
}
getRangesArgs(types, start, end, asc) {
const queueKeys = this.queue.keys;
const transformedTypes = types.map(type => {
return type === 'waiting' ? 'wait' : type;
});
const keys = [queueKeys['']];
const args = [start, end, asc ? '1' : '0', ...transformedTypes];
return keys.concat(args);
}
async getRanges(types, start = 0, end = 1, asc = false) {
const client = await this.queue.client;
const args = this.getRangesArgs(types, start, end, asc);
return this.execCommand(client, 'getRanges', args);
}
getCountsArgs(types) {
const queueKeys = this.queue.keys;
const transformedTypes = types.map(type => {
return type === 'waiting' ? 'wait' : type;
});
const keys = [queueKeys['']];
const args = [...transformedTypes];
return keys.concat(args);
}
async getCounts(types) {
const client = await this.queue.client;
const args = this.getCountsArgs(types);
return this.execCommand(client, 'getCounts', args);
}
moveToCompletedArgs(job, returnvalue, removeOnComplete, token, fetchNext = false) {
const timestamp = Date.now();
return this.moveToFinishedArgs(job, returnvalue, 'returnvalue', removeOnComplete, 'completed', token, timestamp, fetchNext);
}
moveToFailedArgs(job, failedReason, removeOnFailed, token, fetchNext = false) {
const timestamp = Date.now();
return this.moveToFinishedArgs(job, failedReason, 'failedReason', removeOnFailed, 'failed', token, timestamp, fetchNext);
}
async isFinished(jobId, returnValue = false) {
const client = await this.queue.client;
const keys = ['completed', 'failed', jobId].map((key) => {
return this.queue.toKey(key);
});
return this.execCommand(client, 'isFinished', keys.concat([jobId, returnValue ? '1' : '']));
}
async getState(jobId) {
const client = await this.queue.client;
const keys = [
'completed',
'failed',
'delayed',
'active',
'wait',
'paused',
'waiting-children',
'prioritized',
].map((key) => {
return this.queue.toKey(key);
});
if (isRedisVersionLowerThan(this.queue.redisVersion, '6.0.6')) {
return this.execCommand(client, 'getState', keys.concat([jobId]));
}
return this.execCommand(client, 'getStateV2', keys.concat([jobId]));
}
async changeDelay(jobId, delay) {
const client = await this.queue.client;
const args = this.changeDelayArgs(jobId, delay);
const result = await this.execCommand(client, 'changeDelay', args);
if (result < 0) {
throw this.finishedErrors(result, jobId, 'changeDelay', 'delayed');
}
}
changeDelayArgs(jobId, delay) {
//
// Bake in the job id first 12 bits into the timestamp
// to guarantee correct execution order of delayed jobs
// (up to 4096 jobs per given timestamp or 4096 jobs apart per timestamp)
//
// WARNING: Jobs that are so far apart that they wrap around will cause FIFO to fail
//
let timestamp = Date.now() + delay;
if (timestamp > 0) {
timestamp = timestamp * 0x1000 + (+jobId & 0xfff);
}
const keys = ['delayed', jobId].map(name => {
return this.queue.toKey(name);
});
keys.push.apply(keys, [this.queue.keys.events]);
return keys.concat([delay, JSON.stringify(timestamp), jobId]);
}
async changePriority(jobId, priority = 0, lifo = false) {
const client = await this.queue.client;
const args = this.changePriorityArgs(jobId, priority, lifo);
const result = await this.execCommand(client, 'changePriority', args);
if (result < 0) {
throw this.finishedErrors(result, jobId, 'changePriority');
}
}
changePriorityArgs(jobId, priority = 0, lifo = false) {
const keys = [
this.queue.keys.wait,
this.queue.keys.paused,
this.queue.keys.meta,
this.queue.keys.prioritized,
this.queue.keys.pc,
];
return keys.concat([
priority,
this.queue.toKey(jobId),
jobId,
lifo ? 1 : 0,
]);
}
// Note: We have an issue here with jobs using custom job ids
moveToDelayedArgs(jobId, timestamp, token, delay) {
//
// Bake in the job id first 12 bits into the timestamp
// to guarantee correct execution order of delayed jobs
// (up to 4096 jobs per given timestamp or 4096 jobs apart per timestamp)
//
// WARNING: Jobs that are so far apart that they wrap around will cause FIFO to fail
//
timestamp = Math.max(0, timestamp !== null && timestamp !== void 0 ? timestamp : 0);
if (timestamp > 0) {
timestamp = timestamp * 0x1000 + (+jobId & 0xfff);
}
const keys = [
'wait',
'active',
'prioritized',
'delayed',
jobId,
].map(name => {
return this.queue.toKey(name);
});
keys.push.apply(keys, [
this.queue.keys.events,
this.queue.keys.paused,
this.queue.keys.meta,
]);
return keys.concat([
this.queue.keys[''],
Date.now(),
JSON.stringify(timestamp),
jobId,
token,
delay,
]);
}
saveStacktraceArgs(jobId, stacktrace, failedReason) {
const keys = [this.queue.toKey(jobId)];
return keys.concat([stacktrace, failedReason]);
}
moveToWaitingChildrenArgs(jobId, token, opts) {
const timestamp = Date.now();
const childKey = getParentKey(opts.child);
const keys = [`${jobId}:lock`, 'active', 'waiting-children', jobId].map(name => {
return this.queue.toKey(name);
});
return keys.concat([
token,
childKey !== null && childKey !== void 0 ? childKey : '',
JSON.stringify(timestamp),
jobId,
]);
}
async moveToDelayed(jobId, timestamp, delay, token = '0') {
const client = await this.queue.client;
const args = this.moveToDelayedArgs(jobId, timestamp, token, delay);
const result = await this.execCommand(client, 'moveToDelayed', args);
if (result < 0) {
throw this.finishedErrors(result, jobId, 'moveToDelayed', 'active');
}
}
/**
* Move parent job to waiting-children state.
*
* @returns true if job is successfully moved, false if there are pending dependencies.
* @throws JobNotExist
* This exception is thrown if jobId is missing.
* @throws JobLockNotExist
* This exception is thrown if job lock is missing.
* @throws JobNotInState
* This exception is thrown if job is not in active state.
*/
async moveToWaitingChildren(jobId, token, opts = {}) {
const client = await this.queue.client;
const args = this.moveToWaitingChildrenArgs(jobId, token, opts);
const result = await this.execCommand(client, 'moveToWaitingChildren', args);
switch (result) {
case 0:
return true;
case 1:
return false;
default:
throw this.finishedErrors(result, jobId, 'moveToWaitingChildren', 'active');
}
}
/**
* Remove jobs in a specific state.
*
* @returns Id jobs from the deleted records.
*/
async cleanJobsInSet(set, timestamp, limit = 0) {
const client = await this.queue.client;
return this.execCommand(client, 'cleanJobsInSet', [
this.queue.toKey(set),
this.queue.toKey('events'),
this.queue.toKey(''),
timestamp,
limit,
set,
]);
}
retryJobArgs(jobId, lifo, token) {
const keys = [
'active',
'wait',
'paused',
jobId,
'meta',
].map(name => {
return this.queue.toKey(name);
});
keys.push(this.queue.keys.events, this.queue.keys.delayed, this.queue.keys.prioritized, this.queue.keys.pc);
const pushCmd = (lifo ? 'R' : 'L') + 'PUSH';
return keys.concat([
this.queue.toKey(''),
Date.now(),
pushCmd,
jobId,
token,
]);
}
moveJobsToWaitArgs(state, count, timestamp) {
const keys = [
this.queue.toKey(''),
this.queue.keys.events,
this.queue.toKey(state),
this.queue.toKey('wait'),
this.queue.toKey('paused'),
this.queue.toKey('meta'),
];
const args = [count, timestamp, state];
return keys.concat(args);
}
async retryJobs(state = 'failed', count = 1000, timestamp = new Date().getTime()) {
const client = await this.queue.client;
const args = this.moveJobsToWaitArgs(state, count, timestamp);
return this.execCommand(client, 'moveJobsToWait', args);
}
async promoteJobs(count = 1000) {
const client = await this.queue.client;
const args = this.moveJobsToWaitArgs('delayed', count, Number.MAX_VALUE);
return this.execCommand(client, 'moveJobsToWait', args);
}
/**
* Attempts to reprocess a job
*
* @param job -
* @param state - The expected job state. If the job is not found
* on the provided state, then it's not reprocessed. Supported states: 'failed', 'completed'
*
* @returns Returns a promise that evaluates to a return code:
* 1 means the operation was a success
* 0 means the job does not exist
* -1 means the job is currently locked and can't be retried.
* -2 means the job was not found in the expected set
*/
async reprocessJob(job, state) {
const client = await this.queue.client;
const keys = [
this.queue.toKey(job.id),
this.queue.keys.events,
this.queue.toKey(state),
this.queue.keys.wait,
this.queue.keys.meta,
this.queue.keys.paused,
];
const args = [
job.id,
(job.opts.lifo ? 'R' : 'L') + 'PUSH',
state === 'failed' ? 'failedReason' : 'returnvalue',
state,
];
const result = await this.execCommand(client, 'reprocessJob', keys.concat(args));
switch (result) {
case 1:
return;
default:
throw this.finishedErrors(result, job.id, 'reprocessJob', state);
}
}
async moveToActive(client, token, jobId) {
const opts = this.queue.opts;
const queueKeys = this.queue.keys;
const keys = [
queueKeys.wait,
queueKeys.active,
queueKeys.prioritized,
queueKeys.events,
queueKeys.stalled,
queueKeys.limiter,
queueKeys.delayed,
queueKeys.paused,
queueKeys.meta,
queueKeys.pc,
];
const args = [
queueKeys[''],
Date.now(),
jobId || '',
pack({
token,
lockDuration: opts.lockDuration,
limiter: opts.limiter,
}),
];
const result = await this.execCommand(client, 'moveToActive', keys.concat(args));
return raw2NextJobData(result);
}
async promote(jobId) {
const client = await this.queue.client;
const keys = [
this.queue.keys.delayed,
this.queue.keys.wait,
this.queue.keys.paused,
this.queue.keys.meta,
this.queue.keys.prioritized,
this.queue.keys.pc,
this.queue.keys.events,
];
const args = [this.queue.toKey(''), jobId];
const code = await this.execCommand(client, 'promote', keys.concat(args));
if (code < 0) {
throw this.finishedErrors(code, jobId, 'promote', 'delayed');
}
}
/**
* Looks for unlocked jobs in the active queue.
*
* The job was being worked on, but the worker process died and it failed to renew the lock.
* We call these jobs 'stalled'. This is the most common case. We resolve these by moving them
* back to wait to be re-processed. To prevent jobs from cycling endlessly between active and wait,
* (e.g. if the job handler keeps crashing),
* we limit the number stalled job recoveries to settings.maxStalledCount.
*/
async moveStalledJobsToWait() {
const client = await this.queue.client;
const opts = this.queue.opts;
const keys = [
this.queue.keys.stalled,
this.queue.keys.wait,
this.queue.keys.active,
this.queue.keys.failed,
this.queue.keys['stalled-check'],
this.queue.keys.meta,
this.queue.keys.paused,
this.queue.keys.events,
];
const args = [
opts.maxStalledCount,
this.queue.toKey(''),
Date.now(),
opts.stalledInterval,
];
return this.execCommand(client, 'moveStalledJobsToWait', keys.concat(args));
}
/**
* Moves a job back from Active to Wait.
* This script is used when a job has been manually rate limited and needs
* to be moved back to wait from active status.
*
* @param client - Redis client
* @param jobId - Job id
* @returns
*/
async moveJobFromActiveToWait(jobId, token) {
const client = await this.queue.client;
const lockKey = `${this.queue.toKey(jobId)}:lock`;
const keys = [
this.queue.keys.active,
this.queue.keys.wait,
this.queue.keys.stalled,
lockKey,
this.queue.keys.paused,
this.queue.keys.meta,
this.queue.keys.limiter,
this.queue.keys.prioritized,
this.queue.keys.events,
];
const args = [jobId, token, this.queue.toKey(jobId)];
const pttl = await this.execCommand(client, 'moveJobFromActiveToWait', keys.concat(args));
return pttl < 0 ? 0 : pttl;
}
async obliterate(opts) {
const client = await this.queue.client;
const keys = [
this.queue.keys.meta,
this.queue.toKey(''),
];
const args = [opts.count, opts.force ? 'force' : null];
const result = await this.execCommand(client, 'obliterate', keys.concat(args));
if (result < 0) {
switch (result) {
case -1:
throw new Error('Cannot obliterate non-paused queue');
case -2:
throw new Error('Cannot obliterate queue with active jobs');
}
}
return result;
}
/**
* Paginate a set or hash keys.
* @param opts
*
*/
async paginate(key, opts) {
const client = await this.queue.client;
const keys = [key];
const maxIterations = 5;
const pageSize = opts.end >= 0 ? opts.end - opts.start + 1 : Infinity;
let cursor = '0', offset = 0, items, total, rawJobs, page = [], jobs = [];
do {
const args = [
opts.start + page.length,
opts.end,
cursor,
offset,
maxIterations,
];
if (opts.fetchJobs) {
args.push(1);
}
[cursor, offset, items, total, rawJobs] = await this.execCommand(client, 'paginate', keys.concat(args));
page = page.concat(items);
if (rawJobs && rawJobs.length) {
jobs = jobs.concat(rawJobs.map(array2obj));
}
// Important to keep this coercive inequality (!=) instead of strict inequality (!==)
} while (cursor != '0' && page.length < pageSize);
// If we get an array of arrays, it means we are paginating a hash
if (page.length && Array.isArray(page[0])) {
const result = [];
for (let index = 0; index < page.length; index++) {
const [id, value] = page[index];
try {
result.push({ id, v: JSON.parse(value) });
}
catch (err) {
result.push({ id, err: err.message });
}
}
return {
cursor,
items: result,
total,
jobs,
};
}
else {
return {
cursor,
items: page.map(item => ({ id: item })),
total,
jobs,
};
}
}
}
export function raw2NextJobData(raw) {
if (raw) {
const result = [null, raw[1], raw[2], raw[3]];
if (raw[0]) {
result[0] = array2obj(raw[0]);
}
return result;
}
return [];
}
//# sourceMappingURL=scripts.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"scripts.js","sourceRoot":"","sources":["../../../src/classes/scripts.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,oBAAoB;AACpB,YAAY,CAAC;AACb,OAAO,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC;AAEjC,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC;IACvB,UAAU,EAAE,KAAK;IACjB,oBAAoB,EAAE,IAAI;CAC3B,CAAC,CAAC;AAEH,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;AAoBzB,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,uBAAuB,EAAE,MAAM,UAAU,CAAC;AAG5E,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAGrC,MAAM,OAAO,OAAO;IAKlB,YAAsB,KAAmB;QAAnB,UAAK,GAAL,KAAK,CAAc;QACvC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;QAElC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QAEvB,IAAI,CAAC,kBAAkB,GAAG;YACxB,SAAS,CAAC,IAAI;YACd,SAAS,CAAC,MAAM;YAChB,SAAS,CAAC,WAAW;YACrB,SAAS,CAAC,MAAM;YAChB,SAAS,CAAC,OAAO;YACjB,SAAS,CAAC,OAAO;YACjB,SAAS,CAAC,OAAO;YACjB,SAAS,CAAC,MAAM;YAChB,SAAS,CAAC,IAAI;YACd,SAAS,CAAC,EAAE;YACZ,SAAS;YACT,SAAS;YACT,SAAS;SACV,CAAC;IACJ,CAAC;IAEM,WAAW,CAChB,MAAwC,EACxC,WAAmB,EACnB,IAAW;QAEX,MAAM,sBAAsB,GAAG,GAAG,WAAW,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QAChE,OAAa,MAAO,CAAC,sBAAsB,CAAC,CAAC,IAAI,CAAC,CAAC;IACrD,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,OAAe,EAAE,KAAa;QAC9C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QACvC,IAAI,MAAM,CAAC;QACX,IAAI,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE,OAAO,CAAC,EAAE;YAC7D,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,aAAa,EAAE,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;SAC1E;aAAM;YACL,MAAM,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;SAC5C;QACD,OAAO,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAClC,CAAC;IAEO,KAAK,CAAC,aAAa,CACzB,MAAmB,EACnB,GAAY,EACZ,WAAgB,EAChB,IAA+C;QAE/C,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;QAClC,MAAM,IAAI,GAAwB;YAChC,SAAS,CAAC,IAAI;YACd,SAAS,CAAC,MAAM;YAChB,SAAS,CAAC,IAAI;YACd,SAAS,CAAC,EAAE;YACZ,SAAS,CAAC,OAAO;YACjB,SAAS,CAAC,SAAS;YACnB,SAAS,CAAC,MAAM;SACjB,CAAC;QAEF,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QAC7C,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,eAAe,EAAE,IAAI,CAAC,CAAC;IACzD,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAC7B,MAAmB,EACnB,GAAY,EACZ,WAAgB,EAChB,IAA+C;QAE/C,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;QAClC,MAAM,IAAI,GAAwB;YAChC,SAAS,CAAC,IAAI;YACd,SAAS,CAAC,MAAM;YAChB,SAAS,CAAC,IAAI;YACd,SAAS,CAAC,EAAE;YACZ,SAAS,CAAC,WAAW;YACrB,SAAS,CAAC,SAAS;YACnB,SAAS,CAAC,MAAM;YAChB,SAAS,CAAC,EAAE;SACb,CAAC;QAEF,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QAE7C,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,mBAAmB,EAAE,IAAI,CAAC,CAAC;IAC7D,CAAC;IAEO,KAAK,CAAC,YAAY,CACxB,MAAmB,EACnB,GAAY,EACZ,WAAgB,EAChB,IAA+C;QAE/C,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;QAClC,MAAM,IAAI,GAAwB;YAChC,SAAS,CAAC,IAAI;YACd,SAAS,CAAC,EAAE;YACZ,SAAS,CAAC,SAAS;YACnB,SAAS,CAAC,MAAM;SACjB,CAAC;QAEF,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QAE7C,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,cAAc,EAAE,IAAI,CAAC,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,MAAM,CACV,MAAmB,EACnB,GAAY,EACZ,IAAqB,EACrB,KAAa,EACb,aAAyB,EAAE;QAE3B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;QAElC,MAAM,MAAM,GAAwB,GAAG,CAAC,MAAM;YAC5C,CAAC,iCAAM,GAAG,CAAC,MAAM,KAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,IACnD,CAAC,CAAC,IAAI,CAAC;QAET,MAAM,IAAI,GAAG;YACX,SAAS,CAAC,EAAE,CAAC;YACb,OAAO,KAAK,KAAK,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;YACzC,GAAG,CAAC,IAAI;YACR,GAAG,CAAC,SAAS;YACb,GAAG,CAAC,SAAS,IAAI,IAAI;YACrB,UAAU,CAAC,eAAe,IAAI,IAAI;YAClC,UAAU,CAAC,qBAAqB,IAAI,IAAI;YACxC,MAAM;YACN,GAAG,CAAC,YAAY;SACjB,CAAC;QAEF,IAAI,WAAW,CAAC;QAChB,IAAI,IAAI,CAAC,MAAM,EAAE;YACf,MAAM,MAAM,qBACP,IAAI,CAAC,MAAM,CACf,CAAC;YAEF,IAAI,MAAM,CAAC,SAAS,EAAE;gBACpB,MAAM,CAAC,SAAS,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;aAChD;YACD,IAAI,MAAM,CAAC,OAAO,EAAE;gBAClB,MAAM,CAAC,OAAO,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;aAC5C;YAED,WAAW,GAAG,IAAI,iCACb,IAAI,KACP,MAAM,IACN,CAAC;SACJ;aAAM;YACL,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;SAC1B;QAED,IAAI,MAAM,CAAC;QAEX,IAAI,UAAU,CAAC,eAAe,EAAE;YAC9B,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,GAAG,EAAE,WAAW,EAAE,IAAI,CAAC,CAAC;SAClE;aAAM,IAAI,IAAI,CAAC,KAAK,EAAE;YACrB,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,EAAE,WAAW,EAAE,IAAI,CAAC,CAAC;SACnE;aAAM,IAAI,IAAI,CAAC,QAAQ,EAAE;YACxB,MAAM,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,GAAG,EAAE,WAAW,EAAE,IAAI,CAAC,CAAC;SACvE;aAAM;YACL,MAAM,IAAI,GAAwB;gBAChC,SAAS,CAAC,IAAI;gBACd,SAAS,CAAC,MAAM;gBAChB,SAAS,CAAC,IAAI;gBACd,SAAS,CAAC,EAAE;gBACZ,SAAS,CAAC,SAAS;gBACnB,SAAS,CAAC,MAAM;aACjB,CAAC;YACF,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;YAC7C,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,gBAAgB,EAAE,IAAI,CAAC,CAAC;SACjE;QAED,IAAI,MAAM,GAAG,CAAC,EAAE;YACd,MAAM,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,UAAU,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;SACnE;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,KAAc;QACxB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAEvC,IAAI,GAAG,GAAG,MAAM,EACd,GAAG,GAAG,QAAQ,CAAC;QACjB,IAAI,CAAC,KAAK,EAAE;YACV,GAAG,GAAG,QAAQ,CAAC;YACf,GAAG,GAAG,MAAM,CAAC;SACd;QAED,MAAM,IAAI,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,aAAa,CAAC,CAAC,GAAG,CAAC,CAAC,IALine truncated
+221
View File
@@ -0,0 +1,221 @@
/// <reference types="node" />
import { URL } from 'url';
import { AbortController } from 'node-abort-controller';
import { GetNextJobOptions, IoredisListener, JobJsonRaw, Processor, RedisClient, WorkerOptions } from '../interfaces';
import { QueueBase } from './queue-base';
import { Repeat } from './repeat';
import { Job } from './job';
import { RedisConnection } from './redis-connection';
export interface WorkerListener<DataType = any, ResultType = any, NameType extends string = string> extends IoredisListener {
/**
* Listen to 'active' event.
*
* This event is triggered when a job enters the 'active' state.
*/
active: (job: Job<DataType, ResultType, NameType>, prev: string) => void;
/**
* Listen to 'closing' event.
*
* This event is triggered when the worker is closed.
*/
closed: () => void;
/**
* Listen to 'closing' event.
*
* This event is triggered when the worker is closing.
*/
closing: (msg: string) => void;
/**
* Listen to 'completed' event.
*
* This event is triggered when a job has successfully completed.
*/
completed: (job: Job<DataType, ResultType, NameType>, result: ResultType, prev: string) => void;
/**
* Listen to 'drained' event.
*
* This event is triggered when the queue has drained the waiting list.
* Note that there could still be delayed jobs waiting their timers to expire
* and this event will still be triggered as long as the waiting list has emptied.
*/
drained: () => void;
/**
* Listen to 'error' event.
*
* This event is triggered when an error is throw.
*/
error: (failedReason: Error) => void;
/**
* Listen to 'failed' event.
*
* This event is triggered when a job has thrown an exception.
* Note: job parameter could be received as undefined when an stalled job
* reaches the stalled limit and it is deleted by the removeOnFail option.
*/
failed: (job: Job<DataType, ResultType, NameType> | undefined, error: Error, prev: string) => void;
/**
* Listen to 'paused' event.
*
* This event is triggered when the queue is paused.
*/
paused: () => void;
/**
* Listen to 'progress' event.
*
* This event is triggered when a job updates it progress, i.e. the
* Job##updateProgress() method is called. This is useful to notify
* progress or any other data from within a processor to the rest of the
* world.
*/
progress: (job: Job<DataType, ResultType, NameType>, progress: number | object) => void;
/**
* Listen to 'ready' event.
*
* This event is triggered when blockingConnection is ready.
*/
ready: () => void;
/**
* Listen to 'resumed' event.
*
* This event is triggered when the queue is resumed.
*/
resumed: () => void;
/**
* Listen to 'stalled' event.
*
* This event is triggered when a job has stalled and
* has been moved back to the wait list.
*/
stalled: (jobId: string, prev: string) => void;
}
/**
*
* This class represents a worker that is able to process jobs from the queue.
* As soon as the class is instantiated and a connection to Redis is established
* it will start processing jobs.
*
*/
export declare class Worker<DataType = any, ResultType = any, NameType extends string = string> extends QueueBase {
readonly opts: WorkerOptions;
readonly id: string;
private abortDelayController;
private asyncFifoQueue;
private blockingConnection;
private blockUntil;
private childPool;
private drained;
private extendLocksTimer;
private limitUntil;
private resumeWorker;
private stalledCheckTimer;
private waiting;
private _repeat;
protected paused: Promise<void>;
protected processFn: Processor<DataType, ResultType, NameType>;
protected running: boolean;
static RateLimitError(): Error;
constructor(name: string, processor?: string | URL | null | Processor<DataType, ResultType, NameType>, opts?: WorkerOptions, Connection?: typeof RedisConnection);
emit<U extends keyof WorkerListener<DataType, ResultType, NameType>>(event: U, ...args: Parameters<WorkerListener<DataType, ResultType, NameType>[U]>): boolean;
off<U extends keyof WorkerListener<DataType, ResultType, NameType>>(eventName: U, listener: WorkerListener<DataType, ResultType, NameType>[U]): this;
on<U extends keyof WorkerListener<DataType, ResultType, NameType>>(event: U, listener: WorkerListener<DataType, ResultType, NameType>[U]): this;
once<U extends keyof WorkerListener<DataType, ResultType, NameType>>(event: U, listener: WorkerListener<DataType, ResultType, NameType>[U]): this;
protected callProcessJob(job: Job<DataType, ResultType, NameType>, token: string): Promise<ResultType>;
protected createJob(data: JobJsonRaw, jobId: string): Job<DataType, ResultType, NameType>;
/**
*
* Waits until the worker is ready to start processing jobs.
* In general only useful when writing tests.
*
*/
waitUntilReady(): Promise<RedisClient>;
set concurrency(concurrency: number);
get repeat(): Promise<Repeat>;
run(): Promise<void>;
/**
* Returns a promise that resolves to the next job in queue.
* @param token - worker token to be assigned to retrieved job
* @returns a Job or undefined if no job was available in the queue.
*/
getNextJob(token: string, { block }?: GetNextJobOptions): Promise<Job<DataType, ResultType, NameType>>;
private _getNextJob;
/**
* Overrides the rate limit to be active for the next jobs.
*
* @param expireTimeMs - expire time in ms of this rate limit.
*/
rateLimit(expireTimeMs: number): Promise<void>;
protected moveToActive(client: RedisClient, token: string, jobId?: string): Promise<Job<DataType, ResultType, NameType>>;
private waitForJob;
/**
*
* This function is exposed only for testing purposes.
*/
delay(milliseconds?: number, abortController?: AbortController): Promise<void>;
private updateDelays;
protected nextJobFromJobData(jobData?: JobJsonRaw, jobId?: string, token?: string): Promise<Job<DataType, ResultType, NameType>>;
processJob(job: Job<DataType, ResultType, NameType>, token: string, fetchNextCallback: () => boolean, jobsInProgress: Set<{
job: Job;
ts: number;
}>): Promise<void | Job<DataType, ResultType, NameType>>;
/**
*
* Pauses the processing of this queue only for this worker.
*/
pause(doNotWaitActive?: boolean): Promise<void>;
/**
*
* Resumes processing of this worker (if paused).
*/
resume(): void;
/**
*
* Checks if worker is paused.
*
* @returns true if worker is paused, false otherwise.
*/
isPaused(): boolean;
/**
*
* Checks if worker is currently running.
*
* @returns true if worker is running, false otherwise.
*/
isRunning(): boolean;
/**
*
* Closes the worker and related redis connections.
*
* This method waits for current jobs to finalize before returning.
*
* @param force - Use force boolean parameter if you do not want to wait for
* current jobs to be processed.
*
* @returns Promise that resolves when the worker has been closed.
*/
close(force?: boolean): Promise<void>;
/**
*
* Manually starts the stalled checker.
* The check will run once as soon as this method is called, and
* then every opts.stalledInterval milliseconds until the worker is closed.
* Note: Normally you do not need to call this method, since the stalled checker
* is automatically started when the worker starts processing jobs after
* calling run. However if you want to process the jobs manually you need
* to call this method to start the stalled checker.
*
* @see {@link https://docs.bullmq.io/patterns/manually-fetching-jobs}
*/
startStalledCheckTimer(): Promise<void>;
private startLockExtenderTimer;
/**
* Returns a promise that resolves when active jobs are cleared
*
* @returns
*/
private whenCurrentJobsFinished;
private retryIfFailed;
protected extendLocks(jobs: Job[]): Promise<void>;
private moveStalledJobsToWait;
private notifyFailedJobs;
private moveLimitedBackToWait;
}
+630
View File
@@ -0,0 +1,630 @@
import * as fs from 'fs';
import { URL } from 'url';
import * as path from 'path';
import { v4 } from 'uuid';
// Note: this Polyfill is only needed for Node versions < 15.4.0
import { AbortController } from 'node-abort-controller';
import { delay, DELAY_TIME_1, isNotConnectionError, isRedisInstance, WORKER_SUFFIX, } from '../utils';
import { QueueBase } from './queue-base';
import { Repeat } from './repeat';
import { ChildPool } from './child-pool';
import { Job } from './job';
import { RedisConnection } from './redis-connection';
import sandbox from './sandbox';
import { AsyncFifoQueue } from './async-fifo-queue';
import { DelayedError, RateLimitError, RATE_LIMIT_ERROR, WaitingChildrenError, } from './errors';
// 10 seconds is the maximum time a BRPOPLPUSH can block.
const maximumBlockTimeout = 10;
/**
*
* This class represents a worker that is able to process jobs from the queue.
* As soon as the class is instantiated and a connection to Redis is established
* it will start processing jobs.
*
*/
export class Worker extends QueueBase {
static RateLimitError() {
return new RateLimitError();
}
constructor(name, processor, opts = {}, Connection) {
super(name, Object.assign(Object.assign({}, opts), { blockingConnection: true }), Connection);
this.abortDelayController = null;
this.blockUntil = 0;
this.drained = false;
this.extendLocksTimer = null;
this.limitUntil = 0;
this.waiting = null;
this.running = false;
this.opts = Object.assign({ drainDelay: 5, concurrency: 1, lockDuration: 30000, maxStalledCount: 1, stalledInterval: 30000, autorun: true, runRetryDelay: 15000 }, this.opts);
if (this.opts.stalledInterval <= 0) {
throw new Error('stalledInterval must be greater than 0');
}
this.concurrency = this.opts.concurrency;
this.opts.lockRenewTime =
this.opts.lockRenewTime || this.opts.lockDuration / 2;
this.id = v4();
if (processor) {
if (typeof processor === 'function') {
this.processFn = processor;
}
else {
// SANDBOXED
if (processor instanceof URL) {
if (!fs.existsSync(processor)) {
throw new Error(`URL ${processor} does not exist in the local file system`);
}
processor = processor.href;
}
else {
const supportedFileTypes = ['.js', '.ts', '.flow', '.cjs'];
const processorFile = processor +
(supportedFileTypes.includes(path.extname(processor)) ? '' : '.js');
if (!fs.existsSync(processorFile)) {
throw new Error(`File ${processorFile} does not exist`);
}
}
const mainFile = this.opts.useWorkerThreads
? 'main-worker.js'
: 'main.js';
let mainFilePath = path.join(path.dirname(module.filename), `${mainFile}`);
try {
fs.statSync(mainFilePath); // would throw if file not exists
}
catch (_) {
mainFilePath = path.join(process.cwd(), `dist/cjs/classes/${mainFile}`);
fs.statSync(mainFilePath);
}
this.childPool = new ChildPool({
mainFile: mainFilePath,
useWorkerThreads: this.opts.useWorkerThreads,
});
this.processFn = sandbox(processor, this.childPool).bind(this);
}
if (this.opts.autorun) {
this.run().catch(error => this.emit('error', error));
}
}
const connectionName = this.clientName(WORKER_SUFFIX);
this.blockingConnection = new RedisConnection(isRedisInstance(opts.connection)
? opts.connection.duplicate({ connectionName })
: Object.assign(Object.assign({}, opts.connection), { connectionName }), false, true, opts.skipVersionCheck);
this.blockingConnection.on('error', error => this.emit('error', error));
this.blockingConnection.on('ready', () => setTimeout(() => this.emit('ready'), 0));
}
emit(event, ...args) {
return super.emit(event, ...args);
}
off(eventName, listener) {
super.off(eventName, listener);
return this;
}
on(event, listener) {
super.on(event, listener);
return this;
}
once(event, listener) {
super.once(event, listener);
return this;
}
callProcessJob(job, token) {
return this.processFn(job, token);
}
createJob(data, jobId) {
return this.Job.fromJSON(this, data, jobId);
}
/**
*
* Waits until the worker is ready to start processing jobs.
* In general only useful when writing tests.
*
*/
async waitUntilReady() {
await super.waitUntilReady();
return this.blockingConnection.client;
}
set concurrency(concurrency) {
if (typeof concurrency !== 'number' ||
concurrency < 1 ||
!isFinite(concurrency)) {
throw new Error('concurrency must be a finite number greater than 0');
}
this.opts.concurrency = concurrency;
}
get repeat() {
return new Promise(async (resolve) => {
if (!this._repeat) {
const connection = await this.client;
this._repeat = new Repeat(this.name, Object.assign(Object.assign({}, this.opts), { connection }));
this._repeat.on('error', e => this.emit.bind(this, e));
}
resolve(this._repeat);
});
}
async run() {
if (!this.processFn) {
throw new Error('No process function is defined.');
}
if (this.running) {
throw new Error('Worker is already running.');
}
try {
this.running = true;
if (this.closing) {
return;
}
await this.startStalledCheckTimer();
const jobsInProgress = new Set();
this.startLockExtenderTimer(jobsInProgress);
const asyncFifoQueue = (this.asyncFifoQueue =
new AsyncFifoQueue());
let tokenPostfix = 0;
const client = await this.client;
const bclient = await this.blockingConnection.client;
while (!this.closing) {
let numTotal = asyncFifoQueue.numTotal();
while (!this.waiting &&
numTotal < this.opts.concurrency &&
(!this.limitUntil || numTotal == 0)) {
const token = `${this.id}:${tokenPostfix++}`;
const fetchedJob = this.retryIfFailed(() => this._getNextJob(client, bclient, token, { block: true }), this.opts.runRetryDelay);
asyncFifoQueue.add(fetchedJob);
numTotal = asyncFifoQueue.numTotal();
if (this.waiting && numTotal > 1) {
// We have a job waiting but we have others that we could start processing already
break;
}
// We await here so that we fetch jobs in sequence, this is important to avoid unnecessary calls
// to Redis in high concurrency scenarios.
const job = await fetchedJob;
// No more jobs waiting but we have others that could start processing already
if (!job && numTotal > 1) {
break;
}
// If there are potential jobs to be processed and blockUntil is set, we should exit to avoid waiting
// for processing this job.
if (this.blockUntil) {
break;
}
}
// Since there can be undefined jobs in the queue (when a job fails or queue is empty)
// we iterate until we find a job.
let job;
do {
job = await asyncFifoQueue.fetch();
} while (!job &&
asyncFifoQueue.numTotal() > 0 &&
asyncFifoQueue.numQueued() > 0);
if (job) {
const token = job.token;
asyncFifoQueue.add(this.retryIfFailed(() => this.processJob(job, token, () => asyncFifoQueue.numTotal() <= this.opts.concurrency, jobsInProgress), this.opts.runRetryDelay));
}
}
this.running = false;
return asyncFifoQueue.waitAll();
}
catch (error) {
this.running = false;
throw error;
}
}
/**
* Returns a promise that resolves to the next job in queue.
* @param token - worker token to be assigned to retrieved job
* @returns a Job or undefined if no job was available in the queue.
*/
async getNextJob(token, { block = true } = {}) {
return this._getNextJob(await this.client, await this.blockingConnection.client, token, { block });
}
async _getNextJob(client, bclient, token, { block = true } = {}) {
var _a;
if (this.paused) {
if (block) {
await this.paused;
}
else {
return;
}
}
if (this.closing) {
return;
}
if (this.drained && block && !this.limitUntil && !this.waiting) {
this.waiting = this.waitForJob(bclient);
try {
const jobId = await this.waiting;
return this.moveToActive(client, token, jobId);
}
catch (err) {
// Swallow error if locally paused or closing since we did force a disconnection
if (!(this.paused || this.closing) &&
isNotConnectionError(err)) {
throw err;
}
}
finally {
this.waiting = null;
}
}
else {
if (this.limitUntil) {
(_a = this.abortDelayController) === null || _a === void 0 ? void 0 : _a.abort();
this.abortDelayController = new AbortController();
await this.delay(this.limitUntil, this.abortDelayController);
}
return this.moveToActive(client, token);
}
}
/**
* Overrides the rate limit to be active for the next jobs.
*
* @param expireTimeMs - expire time in ms of this rate limit.
*/
async rateLimit(expireTimeMs) {
await this.client.then(client => client.set(this.keys.limiter, Number.MAX_SAFE_INTEGER, 'PX', expireTimeMs));
}
async moveToActive(client, token, jobId) {
// If we get the special delayed job ID, we pick the delay as the next
// block timeout.
if (jobId && jobId.startsWith('0:')) {
this.blockUntil = parseInt(jobId.split(':')[1]) || 0;
// Remove marker from active list.
await client.lrem(this.keys.active, 1, jobId);
if (this.blockUntil > 0) {
return;
}
}
const [jobData, id, limitUntil, delayUntil] = await this.scripts.moveToActive(client, token, jobId);
this.updateDelays(limitUntil, delayUntil);
return this.nextJobFromJobData(jobData, id, token);
}
async waitForJob(bclient) {
if (this.paused) {
return;
}
try {
const opts = this.opts;
if (!this.closing) {
let blockTimeout = Math.max(this.blockUntil
? (this.blockUntil - Date.now()) / 1000
: opts.drainDelay, 0);
let jobId;
// Blocking for less than 50ms is useless.
if (blockTimeout > 0.05) {
blockTimeout = this.blockingConnection.capabilities.canDoubleTimeout
? blockTimeout
: Math.ceil(blockTimeout);
// We restrict the maximum block timeout to 10 second to avoid
// blocking the connection for too long in the case of reconnections
// reference: https://github.com/taskforcesh/bullmq/issues/1658
blockTimeout = Math.min(blockTimeout, maximumBlockTimeout);
jobId = await bclient.brpoplpush(this.keys.wait, this.keys.active, blockTimeout);
}
else {
jobId = await bclient.rpoplpush(this.keys.wait, this.keys.active);
}
this.blockUntil = 0;
return jobId;
}
}
catch (error) {
if (isNotConnectionError(error)) {
this.emit('error', error);
}
if (!this.closing) {
await this.delay();
}
}
finally {
this.waiting = null;
}
}
/**
*
* This function is exposed only for testing purposes.
*/
async delay(milliseconds, abortController) {
await delay(milliseconds || DELAY_TIME_1, abortController);
}
updateDelays(limitUntil = 0, delayUntil = 0) {
this.limitUntil = Math.max(limitUntil, 0) || 0;
this.blockUntil = Math.max(delayUntil, 0) || 0;
}
async nextJobFromJobData(jobData, jobId, token) {
if (!jobData) {
if (!this.drained) {
this.emit('drained');
this.drained = true;
}
}
else {
this.drained = false;
const job = this.createJob(jobData, jobId);
job.token = token;
if (job.opts.repeat) {
const repeat = await this.repeat;
await repeat.addNextRepeatableJob(job.name, job.data, job.opts);
}
return job;
}
}
async processJob(job, token, fetchNextCallback = () => true, jobsInProgress) {
if (!job || this.closing || this.paused) {
return;
}
const handleCompleted = async (result) => {
if (!this.connection.closing) {
const completed = await job.moveToCompleted(result, token, fetchNextCallback() && !(this.closing || this.paused));
this.emit('completed', job, result, 'active');
const [jobData, jobId, limitUntil, delayUntil] = completed || [];
this.updateDelays(limitUntil, delayUntil);
return this.nextJobFromJobData(jobData, jobId, token);
}
};
const handleFailed = async (err) => {
if (!this.connection.closing) {
try {
if (err.message == RATE_LIMIT_ERROR) {
this.limitUntil = await this.moveLimitedBackToWait(job, token);
return;
}
if (err instanceof DelayedError ||
err.message == 'DelayedError' ||
err instanceof WaitingChildrenError ||
err.name == 'WaitingChildrenError') {
return;
}
await job.moveToFailed(err, token);
this.emit('failed', job, err, 'active');
}
catch (err) {
this.emit('error', err);
// It probably means that the job has lost the lock before completion
// A worker will (or already has) moved the job back
// to the waiting list (as stalled)
}
}
};
this.emit('active', job, 'waiting');
const inProgressItem = { job, ts: Date.now() };
try {
jobsInProgress.add(inProgressItem);
const result = await this.callProcessJob(job, token);
return await handleCompleted(result);
}
catch (err) {
return handleFailed(err);
}
finally {
jobsInProgress.delete(inProgressItem);
}
}
/**
*
* Pauses the processing of this queue only for this worker.
*/
async pause(doNotWaitActive) {
if (!this.paused) {
this.paused = new Promise(resolve => {
this.resumeWorker = function () {
resolve();
this.paused = null; // Allow pause to be checked externally for paused state.
this.resumeWorker = null;
};
});
await (!doNotWaitActive && this.whenCurrentJobsFinished());
this.emit('paused');
}
}
/**
*
* Resumes processing of this worker (if paused).
*/
resume() {
if (this.resumeWorker) {
this.resumeWorker();
this.emit('resumed');
}
}
/**
*
* Checks if worker is paused.
*
* @returns true if worker is paused, false otherwise.
*/
isPaused() {
return !!this.paused;
}
/**
*
* Checks if worker is currently running.
*
* @returns true if worker is running, false otherwise.
*/
isRunning() {
return this.running;
}
/**
*
* Closes the worker and related redis connections.
*
* This method waits for current jobs to finalize before returning.
*
* @param force - Use force boolean parameter if you do not want to wait for
* current jobs to be processed.
*
* @returns Promise that resolves when the worker has been closed.
*/
close(force = false) {
if (this.closing) {
return this.closing;
}
this.closing = (async () => {
var _a;
this.emit('closing', 'closing queue');
(_a = this.abortDelayController) === null || _a === void 0 ? void 0 : _a.abort();
const client = await this.blockingConnection.client;
this.resume();
await Promise.resolve()
.finally(() => {
return force || this.whenCurrentJobsFinished(false);
})
.finally(() => {
var _a;
const closePoolPromise = (_a = this.childPool) === null || _a === void 0 ? void 0 : _a.clean();
if (force) {
// since we're not waiting for the job to end attach
// an error handler to avoid crashing the whole process
closePoolPromise === null || closePoolPromise === void 0 ? void 0 : closePoolPromise.catch(err => {
console.error(err); // TODO: emit error in next breaking change version
});
return;
}
return closePoolPromise;
})
.finally(() => clearTimeout(this.extendLocksTimer))
.finally(() => clearTimeout(this.stalledCheckTimer))
.finally(() => client.disconnect())
.finally(() => this.connection.close())
.finally(() => this.emit('closed'));
this.closed = true;
})();
return this.closing;
}
/**
*
* Manually starts the stalled checker.
* The check will run once as soon as this method is called, and
* then every opts.stalledInterval milliseconds until the worker is closed.
* Note: Normally you do not need to call this method, since the stalled checker
* is automatically started when the worker starts processing jobs after
* calling run. However if you want to process the jobs manually you need
* to call this method to start the stalled checker.
*
* @see {@link https://docs.bullmq.io/patterns/manually-fetching-jobs}
*/
async startStalledCheckTimer() {
if (!this.opts.skipStalledCheck) {
clearTimeout(this.stalledCheckTimer);
if (!this.closing) {
try {
await this.checkConnectionError(() => this.moveStalledJobsToWait());
this.stalledCheckTimer = setTimeout(async () => {
await this.startStalledCheckTimer();
}, this.opts.stalledInterval);
}
catch (err) {
this.emit('error', err);
}
}
}
}
startLockExtenderTimer(jobsInProgress) {
if (!this.opts.skipLockRenewal) {
clearTimeout(this.extendLocksTimer);
if (!this.closed) {
this.extendLocksTimer = setTimeout(async () => {
// Get all the jobs whose locks expire in less than 1/2 of the lockRenewTime
const now = Date.now();
const jobsToExtend = [];
for (const item of jobsInProgress) {
const { job, ts } = item;
if (!ts) {
item.ts = now;
continue;
}
if (ts + this.opts.lockRenewTime / 2 < now) {
item.ts = now;
jobsToExtend.push(job);
}
}
try {
if (jobsToExtend.length) {
await this.extendLocks(jobsToExtend);
}
}
catch (err) {
this.emit('error', err);
}
this.startLockExtenderTimer(jobsInProgress);
}, this.opts.lockRenewTime / 2);
}
}
}
/**
* Returns a promise that resolves when active jobs are cleared
*
* @returns
*/
async whenCurrentJobsFinished(reconnect = true) {
//
// Force reconnection of blocking connection to abort blocking redis call immediately.
//
if (this.waiting) {
// If we are not going to reconnect, we will not wait for the disconnection.
await this.blockingConnection.disconnect(reconnect);
}
else {
reconnect = false;
}
if (this.asyncFifoQueue) {
await this.asyncFifoQueue.waitAll();
}
reconnect && (await this.blockingConnection.reconnect());
}
async retryIfFailed(fn, delayInMs) {
const retry = 1;
do {
try {
return await fn();
}
catch (err) {
this.emit('error', err);
if (delayInMs) {
await this.delay(delayInMs);
}
else {
return;
}
}
} while (retry);
}
async extendLocks(jobs) {
try {
const multi = (await this.client).multi();
for (const job of jobs) {
await this.scripts.extendLock(job.id, job.token, this.opts.lockDuration, multi);
}
const result = (await multi.exec());
for (const [err, jobId] of result) {
if (err) {
// TODO: signal process function that the job has been lost.
this.emit('error', new Error(`could not renew lock for job ${jobId}`));
}
}
}
catch (err) {
this.emit('error', err);
}
}
async moveStalledJobsToWait() {
const chunkSize = 50;
const [failed, stalled] = await this.scripts.moveStalledJobsToWait();
stalled.forEach((jobId) => this.emit('stalled', jobId, 'active'));
const jobPromises = [];
for (let i = 0; i < failed.length; i++) {
jobPromises.push(Job.fromId(this, failed[i]));
if ((i + 1) % chunkSize === 0) {
this.notifyFailedJobs(await Promise.all(jobPromises));
jobPromises.length = 0;
}
}
this.notifyFailedJobs(await Promise.all(jobPromises));
}
notifyFailedJobs(failedJobs) {
failedJobs.forEach((job) => this.emit('failed', job, new Error('job stalled more than allowable limit'), 'active'));
}
moveLimitedBackToWait(job, token) {
return this.scripts.moveJobFromActiveToWait(job.id, token);
}
}
//# sourceMappingURL=worker.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../../src/classes/worker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAE1B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,EAAE,EAAE,MAAM,MAAM,CAAC;AAE1B,gEAAgE;AAChE,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAWxD,OAAO,EACL,KAAK,EACL,YAAY,EACZ,oBAAoB,EACpB,eAAe,EACf,aAAa,GACd,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAClC,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,GAAG,EAAE,MAAM,OAAO,CAAC;AAC5B,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,OAAO,MAAM,WAAW,CAAC;AAChC,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EACL,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,oBAAoB,GACrB,MAAM,UAAU,CAAC;AAElB,yDAAyD;AACzD,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAkH/B;;;;;;GAMG;AACH,MAAM,OAAO,MAIX,SAAQ,SAAS;IAyBjB,MAAM,CAAC,cAAc;QACnB,OAAO,IAAI,cAAc,EAAE,CAAC;IAC9B,CAAC;IAED,YACE,IAAY,EACZ,SAA2E,EAC3E,OAAsB,EAAE,EACxB,UAAmC;QAEnC,KAAK,CACH,IAAI,kCAEC,IAAI,KACP,kBAAkB,EAAE,IAAI,KAE1B,UAAU,CACX,CAAC;QAtCI,yBAAoB,GAA2B,IAAI,CAAC;QAOpD,eAAU,GAAG,CAAC,CAAC;QAEf,YAAO,GAAY,KAAK,CAAC;QACzB,qBAAgB,GAA0B,IAAI,CAAC;QAC/C,eAAU,GAAG,CAAC,CAAC;QAGf,YAAO,GAA2B,IAAI,CAAC;QAKrC,YAAO,GAAG,KAAK,CAAC;QAqBxB,IAAI,CAAC,IAAI,mBACP,UAAU,EAAE,CAAC,EACb,WAAW,EAAE,CAAC,EACd,YAAY,EAAE,KAAK,EACnB,eAAe,EAAE,CAAC,EAClB,eAAe,EAAE,KAAK,EACtB,OAAO,EAAE,IAAI,EACb,aAAa,EAAE,KAAK,IACjB,IAAI,CAAC,IAAI,CACb,CAAC;QAEF,IAAI,IAAI,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,EAAE;YAClC,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;SAC3D;QAED,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC;QAEzC,IAAI,CAAC,IAAI,CAAC,aAAa;YACrB,IAAI,CAAC,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;QAExD,IAAI,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC;QAEf,IAAI,SAAS,EAAE;YACb,IAAI,OAAO,SAAS,KAAK,UAAU,EAAE;gBACnC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;aAC5B;iBAAM;gBACL,YAAY;gBACZ,IAAI,SAAS,YAAY,GAAG,EAAE;oBAC5B,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;wBAC7B,MAAM,IAAI,KAAK,CACb,OAAO,SAAS,0CAA0C,CAC3D,CAAC;qBACH;oBACD,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC;iBAC5B;qBAAM;oBACL,MAAM,kBAAkB,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;oBAC3D,MAAM,aAAa,GACjB,SAAS;wBACT,CAAC,kBAAkB,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;oBAEtE,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE;wBACjC,MAAM,IAAI,KAAK,CAAC,QAAQ,aAAa,iBAAiB,CAAC,CAAC;qBACzD;iBACF;gBAED,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB;oBACzC,CAAC,CAAC,gBAAgB;oBAClB,CAAC,CAAC,SAAS,CAAC;gBACd,IAAI,YAAY,GAAG,IAAI,CAAC,IAAI,CAC1B,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,EAC7B,GAAG,QAAQ,EAAE,CACd,CAAC;gBACF,IAAI;oBACF,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,iCAAiC;iBAC7D;gBAAC,OAAO,CAAC,EAAE;oBACV,YAAY,GAAG,IAAI,CAAC,IAAI,CACtB,OAAO,CAAC,GAAG,EAAE,EACb,oBAAoB,QAAQ,EAAE,CAC/B,CAAC;oBACF,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;iBAC3B;gBAED,IAAI,CAAC,SAAS,GAAG,IAAI,SAAS,CAAC;oBAC7B,QAAQ,EAAE,YAAY;oBACtB,gBAAgB,EAAE,IAAI,CAAC,IAAI,CAAC,gBAAgB;iBAC7C,CAAC,CAAC;gBAEH,IAAI,CAAC,SAAS,GAAG,OAAO,CACtB,SAAS,EACT,IAAI,CAAC,SAAS,CACf,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aACd;YAED,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;gBACrB,IAAI,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;aACtD;SACF;QAED,MAAM,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QACtD,IAAI,CAAC,kBAAkB,GAAG,IAAI,eAAe,CAC3C,eAAe,CAAC,IAAI,CAAC,UAAU,CAAC;YAC9B,CAAC,CAAS,IAAI,CAAC,UAAW,CAAC,SAAS,CAAC,EAAE,cAAc,EAAE,CAAC;YACxD,CAAC,iCAAM,IAAI,CAAC,UAAU,KAAE,cAAc,GAAE,EAC1C,KAAK,EACL,IAAI,EACJ,IAAI,CAAC,gBAAgB,CACtB,CAAC;QACF,IAAI,CAAC,kBAAkB,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;QACxE,IAAI,CAAC,kBAAkB,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CACvC,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CACxC,CAAC;IACJ,CAAC;IAED,IAAI,CACF,KAAQ,EACR,GAAG,IAAmE;QAEtE,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,CAAC;IACpC,CAAC;IAED,GAAG,CACD,SAAY,EACZ,QAA2D;QAE3D,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAC/B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,EAAE,CACA,KAAQ,EACR,QAA2D;QAE3D,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAC1B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CACF,KAAQ,EACR,QAA2D;QAE3D,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAC5B,OAAO,IAAI,CAAC;IACd,CAAC;IAES,cAAc,CACtB,GAAwC,EACxC,KAAa;QAEb,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACpC,CAAC;IAES,SAAS,CACjB,IAAgB,EAChB,KAAa;QAEb,OAAO,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAoB,EAAE,IAAI,EAAE,KAAK,CAIzD,CAAC;IACJ,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,cAAc;QAClB,MAAM,KAAK,CAAC,cAAc,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC;IACxC,CAAC;IAED,IAAI,WAAW,CAAC,WAAmB;QACjC,IACE,OAAO,WAAW,KAAK,QAAQ;YAC/B,WAAW,GAAG,CAAC;YACf,CAAC,QAAQ,CAAC,WAAW,CAAC,EACtB;YACA,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;SACvE;QACD,IAAI,CAAC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACtC,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,OAAO,CAAS,KAAK,EAAC,OAAO,EAAC,EAAE;YACzC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;gBACjB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC;gBACrC,IAAI,CAAC,OAAO,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,IAAILine truncated
+124
View File
@@ -0,0 +1,124 @@
--[[
Adds a delayed job to the queue by doing the following:
- Increases the job counter if needed.
- Creates a new job key with the job data.
- computes timestamp.
- adds to delayed zset.
- Emits a global event 'delayed' if the job is delayed.
Input:
KEYS[1] 'wait',
KEYS[2] 'paused'
KEYS[3] 'meta'
KEYS[4] 'id'
KEYS[5] 'delayed'
KEYS[6] 'completed'
KEYS[7] events stream key
ARGV[1] msgpacked arguments array
[1] key prefix,
[2] custom id (use custom instead of one generated automatically)
[3] name
[4] timestamp
[5] parentKey?
x [6] waitChildrenKey key.
[7] parent dependencies key.
[8] parent? {id, queueKey}
[9] repeat job key
ARGV[2] Json stringified job data
ARGV[3] msgpacked options
Output:
jobId - OK
-5 - Missing parent key
]]
local waitKey = KEYS[1]
local pausedKey = KEYS[2]
local metaKey = KEYS[3]
local idKey = KEYS[4]
local delayedKey = KEYS[5]
local completedKey = KEYS[6]
local eventsKey = KEYS[7]
local jobId
local jobIdKey
local rcall = redis.call
local args = cmsgpack.unpack(ARGV[1])
local data = ARGV[2]
local opts = cmsgpack.unpack(ARGV[3])
local parentKey = args[5]
local repeatJobKey = args[9]
local parent = args[8]
local parentData
-- Includes
--- @include "includes/storeJob"
--- @include "includes/addDelayMarkerIfNeeded"
--- @include "includes/getTargetQueueList"
--- @include "includes/getNextDelayedTimestamp"
--- @include "includes/updateExistingJobsParent"
--- @include "includes/getOrSetMaxEvents"
if parentKey ~= nil then
if rcall("EXISTS", parentKey) ~= 1 then return -5 end
parentData = cjson.encode(parent)
end
local jobCounter = rcall("INCR", idKey)
local maxEvents = getOrSetMaxEvents(metaKey)
local parentDependenciesKey = args[7]
local timestamp = args[4]
if args[2] == "" then
jobId = jobCounter
jobIdKey = args[1] .. jobId
else
-- Refactor to: handleDuplicateJob.lua
jobId = args[2]
jobIdKey = args[1] .. jobId
if rcall("EXISTS", jobIdKey) == 1 then
updateExistingJobsParent(parentKey, parent, parentData,
parentDependenciesKey, completedKey, jobIdKey,
jobId, timestamp)
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
"duplicated", "jobId", jobId)
return jobId .. "" -- convert to string
end
end
-- Store the job.
local delay, priority = storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2],
opts, timestamp, parentKey, parentData,
repeatJobKey)
-- Compute delayed timestamp and the score.
local delayedTimestamp = (delay > 0 and (timestamp + delay)) or 0
local score = delayedTimestamp * 0x1000 + bit.band(jobCounter, 0xfff)
rcall("ZADD", delayedKey, score, jobId)
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
"jobId", jobId, "delay", delayedTimestamp)
-- If wait list is empty, and this delayed job is the next one to be processed,
-- then we need to signal the workers by adding a dummy job (jobId 0:delay) to the wait list.
local target = getTargetQueueList(metaKey, KEYS[1], KEYS[2])
addDelayMarkerIfNeeded(target, delayedKey)
-- Check if this job is a child of another job, if so add it to the parents dependencies
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
-- fail in this case.
if parentDependenciesKey ~= nil then
rcall("SADD", parentDependenciesKey, jobIdKey)
end
return jobId .. "" -- convert to string
+101
View File
@@ -0,0 +1,101 @@
--[[
Adds a parent job to the queue by doing the following:
- Increases the job counter if needed.
- Creates a new job key with the job data.
- adds the job to the waiting-children zset
Input:
KEYS[1] 'meta'
KEYS[2] 'id'
KEYS[3] 'completed'
KEYS[4] events stream key
ARGV[1] msgpacked arguments array
[1] key prefix,
[2] custom id (will not generate one automatically)
[3] name
[4] timestamp
[5] parentKey?
[6] waitChildrenKey key.
[7] parent dependencies key.
[8] parent? {id, queueKey}
[9] repeat job key
ARGV[2] Json stringified job data
ARGV[3] msgpacked options
Output:
jobId - OK
-5 - Missing parent key
]]
local metaKey = KEYS[1]
local idKey = KEYS[2]
local completedKey = KEYS[3]
local eventsKey = KEYS[4]
local jobId
local jobIdKey
local rcall = redis.call
local args = cmsgpack.unpack(ARGV[1])
local data = ARGV[2]
local opts = cmsgpack.unpack(ARGV[3])
local parentKey = args[5]
local repeatJobKey = args[9]
local parent = args[8]
local parentData
-- Includes
--- @include "includes/storeJob"
--- @include "includes/updateExistingJobsParent"
--- @include "includes/getOrSetMaxEvents"
if parentKey ~= nil then
if rcall("EXISTS", parentKey) ~= 1 then return -5 end
parentData = cjson.encode(parent)
end
local jobCounter = rcall("INCR", idKey)
local maxEvents = getOrSetMaxEvents(metaKey)
local parentDependenciesKey = args[7]
local timestamp = args[4]
if args[2] == "" then
jobId = jobCounter
jobIdKey = args[1] .. jobId
else
jobId = args[2]
jobIdKey = args[1] .. jobId
if rcall("EXISTS", jobIdKey) == 1 then
updateExistingJobsParent(parentKey, parent, parentData,
parentDependenciesKey, completedKey, jobIdKey,
jobId, timestamp)
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
"duplicated", "jobId", jobId)
return jobId .. "" -- convert to string
end
end
-- Store the job.
storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2], opts, timestamp,
parentKey, parentData, repeatJobKey)
local waitChildrenKey = args[6]
rcall("ZADD", waitChildrenKey, timestamp, jobId)
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
"waiting-children", "jobId", jobId)
-- Check if this job is a child of another job, if so add it to the parents dependencies
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
-- fail in this case.
if parentDependenciesKey ~= nil then
rcall("SADD", parentDependenciesKey, jobIdKey)
end
return jobId .. "" -- convert to string
+117
View File
@@ -0,0 +1,117 @@
--[[
Adds a priotitized job to the queue by doing the following:
- Increases the job counter if needed.
- Creates a new job key with the job data.
- Adds the job to the "added" list so that workers gets notified.
Input:
KEYS[1] 'wait',
KEYS[2] 'paused'
KEYS[3] 'meta'
KEYS[4] 'id'
KEYS[5] 'prioritized'
KEYS[6] 'completed'
KEYS[7] events stream key
KEYS[8] 'pc' priority counter
ARGV[1] msgpacked arguments array
[1] key prefix,
[2] custom id (will not generate one automatically)
[3] name
[4] timestamp
[5] parentKey?
[6] waitChildrenKey key.
[7] parent dependencies key.
[8] parent? {id, queueKey}
[9] repeat job key
ARGV[2] Json stringified job data
ARGV[3] msgpacked options
Output:
jobId - OK
-5 - Missing parent key
]]
local waitKey = KEYS[1]
local pausedKey = KEYS[2]
local metaKey = KEYS[3]
local idKey = KEYS[4]
local priorityKey = KEYS[5]
local completedKey = KEYS[6]
local eventsKey = KEYS[7]
local priorityCounterKey = KEYS[8]
local jobId
local jobIdKey
local rcall = redis.call
local args = cmsgpack.unpack(ARGV[1])
local data = ARGV[2]
local opts = cmsgpack.unpack(ARGV[3])
local parentKey = args[5]
local repeatJobKey = args[9]
local parent = args[8]
local parentData
-- Includes
--- @include "includes/storeJob"
--- @include "includes/addJobWithPriority"
--- @include "includes/getTargetQueueList"
--- @include "includes/updateExistingJobsParent"
--- @include "includes/getOrSetMaxEvents"
if parentKey ~= nil then
if rcall("EXISTS", parentKey) ~= 1 then return -5 end
parentData = cjson.encode(parent)
end
local jobCounter = rcall("INCR", idKey)
local maxEvents = getOrSetMaxEvents(metaKey)
local parentDependenciesKey = args[7]
local timestamp = args[4]
if args[2] == "" then
jobId = jobCounter
jobIdKey = args[1] .. jobId
else
jobId = args[2]
jobIdKey = args[1] .. jobId
if rcall("EXISTS", jobIdKey) == 1 then
updateExistingJobsParent(parentKey, parent, parentData,
parentDependenciesKey, completedKey, jobIdKey,
jobId, timestamp)
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
"duplicated", "jobId", jobId)
return jobId .. "" -- convert to string
end
end
-- Store the job.
local delay, priority = storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2],
opts, timestamp, parentKey, parentData,
repeatJobKey)
local target, paused = getTargetQueueList(metaKey, waitKey, pausedKey)
addJobWithPriority(waitKey, priorityKey, priority, paused, jobId,
priorityCounterKey)
-- Emit waiting event
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting",
"jobId", jobId)
-- Check if this job is a child of another job, if so add it to the parents dependencies
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
-- fail in this case.
if parentDependenciesKey ~= nil then
rcall("SADD", parentDependenciesKey, jobIdKey)
end
return jobId .. "" -- convert to string
+117
View File
@@ -0,0 +1,117 @@
--[[
Adds a job to the queue by doing the following:
- Increases the job counter if needed.
- Creates a new job key with the job data.
- if delayed:
- computes timestamp.
- adds to delayed zset.
- Emits a global event 'delayed' if the job is delayed.
- if not delayed
- Adds the jobId to the wait/paused list in one of three ways:
- LIFO
- FIFO
- prioritized.
- Adds the job to the "added" list so that workers gets notified.
Input:
KEYS[1] 'wait',
KEYS[2] 'paused'
KEYS[3] 'meta'
KEYS[4] 'id'
KEYS[5] 'completed'
KEYS[6] events stream key
ARGV[1] msgpacked arguments array
[1] key prefix,
[2] custom id (will not generate one automatically)
[3] name
[4] timestamp
[5] parentKey?
[6] waitChildrenKey key.
[7] parent dependencies key.
[8] parent? {id, queueKey}
[9] repeat job key
ARGV[2] Json stringified job data
ARGV[3] msgpacked options
Output:
jobId - OK
-5 - Missing parent key
]]
local eventsKey = KEYS[6]
local jobId
local jobIdKey
local rcall = redis.call
local args = cmsgpack.unpack(ARGV[1])
local data = ARGV[2]
local opts = cmsgpack.unpack(ARGV[3])
local parentKey = args[5]
local repeatJobKey = args[9]
local parent = args[8]
local parentData
-- Includes
--- @include "includes/storeJob"
--- @include "includes/updateExistingJobsParent"
--- @include "includes/getTargetQueueList"
--- @include "includes/getOrSetMaxEvents"
if parentKey ~= nil then
if rcall("EXISTS", parentKey) ~= 1 then return -5 end
parentData = cjson.encode(parent)
end
local jobCounter = rcall("INCR", KEYS[4])
local metaKey = KEYS[3]
local maxEvents = getOrSetMaxEvents(metaKey)
local parentDependenciesKey = args[7]
local timestamp = args[4]
if args[2] == "" then
jobId = jobCounter
jobIdKey = args[1] .. jobId
else
jobId = args[2]
jobIdKey = args[1] .. jobId
if rcall("EXISTS", jobIdKey) == 1 then
updateExistingJobsParent(parentKey, parent, parentData,
parentDependenciesKey, KEYS[5], jobIdKey,
jobId, timestamp)
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
"duplicated", "jobId", jobId)
return jobId .. "" -- convert to string
end
end
-- Store the job.
storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2], opts, timestamp,
parentKey, parentData, repeatJobKey)
local target, paused = getTargetQueueList(metaKey, KEYS[1], KEYS[2])
-- LIFO or FIFO
local pushCmd = opts['lifo'] and 'RPUSH' or 'LPUSH'
rcall(pushCmd, target, jobId)
-- Emit waiting event
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting",
"jobId", jobId)
-- Check if this job is a child of another job, if so add it to the parents dependencies
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
-- fail in this case.
if parentDependenciesKey ~= nil then
rcall("SADD", parentDependenciesKey, jobIdKey)
end
return jobId .. "" -- convert to string
+40
View File
@@ -0,0 +1,40 @@
--[[
Change job delay when it is in delayed set.
Input:
KEYS[1] delayed key
KEYS[2] job key
KEYS[3] events stream
ARGV[1] delay
ARGV[2] delayedTimestamp
ARGV[3] the id of the job
Output:
0 - OK
-1 - Missing job.
-3 - Job not in delayed set.
Events:
- delayed key.
]]
local rcall = redis.call
if rcall("EXISTS", KEYS[2]) == 1 then
local jobId = ARGV[3]
local score = tonumber(ARGV[2])
local delayedTimestamp = (score / 0x1000)
local numRemovedElements = rcall("ZREM", KEYS[1], jobId)
if numRemovedElements < 1 then
return -3
end
rcall("HSET", KEYS[2], "delay", tonumber(ARGV[1]))
rcall("ZADD", KEYS[1], score, jobId)
rcall("XADD", KEYS[3], "*", "event", "delayed", "jobId", jobId, "delay", delayedTimestamp)
return 0
else
return -1
end
+52
View File
@@ -0,0 +1,52 @@
--[[
Change job priority
Input:
KEYS[1] 'wait',
KEYS[2] 'paused'
KEYS[3] 'meta'
KEYS[4] 'prioritized'
KEYS[5] 'pc' priority counter
ARGV[1] priority value
ARGV[2] job key
ARGV[3] job id
ARGV[4] lifo
Output:
0 - OK
-1 - Missing job
]]
local jobKey = ARGV[2]
local jobId = ARGV[3]
local priority = tonumber(ARGV[1])
local rcall = redis.call
-- Includes
--- @include "includes/addJobWithPriority"
--- @include "includes/getTargetQueueList"
if rcall("EXISTS", jobKey) == 1 then
local target, paused = getTargetQueueList(KEYS[3], KEYS[1], KEYS[2])
if rcall("ZREM", KEYS[4], jobId) > 0 then
addJobWithPriority(KEYS[1], KEYS[4], priority, paused, jobId, KEYS[5])
else
local numRemovedElements = rcall("LREM", target, -1, jobId)
if numRemovedElements > 0 then
-- Standard or priority add
if priority == 0 then
-- LIFO or FIFO
local pushCmd = ARGV[4] == '1' and 'RPUSH' or 'LPUSH';
rcall(pushCmd, target, jobId)
else
addJobWithPriority(KEYS[1], KEYS[4], priority, paused, jobId, KEYS[5])
end
end
end
rcall("HSET", jobKey, "priority", priority)
return 0
else
return -1
end
+54
View File
@@ -0,0 +1,54 @@
--[[
Remove jobs from the specific set.
Input:
KEYS[1] set key,
KEYS[2] events stream key
ARGV[1] jobKey prefix
ARGV[2] timestamp
ARGV[3] limit the number of jobs to be removed. 0 is unlimited
ARGV[4] set name, can be any of 'wait', 'active', 'paused', 'delayed', 'completed', or 'failed'
]]
local rcall = redis.call
local rangeStart = 0
local rangeEnd = -1
local limit = tonumber(ARGV[3])
-- If we're only deleting _n_ items, avoid retrieving all items
-- for faster performance
--
-- Start from the tail of the list, since that's where oldest elements
-- are generally added for FIFO lists
if limit > 0 then
rangeStart = -1 - limit + 1
rangeEnd = -1
end
-- Includes
--- @include "includes/cleanList"
--- @include "includes/cleanSet"
local result
if ARGV[4] == "active" then
result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], false)
elseif ARGV[4] == "delayed" then
rangeEnd = "+inf"
result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
{"processedOn", "timestamp"}, false)
elseif ARGV[4] == "prioritized" then
rangeEnd = "+inf"
result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
{"timestamp"}, false)
elseif ARGV[4] == "wait" or ARGV[4] == "paused" then
result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], true)
else
rangeEnd = ARGV[2]
result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
{"finishedOn"}, true)
end
rcall("XADD", KEYS[2], "*", "event", "cleaned", "count", result[2])
return result[1]
+26
View File
@@ -0,0 +1,26 @@
--[[
Drains the queue, removes all jobs that are waiting
or delayed, but not active, completed or failed
Input:
KEYS[1] 'wait',
KEYS[2] 'paused'
KEYS[3] 'delayed'
KEYS[4] 'prioritized'
ARGV[1] queue key prefix
]]
local rcall = redis.call
local queueBaseKey = ARGV[1]
--- @include "includes/removeListJobs"
--- @include "includes/removeZSetJobs"
removeListJobs(KEYS[1], true, queueBaseKey, 0) --wait
removeListJobs(KEYS[2], true, queueBaseKey, 0) --paused
if KEYS[3] ~= "" then
removeZSetJobs(KEYS[3], true, queueBaseKey, 0) --delayed
end
removeZSetJobs(KEYS[4], true, queueBaseKey, 0) --prioritized
+23
View File
@@ -0,0 +1,23 @@
--[[
Extend lock and removes the job from the stalled set.
Input:
KEYS[1] 'lock',
KEYS[2] 'stalled'
ARGV[1] token
ARGV[2] lock duration in milliseconds
ARGV[3] jobid
Output:
"1" if lock extented succesfully.
]]
local rcall = redis.call
if rcall("GET", KEYS[1]) == ARGV[1] then
-- if rcall("SET", KEYS[1], ARGV[1], "PX", ARGV[2], "XX") then
if rcall("SET", KEYS[1], ARGV[1], "PX", ARGV[2]) then
rcall("SREM", KEYS[2], ARGV[3])
return 1
end
end
return 0
+35
View File
@@ -0,0 +1,35 @@
--[[
Get counts per provided states
Input:
KEYS[1] 'prefix'
ARGV[1...] types
]]
local rcall = redis.call;
local prefix = KEYS[1]
local results = {}
for i = 1, #ARGV do
local stateKey = prefix .. ARGV[i]
if ARGV[i] == "wait" or ARGV[i] == "paused" then
local marker = rcall("LINDEX", stateKey, -1)
if marker and string.sub(marker, 1, 2) == "0:" then
local count = rcall("LLEN", stateKey)
if count > 1 then
rcall("RPOP", stateKey)
results[#results+1] = count-1
else
results[#results+1] = 0
end
else
results[#results+1] = rcall("LLEN", stateKey)
end
elseif ARGV[i] == "active" then
results[#results+1] = rcall("LLEN", stateKey)
else
results[#results+1] = rcall("ZCARD", stateKey)
end
end
return results
+69
View File
@@ -0,0 +1,69 @@
--[[
Get job ids per provided states
Input:
KEYS[1] 'prefix'
ARGV[1] start
ARGV[2] end
ARGV[3] asc
ARGV[4...] types
]]
local rcall = redis.call
local prefix = KEYS[1]
local rangeStart = tonumber(ARGV[1])
local rangeEnd = tonumber(ARGV[2])
local asc = ARGV[3]
local results = {}
local function getRangeInList(listKey, asc, rangeStart, rangeEnd, results)
if asc == "1" then
local modifiedRangeStart
local modifiedRangeEnd
if rangeStart == -1 then
modifiedRangeStart = 0
else
modifiedRangeStart = -(rangeStart + 1)
end
if rangeEnd == -1 then
modifiedRangeEnd = 0
else
modifiedRangeEnd = -(rangeEnd + 1)
end
results[#results+1] = rcall("LRANGE", listKey,
modifiedRangeEnd,
modifiedRangeStart)
else
results[#results+1] = rcall("LRANGE", listKey, rangeStart, rangeEnd)
end
end
for i = 4, #ARGV do
local stateKey = prefix .. ARGV[i]
if ARGV[i] == "wait" or ARGV[i] == "paused" then
local marker = rcall("LINDEX", stateKey, -1)
if marker and string.sub(marker, 1, 2) == "0:" then
local count = rcall("LLEN", stateKey)
if count > 1 then
rcall("RPOP", stateKey)
getRangeInList(stateKey, asc, rangeStart, rangeEnd, results)
else
results[#results+1] = {}
end
else
getRangeInList(stateKey, asc, rangeStart, rangeEnd, results)
end
elseif ARGV[i] == "active" then
getRangeInList(stateKey, asc, rangeStart, rangeEnd, results)
else
if asc == "1" then
results[#results+1] = rcall("ZRANGE", stateKey, rangeStart, rangeEnd)
else
results[#results+1] = rcall("ZREVRANGE", stateKey, rangeStart, rangeEnd)
end
end
end
return results
+65
View File
@@ -0,0 +1,65 @@
--[[
Get a job state
Input:
KEYS[1] 'completed' key,
KEYS[2] 'failed' key
KEYS[3] 'delayed' key
KEYS[4] 'active' key
KEYS[5] 'wait' key
KEYS[6] 'paused' key
KEYS[7] 'waiting-children' key
KEYS[8] 'prioritized' key
ARGV[1] job id
Output:
'completed'
'failed'
'delayed'
'active'
'prioritized'
'waiting'
'waiting-children'
'unknown'
]]
local rcall = redis.call
if rcall("ZSCORE", KEYS[1], ARGV[1]) ~= false then
return "completed"
end
if rcall("ZSCORE", KEYS[2], ARGV[1]) ~= false then
return "failed"
end
if rcall("ZSCORE", KEYS[3], ARGV[1]) ~= false then
return "delayed"
end
if rcall("ZSCORE", KEYS[8], ARGV[1]) ~= false then
return "prioritized"
end
-- Includes
--- @include "includes/checkItemInList"
local active_items = rcall("LRANGE", KEYS[4] , 0, -1)
if checkItemInList(active_items, ARGV[1]) ~= nil then
return "active"
end
local wait_items = rcall("LRANGE", KEYS[5] , 0, -1)
if checkItemInList(wait_items, ARGV[1]) ~= nil then
return "waiting"
end
local paused_items = rcall("LRANGE", KEYS[6] , 0, -1)
if checkItemInList(paused_items, ARGV[1]) ~= nil then
return "waiting"
end
if rcall("ZSCORE", KEYS[7], ARGV[1]) ~= false then
return "waiting-children"
end
return "unknown"
+58
View File
@@ -0,0 +1,58 @@
--[[
Get a job state
Input:
KEYS[1] 'completed' key,
KEYS[2] 'failed' key
KEYS[3] 'delayed' key
KEYS[4] 'active' key
KEYS[5] 'wait' key
KEYS[6] 'paused' key
KEYS[7] 'waiting-children' key
KEYS[8] 'prioritized' key
ARGV[1] job id
Output:
'completed'
'failed'
'delayed'
'active'
'waiting'
'waiting-children'
'unknown'
]]
local rcall = redis.call
if rcall("ZSCORE", KEYS[1], ARGV[1]) ~= false then
return "completed"
end
if rcall("ZSCORE", KEYS[2], ARGV[1]) ~= false then
return "failed"
end
if rcall("ZSCORE", KEYS[3], ARGV[1]) ~= false then
return "delayed"
end
if rcall("ZSCORE", KEYS[8], ARGV[1]) ~= false then
return "prioritized"
end
if rcall("LPOS", KEYS[4] , ARGV[1]) ~= false then
return "active"
end
if rcall("LPOS", KEYS[5] , ARGV[1]) ~= false then
return "waiting"
end
if rcall("LPOS", KEYS[6] , ARGV[1]) ~= false then
return "waiting"
end
if rcall("ZSCORE", KEYS[7] , ARGV[1]) ~= false then
return "waiting-children"
end
return "unknown"
@@ -0,0 +1,27 @@
--[[
Add delay marker if needed.
]]
-- Includes
--- @include "getNextDelayedTimestamp"
local function addDelayMarkerIfNeeded(targetKey, delayedKey)
local waitLen = rcall("LLEN", targetKey)
if waitLen <= 1 then
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
if nextTimestamp ~= nil then
-- Check if there is already a marker with older timestamp
-- if there is, we need to replace it.
if waitLen == 1 then
local marker = rcall("LINDEX", targetKey, 0)
local oldTimestamp = tonumber(marker:sub(3))
if oldTimestamp and oldTimestamp > nextTimestamp then
rcall("LSET", targetKey, 0, "0:" .. nextTimestamp)
end
else
-- if there is no marker, then we need to add one
rcall("LPUSH", targetKey, "0:" .. nextTimestamp)
end
end
end
end
+15
View File
@@ -0,0 +1,15 @@
--[[
Function to add job considering priority.
]]
-- Includes
--- @include "addPriorityMarkerIfNeeded"
--- @include "getPriorityScore"
local function addJobWithPriority(waitKey, prioritizedKey, priority, paused, jobId, priorityCounterKey)
local score = getPriorityScore(priority, priorityCounterKey)
rcall("ZADD", prioritizedKey, score, jobId)
if not paused then
addPriorityMarkerIfNeeded(waitKey)
end
end
@@ -0,0 +1,12 @@
--[[
Function priority marker to wait if needed
in order to wake up our workers and to respect priority
order as much as possible
]]
local function addPriorityMarkerIfNeeded(waitKey)
local waitLen = rcall("LLEN", waitKey)
if waitLen == 0 then
rcall("LPUSH", waitKey, "0:0")
end
end
+18
View File
@@ -0,0 +1,18 @@
--[[
Function to loop in batches.
Just a bit of warning, some commands as ZREM
could receive a maximum of 7000 parameters per call.
]]
local function batches(n, batchSize)
local i = 0
return function()
local from = i * batchSize + 1
i = i + 1
if (from <= n) then
local to = math.min(from + batchSize - 1, n)
return from, to
end
end
end
+12
View File
@@ -0,0 +1,12 @@
--[[
Functions to check if a item belongs to a list.
]]
local function checkItemInList(list, item)
for _, v in pairs(list) do
if v == item then
return 1
end
end
return nil
end
+139
View File
@@ -0,0 +1,139 @@
--[[
Move stalled jobs to wait.
Input:
stalledKey 'stalled' (SET)
waitKey 'wait', (LIST)
activeKey 'active', (LIST)
failedKey 'failed', (ZSET)
stalledCheckKey 'stalled-check', (KEY)
metaKey 'meta', (KEY)
pausedKey 'paused', (LIST)
eventStreamKey 'event stream' (STREAM)
maxStalledJobCount Max stalled job count
queueKeyPrefix queue.toKey('')
timestamp timestamp
maxCheckTime max check time
Events:
'stalled' with stalled job id.
]]
local rcall = redis.call
-- Includes
--- @include "batches"
--- @include "getTargetQueueList"
--- @include "removeJob"
--- @include "removeJobsByMaxAge"
--- @include "removeJobsByMaxCount"
--- @include "trimEvents"
-- Check if we need to check for stalled jobs now.
local function checkStalledJobs(stalledKey, waitKey, activeKey, failedKey,
stalledCheckKey, metaKey, pausedKey,
eventStreamKey, maxStalledJobCount,
queueKeyPrefix, timestamp, maxCheckTime)
if rcall("EXISTS", stalledCheckKey) == 1 then return {{}, {}} end
rcall("SET", stalledCheckKey, timestamp, "PX", maxCheckTime)
-- Trim events before emiting them to avoid trimming events emitted in this script
trimEvents(metaKey, eventStreamKey)
-- Move all stalled jobs to wait
local stalling = rcall('SMEMBERS', stalledKey)
local stalled = {}
local failed = {}
if (#stalling > 0) then
rcall('DEL', stalledKey)
local MAX_STALLED_JOB_COUNT = tonumber(maxStalledJobCount)
-- Remove from active list
for i, jobId in ipairs(stalling) do
if string.sub(jobId, 1, 2) == "0:" then
-- If the jobId is a delay marker ID we just remove it.
rcall("LREM", activeKey, 1, jobId)
else
local jobKey = queueKeyPrefix .. jobId
-- Check that the lock is also missing, then we can handle this job as really stalled.
if (rcall("EXISTS", jobKey .. ":lock") == 0) then
-- Remove from the active queue.
local removed = rcall("LREM", activeKey, 1, jobId)
if (removed > 0) then
-- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
local stalledCount =
rcall("HINCRBY", jobKey, "stalledCounter", 1)
if (stalledCount > MAX_STALLED_JOB_COUNT) then
local rawOpts = rcall("HGET", jobKey, "opts")
local opts = cjson.decode(rawOpts)
local removeOnFailType = type(opts["removeOnFail"])
rcall("ZADD", failedKey, timestamp, jobId)
local failedReason =
"job stalled more than allowable limit"
rcall("HMSET", jobKey, "failedReason", failedReason,
"finishedOn", timestamp)
rcall("XADD", eventStreamKey, "*", "event",
"failed", "jobId", jobId, 'prev', 'active',
'failedReason', failedReason)
if removeOnFailType == "number" then
removeJobsByMaxCount(opts["removeOnFail"],
failedKey, queueKeyPrefix)
elseif removeOnFailType == "boolean" then
if opts["removeOnFail"] then
removeJob(jobId, false, queueKeyPrefix)
rcall("ZREM", failedKey, jobId)
end
elseif removeOnFailType ~= "nil" then
local maxAge = opts["removeOnFail"]["age"]
local maxCount = opts["removeOnFail"]["count"]
if maxAge ~= nil then
removeJobsByMaxAge(timestamp, maxAge,
failedKey, queueKeyPrefix)
end
if maxCount ~= nil and maxCount > 0 then
removeJobsByMaxCount(maxCount, failedKey,
queueKeyPrefix)
end
end
table.insert(failed, jobId)
else
local target =
getTargetQueueList(metaKey, waitKey, pausedKey)
-- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
rcall("RPUSH", target, jobId)
rcall("XADD", eventStreamKey, "*", "event",
"waiting", "jobId", jobId, 'prev', 'active')
-- Emit the stalled event
rcall("XADD", eventStreamKey, "*", "event",
"stalled", "jobId", jobId)
table.insert(stalled, jobId)
end
end
end
end
end
end
-- Mark potentially stalled jobs
local active = rcall('LRANGE', activeKey, 0, -1)
if (#active > 0) then
for from, to in batches(#active, 7000) do
rcall('SADD', stalledKey, unpack(active, from, to))
end
end
return {failed, stalled}
end
+47
View File
@@ -0,0 +1,47 @@
--[[
Function to clean job list.
Returns jobIds and deleted count number.
]]
-- Includes
--- @include "getTimestamp"
--- @include "removeJob"
local function cleanList(listKey, jobKeyPrefix, rangeStart, rangeEnd,
timestamp, isWaiting)
local jobs = rcall("LRANGE", listKey, rangeStart, rangeEnd)
local deleted = {}
local deletedCount = 0
local jobTS
local deletionMarker = ''
local jobIdsLen = #jobs
for i, job in ipairs(jobs) do
if limit > 0 and deletedCount >= limit then
break
end
local jobKey = jobKeyPrefix .. job
if (isWaiting or rcall("EXISTS", jobKey .. ":lock") == 0) then
-- Find the right timestamp of the job to compare to maxTimestamp:
-- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed
-- * processedOn represents when the job was last attempted, but it doesn't get populated until
-- the job is first tried
-- * timestamp is the original job submission time
-- Fetch all three of these (in that order) and use the first one that is set so that we'll leave jobs
-- that have been active within the grace period:
jobTS = getTimestamp(jobKey, {"finishedOn", "processedOn", "timestamp"})
if (not jobTS or jobTS <= timestamp) then
-- replace the entry with a deletion marker; the actual deletion will
-- occur at the end of the script
rcall("LSET", listKey, rangeEnd - jobIdsLen + i, deletionMarker)
removeJob(job, true, jobKeyPrefix)
deletedCount = deletedCount + 1
table.insert(deleted, job)
end
end
end
rcall("LREM", listKey, 0, deletionMarker)
return {deleted, deletedCount}
end
+45
View File
@@ -0,0 +1,45 @@
--[[
Function to clean job set.
Returns jobIds and deleted count number.
]]
-- Includes
--- @include "batches"
--- @include "getJobsInZset"
--- @include "getTimestamp"
--- @include "removeJob"
local function cleanSet(setKey, jobKeyPrefix, rangeEnd, timestamp, limit, attributes, isFinished)
local jobs = getJobsInZset(setKey, rangeEnd, limit)
local deleted = {}
local deletedCount = 0
local jobTS
for i, job in ipairs(jobs) do
if limit > 0 and deletedCount >= limit then
break
end
local jobKey = jobKeyPrefix .. job
if isFinished then
removeJob(job, true, jobKeyPrefix)
deletedCount = deletedCount + 1
table.insert(deleted, job)
else
-- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed
jobTS = getTimestamp(jobKey, attributes)
if (not jobTS or jobTS <= timestamp) then
removeJob(job, true, jobKeyPrefix)
deletedCount = deletedCount + 1
table.insert(deleted, job)
end
end
end
if(#deleted > 0) then
for from, to in batches(#deleted, 7000) do
rcall("ZREM", setKey, unpack(deleted, from, to))
end
end
return {deleted, deletedCount}
end
+46
View File
@@ -0,0 +1,46 @@
--[[
Functions to collect metrics based on a current and previous count of jobs.
Granualarity is fixed at 1 minute.
]]
--- @include "batches"
local function collectMetrics(metaKey, dataPointsList, maxDataPoints,
timestamp)
-- Increment current count
local count = rcall("HINCRBY", metaKey, "count", 1) - 1
-- Compute how many data points we need to add to the list, N.
local prevTS = rcall("HGET", metaKey, "prevTS")
if not prevTS then
-- If prevTS is nil, set it to the current timestamp
rcall("HSET", metaKey, "prevTS", timestamp, "prevCount", 0)
return
end
local N = math.floor((timestamp - prevTS) / 60000)
if N > 0 then
local delta = count - rcall("HGET", metaKey, "prevCount")
-- If N > 1, add N-1 zeros to the list
if N > 1 then
local points = {}
points[1] = delta
for i = 2, N do
points[i] = 0
end
for from, to in batches(#points, 7000) do
rcall("LPUSH", dataPointsList, unpack(points, from, to))
end
else
-- LPUSH delta to the list
rcall("LPUSH", dataPointsList, delta)
end
-- LTRIM to keep list to its max size
rcall("LTRIM", dataPointsList, 0, maxDataPoints - 1)
-- update prev count with current count
rcall("HSET", metaKey, "prevCount", count, "prevTS", timestamp)
end
end
Loaded 100 of 4962 files, more files were not shown because too many files have changed in this diff. Show more