Initial commit of working RSS Aggregator build
This commit is contained in:
+281
@@ -0,0 +1,281 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.addDelayedJob = void 0;
|
||||
const content = `--[[
|
||||
Adds a delayed job to the queue by doing the following:
|
||||
- Increases the job counter if needed.
|
||||
- Creates a new job key with the job data.
|
||||
- computes timestamp.
|
||||
- adds to delayed zset.
|
||||
- Emits a global event 'delayed' if the job is delayed.
|
||||
Input:
|
||||
KEYS[1] 'wait',
|
||||
KEYS[2] 'paused'
|
||||
KEYS[3] 'meta'
|
||||
KEYS[4] 'id'
|
||||
KEYS[5] 'delayed'
|
||||
KEYS[6] 'completed'
|
||||
KEYS[7] events stream key
|
||||
ARGV[1] msgpacked arguments array
|
||||
[1] key prefix,
|
||||
[2] custom id (use custom instead of one generated automatically)
|
||||
[3] name
|
||||
[4] timestamp
|
||||
[5] parentKey?
|
||||
x [6] waitChildrenKey key.
|
||||
[7] parent dependencies key.
|
||||
[8] parent? {id, queueKey}
|
||||
[9] repeat job key
|
||||
ARGV[2] Json stringified job data
|
||||
ARGV[3] msgpacked options
|
||||
Output:
|
||||
jobId - OK
|
||||
-5 - Missing parent key
|
||||
]]
|
||||
local waitKey = KEYS[1]
|
||||
local pausedKey = KEYS[2]
|
||||
local metaKey = KEYS[3]
|
||||
local idKey = KEYS[4]
|
||||
local delayedKey = KEYS[5]
|
||||
local completedKey = KEYS[6]
|
||||
local eventsKey = KEYS[7]
|
||||
local jobId
|
||||
local jobIdKey
|
||||
local rcall = redis.call
|
||||
local args = cmsgpack.unpack(ARGV[1])
|
||||
local data = ARGV[2]
|
||||
local opts = cmsgpack.unpack(ARGV[3])
|
||||
local parentKey = args[5]
|
||||
local repeatJobKey = args[9]
|
||||
local parent = args[8]
|
||||
local parentData
|
||||
-- Includes
|
||||
--[[
|
||||
Function to store a job
|
||||
]]
|
||||
local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
|
||||
parentKey, parentData, repeatJobKey)
|
||||
local jsonOpts = cjson.encode(opts)
|
||||
local delay = opts['delay'] or 0
|
||||
local priority = opts['priority'] or 0
|
||||
local optionalValues = {}
|
||||
if parentKey ~= nil then
|
||||
table.insert(optionalValues, "parentKey")
|
||||
table.insert(optionalValues, parentKey)
|
||||
table.insert(optionalValues, "parent")
|
||||
table.insert(optionalValues, parentData)
|
||||
end
|
||||
if repeatJobKey ~= nil then
|
||||
table.insert(optionalValues, "rjk")
|
||||
table.insert(optionalValues, repeatJobKey)
|
||||
end
|
||||
rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
|
||||
"timestamp", timestamp, "delay", delay, "priority", priority,
|
||||
unpack(optionalValues))
|
||||
rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
|
||||
return delay, priority
|
||||
end
|
||||
--[[
|
||||
Add delay marker if needed.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Function to return the next delayed job timestamp.
|
||||
]]
|
||||
local function getNextDelayedTimestamp(delayedKey)
|
||||
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
|
||||
if #result then
|
||||
local nextTimestamp = tonumber(result[2])
|
||||
if (nextTimestamp ~= nil) then
|
||||
nextTimestamp = nextTimestamp / 0x1000
|
||||
end
|
||||
return nextTimestamp
|
||||
end
|
||||
end
|
||||
local function addDelayMarkerIfNeeded(targetKey, delayedKey)
|
||||
local waitLen = rcall("LLEN", targetKey)
|
||||
if waitLen <= 1 then
|
||||
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
|
||||
if nextTimestamp ~= nil then
|
||||
-- Check if there is already a marker with older timestamp
|
||||
-- if there is, we need to replace it.
|
||||
if waitLen == 1 then
|
||||
local marker = rcall("LINDEX", targetKey, 0)
|
||||
local oldTimestamp = tonumber(marker:sub(3))
|
||||
if oldTimestamp and oldTimestamp > nextTimestamp then
|
||||
rcall("LSET", targetKey, 0, "0:" .. nextTimestamp)
|
||||
end
|
||||
else
|
||||
-- if there is no marker, then we need to add one
|
||||
rcall("LPUSH", targetKey, "0:" .. nextTimestamp)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Function to check for the meta.paused key to decide if we are paused or not
|
||||
(since an empty list and !EXISTS are not really the same).
|
||||
]]
|
||||
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
|
||||
if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
|
||||
return waitKey, false
|
||||
else
|
||||
return pausedKey, true
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Validate and move or add dependencies to parent.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Validate and move parent to active if needed.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Function to add job considering priority.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Function priority marker to wait if needed
|
||||
in order to wake up our workers and to respect priority
|
||||
order as much as possible
|
||||
]]
|
||||
local function addPriorityMarkerIfNeeded(waitKey)
|
||||
local waitLen = rcall("LLEN", waitKey)
|
||||
if waitLen == 0 then
|
||||
rcall("LPUSH", waitKey, "0:0")
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Function to get priority score.
|
||||
]]
|
||||
local function getPriorityScore(priority, priorityCounterKey)
|
||||
local prioCounter = rcall("INCR", priorityCounterKey)
|
||||
return priority * 0x100000000 + prioCounter % 0x100000000
|
||||
end
|
||||
local function addJobWithPriority(waitKey, prioritizedKey, priority, paused, jobId, priorityCounterKey)
|
||||
local score = getPriorityScore(priority, priorityCounterKey)
|
||||
rcall("ZADD", prioritizedKey, score, jobId)
|
||||
if not paused then
|
||||
addPriorityMarkerIfNeeded(waitKey)
|
||||
end
|
||||
end
|
||||
local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
|
||||
local isParentActive = rcall("ZSCORE", parentQueueKey .. ":waiting-children", parentId)
|
||||
if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then
|
||||
rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
|
||||
local parentWaitKey = parentQueueKey .. ":wait"
|
||||
local parentTarget, paused = getTargetQueueList(parentQueueKey .. ":meta", parentWaitKey,
|
||||
parentQueueKey .. ":paused")
|
||||
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
||||
local priority = tonumber(jobAttributes[1]) or 0
|
||||
local delay = tonumber(jobAttributes[2]) or 0
|
||||
if delay > 0 then
|
||||
local delayedTimestamp = tonumber(timestamp) + delay
|
||||
local score = delayedTimestamp * 0x1000
|
||||
local parentDelayedKey = parentQueueKey .. ":delayed"
|
||||
rcall("ZADD", parentDelayedKey, score, parentId)
|
||||
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId,
|
||||
"delay", delayedTimestamp)
|
||||
addDelayMarkerIfNeeded(parentTarget, parentDelayedKey)
|
||||
else
|
||||
if priority == 0 then
|
||||
rcall("RPUSH", parentTarget, parentId)
|
||||
else
|
||||
addJobWithPriority(parentWaitKey, parentQueueKey .. ":prioritized", priority, paused,
|
||||
parentId, parentQueueKey .. ":pc")
|
||||
end
|
||||
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId,
|
||||
"prev", "waiting-children")
|
||||
end
|
||||
end
|
||||
end
|
||||
local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
|
||||
parentId, jobIdKey, returnvalue, timestamp )
|
||||
local processedSet = parentKey .. ":processed"
|
||||
rcall("HSET", processedSet, jobIdKey, returnvalue)
|
||||
moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
|
||||
end
|
||||
--[[
|
||||
This function is used to update the parent's dependencies if the job
|
||||
is already completed and about to be ignored. The parent must get its
|
||||
dependencies updated to avoid the parent job being stuck forever in
|
||||
the waiting-children state.
|
||||
]]
|
||||
local function updateExistingJobsParent(parentKey, parent, parentData,
|
||||
parentDependenciesKey, completedKey,
|
||||
jobIdKey, jobId, timestamp)
|
||||
if parentKey ~= nil then
|
||||
if rcall("ZSCORE", completedKey, jobId) ~= false then
|
||||
local returnvalue = rcall("HGET", jobIdKey, "returnvalue")
|
||||
updateParentDepsIfNeeded(parentKey, parent['queueKey'],
|
||||
parentDependenciesKey, parent['id'],
|
||||
jobIdKey, returnvalue, timestamp)
|
||||
else
|
||||
if parentDependenciesKey ~= nil then
|
||||
rcall("SADD", parentDependenciesKey, jobIdKey)
|
||||
end
|
||||
end
|
||||
rcall("HMSET", jobIdKey, "parentKey", parentKey, "parent", parentData)
|
||||
end
|
||||
end
|
||||
local function getOrSetMaxEvents(metaKey)
|
||||
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
||||
if not maxEvents then
|
||||
maxEvents = 10000
|
||||
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
|
||||
end
|
||||
return maxEvents
|
||||
end
|
||||
if parentKey ~= nil then
|
||||
if rcall("EXISTS", parentKey) ~= 1 then return -5 end
|
||||
parentData = cjson.encode(parent)
|
||||
end
|
||||
local jobCounter = rcall("INCR", idKey)
|
||||
local maxEvents = getOrSetMaxEvents(metaKey)
|
||||
local parentDependenciesKey = args[7]
|
||||
local timestamp = args[4]
|
||||
if args[2] == "" then
|
||||
jobId = jobCounter
|
||||
jobIdKey = args[1] .. jobId
|
||||
else
|
||||
-- Refactor to: handleDuplicateJob.lua
|
||||
jobId = args[2]
|
||||
jobIdKey = args[1] .. jobId
|
||||
if rcall("EXISTS", jobIdKey) == 1 then
|
||||
updateExistingJobsParent(parentKey, parent, parentData,
|
||||
parentDependenciesKey, completedKey, jobIdKey,
|
||||
jobId, timestamp)
|
||||
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
|
||||
"duplicated", "jobId", jobId)
|
||||
return jobId .. "" -- convert to string
|
||||
end
|
||||
end
|
||||
-- Store the job.
|
||||
local delay, priority = storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2],
|
||||
opts, timestamp, parentKey, parentData,
|
||||
repeatJobKey)
|
||||
-- Compute delayed timestamp and the score.
|
||||
local delayedTimestamp = (delay > 0 and (timestamp + delay)) or 0
|
||||
local score = delayedTimestamp * 0x1000 + bit.band(jobCounter, 0xfff)
|
||||
rcall("ZADD", delayedKey, score, jobId)
|
||||
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
|
||||
"jobId", jobId, "delay", delayedTimestamp)
|
||||
-- If wait list is empty, and this delayed job is the next one to be processed,
|
||||
-- then we need to signal the workers by adding a dummy job (jobId 0:delay) to the wait list.
|
||||
local target = getTargetQueueList(metaKey, KEYS[1], KEYS[2])
|
||||
addDelayMarkerIfNeeded(target, delayedKey)
|
||||
-- Check if this job is a child of another job, if so add it to the parents dependencies
|
||||
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
|
||||
-- fail in this case.
|
||||
if parentDependenciesKey ~= nil then
|
||||
rcall("SADD", parentDependenciesKey, jobIdKey)
|
||||
end
|
||||
return jobId .. "" -- convert to string
|
||||
`;
|
||||
exports.addDelayedJob = {
|
||||
name: 'addDelayedJob',
|
||||
content,
|
||||
keys: 7,
|
||||
};
|
||||
//# sourceMappingURL=addDelayedJob-7.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"addDelayedJob-7.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-7.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+Qf,CAAC;AACW,QAAA,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
||||
+265
@@ -0,0 +1,265 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.addParentJob = void 0;
|
||||
const content = `--[[
|
||||
Adds a parent job to the queue by doing the following:
|
||||
- Increases the job counter if needed.
|
||||
- Creates a new job key with the job data.
|
||||
- adds the job to the waiting-children zset
|
||||
Input:
|
||||
KEYS[1] 'meta'
|
||||
KEYS[2] 'id'
|
||||
KEYS[3] 'completed'
|
||||
KEYS[4] events stream key
|
||||
ARGV[1] msgpacked arguments array
|
||||
[1] key prefix,
|
||||
[2] custom id (will not generate one automatically)
|
||||
[3] name
|
||||
[4] timestamp
|
||||
[5] parentKey?
|
||||
[6] waitChildrenKey key.
|
||||
[7] parent dependencies key.
|
||||
[8] parent? {id, queueKey}
|
||||
[9] repeat job key
|
||||
ARGV[2] Json stringified job data
|
||||
ARGV[3] msgpacked options
|
||||
Output:
|
||||
jobId - OK
|
||||
-5 - Missing parent key
|
||||
]]
|
||||
local metaKey = KEYS[1]
|
||||
local idKey = KEYS[2]
|
||||
local completedKey = KEYS[3]
|
||||
local eventsKey = KEYS[4]
|
||||
local jobId
|
||||
local jobIdKey
|
||||
local rcall = redis.call
|
||||
local args = cmsgpack.unpack(ARGV[1])
|
||||
local data = ARGV[2]
|
||||
local opts = cmsgpack.unpack(ARGV[3])
|
||||
local parentKey = args[5]
|
||||
local repeatJobKey = args[9]
|
||||
local parent = args[8]
|
||||
local parentData
|
||||
-- Includes
|
||||
--[[
|
||||
Function to store a job
|
||||
]]
|
||||
local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
|
||||
parentKey, parentData, repeatJobKey)
|
||||
local jsonOpts = cjson.encode(opts)
|
||||
local delay = opts['delay'] or 0
|
||||
local priority = opts['priority'] or 0
|
||||
local optionalValues = {}
|
||||
if parentKey ~= nil then
|
||||
table.insert(optionalValues, "parentKey")
|
||||
table.insert(optionalValues, parentKey)
|
||||
table.insert(optionalValues, "parent")
|
||||
table.insert(optionalValues, parentData)
|
||||
end
|
||||
if repeatJobKey ~= nil then
|
||||
table.insert(optionalValues, "rjk")
|
||||
table.insert(optionalValues, repeatJobKey)
|
||||
end
|
||||
rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
|
||||
"timestamp", timestamp, "delay", delay, "priority", priority,
|
||||
unpack(optionalValues))
|
||||
rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
|
||||
return delay, priority
|
||||
end
|
||||
--[[
|
||||
Validate and move or add dependencies to parent.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Validate and move parent to active if needed.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Add delay marker if needed.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Function to return the next delayed job timestamp.
|
||||
]]
|
||||
local function getNextDelayedTimestamp(delayedKey)
|
||||
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
|
||||
if #result then
|
||||
local nextTimestamp = tonumber(result[2])
|
||||
if (nextTimestamp ~= nil) then
|
||||
nextTimestamp = nextTimestamp / 0x1000
|
||||
end
|
||||
return nextTimestamp
|
||||
end
|
||||
end
|
||||
local function addDelayMarkerIfNeeded(targetKey, delayedKey)
|
||||
local waitLen = rcall("LLEN", targetKey)
|
||||
if waitLen <= 1 then
|
||||
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
|
||||
if nextTimestamp ~= nil then
|
||||
-- Check if there is already a marker with older timestamp
|
||||
-- if there is, we need to replace it.
|
||||
if waitLen == 1 then
|
||||
local marker = rcall("LINDEX", targetKey, 0)
|
||||
local oldTimestamp = tonumber(marker:sub(3))
|
||||
if oldTimestamp and oldTimestamp > nextTimestamp then
|
||||
rcall("LSET", targetKey, 0, "0:" .. nextTimestamp)
|
||||
end
|
||||
else
|
||||
-- if there is no marker, then we need to add one
|
||||
rcall("LPUSH", targetKey, "0:" .. nextTimestamp)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Function to add job considering priority.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Function priority marker to wait if needed
|
||||
in order to wake up our workers and to respect priority
|
||||
order as much as possible
|
||||
]]
|
||||
local function addPriorityMarkerIfNeeded(waitKey)
|
||||
local waitLen = rcall("LLEN", waitKey)
|
||||
if waitLen == 0 then
|
||||
rcall("LPUSH", waitKey, "0:0")
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Function to get priority score.
|
||||
]]
|
||||
local function getPriorityScore(priority, priorityCounterKey)
|
||||
local prioCounter = rcall("INCR", priorityCounterKey)
|
||||
return priority * 0x100000000 + prioCounter % 0x100000000
|
||||
end
|
||||
local function addJobWithPriority(waitKey, prioritizedKey, priority, paused, jobId, priorityCounterKey)
|
||||
local score = getPriorityScore(priority, priorityCounterKey)
|
||||
rcall("ZADD", prioritizedKey, score, jobId)
|
||||
if not paused then
|
||||
addPriorityMarkerIfNeeded(waitKey)
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Function to check for the meta.paused key to decide if we are paused or not
|
||||
(since an empty list and !EXISTS are not really the same).
|
||||
]]
|
||||
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
|
||||
if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
|
||||
return waitKey, false
|
||||
else
|
||||
return pausedKey, true
|
||||
end
|
||||
end
|
||||
local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
|
||||
local isParentActive = rcall("ZSCORE", parentQueueKey .. ":waiting-children", parentId)
|
||||
if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then
|
||||
rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
|
||||
local parentWaitKey = parentQueueKey .. ":wait"
|
||||
local parentTarget, paused = getTargetQueueList(parentQueueKey .. ":meta", parentWaitKey,
|
||||
parentQueueKey .. ":paused")
|
||||
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
||||
local priority = tonumber(jobAttributes[1]) or 0
|
||||
local delay = tonumber(jobAttributes[2]) or 0
|
||||
if delay > 0 then
|
||||
local delayedTimestamp = tonumber(timestamp) + delay
|
||||
local score = delayedTimestamp * 0x1000
|
||||
local parentDelayedKey = parentQueueKey .. ":delayed"
|
||||
rcall("ZADD", parentDelayedKey, score, parentId)
|
||||
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId,
|
||||
"delay", delayedTimestamp)
|
||||
addDelayMarkerIfNeeded(parentTarget, parentDelayedKey)
|
||||
else
|
||||
if priority == 0 then
|
||||
rcall("RPUSH", parentTarget, parentId)
|
||||
else
|
||||
addJobWithPriority(parentWaitKey, parentQueueKey .. ":prioritized", priority, paused,
|
||||
parentId, parentQueueKey .. ":pc")
|
||||
end
|
||||
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId,
|
||||
"prev", "waiting-children")
|
||||
end
|
||||
end
|
||||
end
|
||||
local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
|
||||
parentId, jobIdKey, returnvalue, timestamp )
|
||||
local processedSet = parentKey .. ":processed"
|
||||
rcall("HSET", processedSet, jobIdKey, returnvalue)
|
||||
moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
|
||||
end
|
||||
--[[
|
||||
This function is used to update the parent's dependencies if the job
|
||||
is already completed and about to be ignored. The parent must get its
|
||||
dependencies updated to avoid the parent job being stuck forever in
|
||||
the waiting-children state.
|
||||
]]
|
||||
local function updateExistingJobsParent(parentKey, parent, parentData,
|
||||
parentDependenciesKey, completedKey,
|
||||
jobIdKey, jobId, timestamp)
|
||||
if parentKey ~= nil then
|
||||
if rcall("ZSCORE", completedKey, jobId) ~= false then
|
||||
local returnvalue = rcall("HGET", jobIdKey, "returnvalue")
|
||||
updateParentDepsIfNeeded(parentKey, parent['queueKey'],
|
||||
parentDependenciesKey, parent['id'],
|
||||
jobIdKey, returnvalue, timestamp)
|
||||
else
|
||||
if parentDependenciesKey ~= nil then
|
||||
rcall("SADD", parentDependenciesKey, jobIdKey)
|
||||
end
|
||||
end
|
||||
rcall("HMSET", jobIdKey, "parentKey", parentKey, "parent", parentData)
|
||||
end
|
||||
end
|
||||
local function getOrSetMaxEvents(metaKey)
|
||||
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
||||
if not maxEvents then
|
||||
maxEvents = 10000
|
||||
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
|
||||
end
|
||||
return maxEvents
|
||||
end
|
||||
if parentKey ~= nil then
|
||||
if rcall("EXISTS", parentKey) ~= 1 then return -5 end
|
||||
parentData = cjson.encode(parent)
|
||||
end
|
||||
local jobCounter = rcall("INCR", idKey)
|
||||
local maxEvents = getOrSetMaxEvents(metaKey)
|
||||
local parentDependenciesKey = args[7]
|
||||
local timestamp = args[4]
|
||||
if args[2] == "" then
|
||||
jobId = jobCounter
|
||||
jobIdKey = args[1] .. jobId
|
||||
else
|
||||
jobId = args[2]
|
||||
jobIdKey = args[1] .. jobId
|
||||
if rcall("EXISTS", jobIdKey) == 1 then
|
||||
updateExistingJobsParent(parentKey, parent, parentData,
|
||||
parentDependenciesKey, completedKey, jobIdKey,
|
||||
jobId, timestamp)
|
||||
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
|
||||
"duplicated", "jobId", jobId)
|
||||
return jobId .. "" -- convert to string
|
||||
end
|
||||
end
|
||||
-- Store the job.
|
||||
storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2], opts, timestamp,
|
||||
parentKey, parentData, repeatJobKey)
|
||||
local waitChildrenKey = args[6]
|
||||
rcall("ZADD", waitChildrenKey, timestamp, jobId)
|
||||
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
|
||||
"waiting-children", "jobId", jobId)
|
||||
-- Check if this job is a child of another job, if so add it to the parents dependencies
|
||||
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
|
||||
-- fail in this case.
|
||||
if parentDependenciesKey ~= nil then
|
||||
rcall("SADD", parentDependenciesKey, jobIdKey)
|
||||
end
|
||||
return jobId .. "" -- convert to string
|
||||
`;
|
||||
exports.addParentJob = {
|
||||
name: 'addParentJob',
|
||||
content,
|
||||
keys: 4,
|
||||
};
|
||||
//# sourceMappingURL=addParentJob-4.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"addParentJob-4.js","sourceRoot":"","sources":["../../../src/scripts/addParentJob-4.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+Pf,CAAC;AACW,QAAA,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
||||
+276
@@ -0,0 +1,276 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.addPrioritizedJob = void 0;
|
||||
const content = `--[[
|
||||
Adds a priotitized job to the queue by doing the following:
|
||||
- Increases the job counter if needed.
|
||||
- Creates a new job key with the job data.
|
||||
- Adds the job to the "added" list so that workers gets notified.
|
||||
Input:
|
||||
KEYS[1] 'wait',
|
||||
KEYS[2] 'paused'
|
||||
KEYS[3] 'meta'
|
||||
KEYS[4] 'id'
|
||||
KEYS[5] 'prioritized'
|
||||
KEYS[6] 'completed'
|
||||
KEYS[7] events stream key
|
||||
KEYS[8] 'pc' priority counter
|
||||
ARGV[1] msgpacked arguments array
|
||||
[1] key prefix,
|
||||
[2] custom id (will not generate one automatically)
|
||||
[3] name
|
||||
[4] timestamp
|
||||
[5] parentKey?
|
||||
[6] waitChildrenKey key.
|
||||
[7] parent dependencies key.
|
||||
[8] parent? {id, queueKey}
|
||||
[9] repeat job key
|
||||
ARGV[2] Json stringified job data
|
||||
ARGV[3] msgpacked options
|
||||
Output:
|
||||
jobId - OK
|
||||
-5 - Missing parent key
|
||||
]]
|
||||
local waitKey = KEYS[1]
|
||||
local pausedKey = KEYS[2]
|
||||
local metaKey = KEYS[3]
|
||||
local idKey = KEYS[4]
|
||||
local priorityKey = KEYS[5]
|
||||
local completedKey = KEYS[6]
|
||||
local eventsKey = KEYS[7]
|
||||
local priorityCounterKey = KEYS[8]
|
||||
local jobId
|
||||
local jobIdKey
|
||||
local rcall = redis.call
|
||||
local args = cmsgpack.unpack(ARGV[1])
|
||||
local data = ARGV[2]
|
||||
local opts = cmsgpack.unpack(ARGV[3])
|
||||
local parentKey = args[5]
|
||||
local repeatJobKey = args[9]
|
||||
local parent = args[8]
|
||||
local parentData
|
||||
-- Includes
|
||||
--[[
|
||||
Function to store a job
|
||||
]]
|
||||
local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
|
||||
parentKey, parentData, repeatJobKey)
|
||||
local jsonOpts = cjson.encode(opts)
|
||||
local delay = opts['delay'] or 0
|
||||
local priority = opts['priority'] or 0
|
||||
local optionalValues = {}
|
||||
if parentKey ~= nil then
|
||||
table.insert(optionalValues, "parentKey")
|
||||
table.insert(optionalValues, parentKey)
|
||||
table.insert(optionalValues, "parent")
|
||||
table.insert(optionalValues, parentData)
|
||||
end
|
||||
if repeatJobKey ~= nil then
|
||||
table.insert(optionalValues, "rjk")
|
||||
table.insert(optionalValues, repeatJobKey)
|
||||
end
|
||||
rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
|
||||
"timestamp", timestamp, "delay", delay, "priority", priority,
|
||||
unpack(optionalValues))
|
||||
rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
|
||||
return delay, priority
|
||||
end
|
||||
--[[
|
||||
Function to add job considering priority.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Function priority marker to wait if needed
|
||||
in order to wake up our workers and to respect priority
|
||||
order as much as possible
|
||||
]]
|
||||
local function addPriorityMarkerIfNeeded(waitKey)
|
||||
local waitLen = rcall("LLEN", waitKey)
|
||||
if waitLen == 0 then
|
||||
rcall("LPUSH", waitKey, "0:0")
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Function to get priority score.
|
||||
]]
|
||||
local function getPriorityScore(priority, priorityCounterKey)
|
||||
local prioCounter = rcall("INCR", priorityCounterKey)
|
||||
return priority * 0x100000000 + prioCounter % 0x100000000
|
||||
end
|
||||
local function addJobWithPriority(waitKey, prioritizedKey, priority, paused, jobId, priorityCounterKey)
|
||||
local score = getPriorityScore(priority, priorityCounterKey)
|
||||
rcall("ZADD", prioritizedKey, score, jobId)
|
||||
if not paused then
|
||||
addPriorityMarkerIfNeeded(waitKey)
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Function to check for the meta.paused key to decide if we are paused or not
|
||||
(since an empty list and !EXISTS are not really the same).
|
||||
]]
|
||||
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
|
||||
if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
|
||||
return waitKey, false
|
||||
else
|
||||
return pausedKey, true
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Validate and move or add dependencies to parent.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Validate and move parent to active if needed.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Add delay marker if needed.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Function to return the next delayed job timestamp.
|
||||
]]
|
||||
local function getNextDelayedTimestamp(delayedKey)
|
||||
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
|
||||
if #result then
|
||||
local nextTimestamp = tonumber(result[2])
|
||||
if (nextTimestamp ~= nil) then
|
||||
nextTimestamp = nextTimestamp / 0x1000
|
||||
end
|
||||
return nextTimestamp
|
||||
end
|
||||
end
|
||||
local function addDelayMarkerIfNeeded(targetKey, delayedKey)
|
||||
local waitLen = rcall("LLEN", targetKey)
|
||||
if waitLen <= 1 then
|
||||
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
|
||||
if nextTimestamp ~= nil then
|
||||
-- Check if there is already a marker with older timestamp
|
||||
-- if there is, we need to replace it.
|
||||
if waitLen == 1 then
|
||||
local marker = rcall("LINDEX", targetKey, 0)
|
||||
local oldTimestamp = tonumber(marker:sub(3))
|
||||
if oldTimestamp and oldTimestamp > nextTimestamp then
|
||||
rcall("LSET", targetKey, 0, "0:" .. nextTimestamp)
|
||||
end
|
||||
else
|
||||
-- if there is no marker, then we need to add one
|
||||
rcall("LPUSH", targetKey, "0:" .. nextTimestamp)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
|
||||
local isParentActive = rcall("ZSCORE", parentQueueKey .. ":waiting-children", parentId)
|
||||
if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then
|
||||
rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
|
||||
local parentWaitKey = parentQueueKey .. ":wait"
|
||||
local parentTarget, paused = getTargetQueueList(parentQueueKey .. ":meta", parentWaitKey,
|
||||
parentQueueKey .. ":paused")
|
||||
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
||||
local priority = tonumber(jobAttributes[1]) or 0
|
||||
local delay = tonumber(jobAttributes[2]) or 0
|
||||
if delay > 0 then
|
||||
local delayedTimestamp = tonumber(timestamp) + delay
|
||||
local score = delayedTimestamp * 0x1000
|
||||
local parentDelayedKey = parentQueueKey .. ":delayed"
|
||||
rcall("ZADD", parentDelayedKey, score, parentId)
|
||||
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId,
|
||||
"delay", delayedTimestamp)
|
||||
addDelayMarkerIfNeeded(parentTarget, parentDelayedKey)
|
||||
else
|
||||
if priority == 0 then
|
||||
rcall("RPUSH", parentTarget, parentId)
|
||||
else
|
||||
addJobWithPriority(parentWaitKey, parentQueueKey .. ":prioritized", priority, paused,
|
||||
parentId, parentQueueKey .. ":pc")
|
||||
end
|
||||
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId,
|
||||
"prev", "waiting-children")
|
||||
end
|
||||
end
|
||||
end
|
||||
local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
|
||||
parentId, jobIdKey, returnvalue, timestamp )
|
||||
local processedSet = parentKey .. ":processed"
|
||||
rcall("HSET", processedSet, jobIdKey, returnvalue)
|
||||
moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
|
||||
end
|
||||
--[[
|
||||
This function is used to update the parent's dependencies if the job
|
||||
is already completed and about to be ignored. The parent must get its
|
||||
dependencies updated to avoid the parent job being stuck forever in
|
||||
the waiting-children state.
|
||||
]]
|
||||
local function updateExistingJobsParent(parentKey, parent, parentData,
|
||||
parentDependenciesKey, completedKey,
|
||||
jobIdKey, jobId, timestamp)
|
||||
if parentKey ~= nil then
|
||||
if rcall("ZSCORE", completedKey, jobId) ~= false then
|
||||
local returnvalue = rcall("HGET", jobIdKey, "returnvalue")
|
||||
updateParentDepsIfNeeded(parentKey, parent['queueKey'],
|
||||
parentDependenciesKey, parent['id'],
|
||||
jobIdKey, returnvalue, timestamp)
|
||||
else
|
||||
if parentDependenciesKey ~= nil then
|
||||
rcall("SADD", parentDependenciesKey, jobIdKey)
|
||||
end
|
||||
end
|
||||
rcall("HMSET", jobIdKey, "parentKey", parentKey, "parent", parentData)
|
||||
end
|
||||
end
|
||||
local function getOrSetMaxEvents(metaKey)
|
||||
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
||||
if not maxEvents then
|
||||
maxEvents = 10000
|
||||
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
|
||||
end
|
||||
return maxEvents
|
||||
end
|
||||
if parentKey ~= nil then
|
||||
if rcall("EXISTS", parentKey) ~= 1 then return -5 end
|
||||
parentData = cjson.encode(parent)
|
||||
end
|
||||
local jobCounter = rcall("INCR", idKey)
|
||||
local maxEvents = getOrSetMaxEvents(metaKey)
|
||||
local parentDependenciesKey = args[7]
|
||||
local timestamp = args[4]
|
||||
if args[2] == "" then
|
||||
jobId = jobCounter
|
||||
jobIdKey = args[1] .. jobId
|
||||
else
|
||||
jobId = args[2]
|
||||
jobIdKey = args[1] .. jobId
|
||||
if rcall("EXISTS", jobIdKey) == 1 then
|
||||
updateExistingJobsParent(parentKey, parent, parentData,
|
||||
parentDependenciesKey, completedKey, jobIdKey,
|
||||
jobId, timestamp)
|
||||
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
|
||||
"duplicated", "jobId", jobId)
|
||||
return jobId .. "" -- convert to string
|
||||
end
|
||||
end
|
||||
-- Store the job.
|
||||
local delay, priority = storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2],
|
||||
opts, timestamp, parentKey, parentData,
|
||||
repeatJobKey)
|
||||
local target, paused = getTargetQueueList(metaKey, waitKey, pausedKey)
|
||||
addJobWithPriority(waitKey, priorityKey, priority, paused, jobId,
|
||||
priorityCounterKey)
|
||||
-- Emit waiting event
|
||||
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting",
|
||||
"jobId", jobId)
|
||||
-- Check if this job is a child of another job, if so add it to the parents dependencies
|
||||
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
|
||||
-- fail in this case.
|
||||
if parentDependenciesKey ~= nil then
|
||||
rcall("SADD", parentDependenciesKey, jobIdKey)
|
||||
end
|
||||
return jobId .. "" -- convert to string
|
||||
`;
|
||||
exports.addPrioritizedJob = {
|
||||
name: 'addPrioritizedJob',
|
||||
content,
|
||||
keys: 8,
|
||||
};
|
||||
//# sourceMappingURL=addPrioritizedJob-8.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"addPrioritizedJob-8.js","sourceRoot":"","sources":["../../../src/scripts/addPrioritizedJob-8.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0Qf,CAAC;AACW,QAAA,iBAAiB,GAAG;IAC/B,IAAI,EAAE,mBAAmB;IACzB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
||||
+277
@@ -0,0 +1,277 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.addStandardJob = void 0;
|
||||
const content = `--[[
|
||||
Adds a job to the queue by doing the following:
|
||||
- Increases the job counter if needed.
|
||||
- Creates a new job key with the job data.
|
||||
- if delayed:
|
||||
- computes timestamp.
|
||||
- adds to delayed zset.
|
||||
- Emits a global event 'delayed' if the job is delayed.
|
||||
- if not delayed
|
||||
- Adds the jobId to the wait/paused list in one of three ways:
|
||||
- LIFO
|
||||
- FIFO
|
||||
- prioritized.
|
||||
- Adds the job to the "added" list so that workers gets notified.
|
||||
Input:
|
||||
KEYS[1] 'wait',
|
||||
KEYS[2] 'paused'
|
||||
KEYS[3] 'meta'
|
||||
KEYS[4] 'id'
|
||||
KEYS[5] 'completed'
|
||||
KEYS[6] events stream key
|
||||
ARGV[1] msgpacked arguments array
|
||||
[1] key prefix,
|
||||
[2] custom id (will not generate one automatically)
|
||||
[3] name
|
||||
[4] timestamp
|
||||
[5] parentKey?
|
||||
[6] waitChildrenKey key.
|
||||
[7] parent dependencies key.
|
||||
[8] parent? {id, queueKey}
|
||||
[9] repeat job key
|
||||
ARGV[2] Json stringified job data
|
||||
ARGV[3] msgpacked options
|
||||
Output:
|
||||
jobId - OK
|
||||
-5 - Missing parent key
|
||||
]]
|
||||
local eventsKey = KEYS[6]
|
||||
local jobId
|
||||
local jobIdKey
|
||||
local rcall = redis.call
|
||||
local args = cmsgpack.unpack(ARGV[1])
|
||||
local data = ARGV[2]
|
||||
local opts = cmsgpack.unpack(ARGV[3])
|
||||
local parentKey = args[5]
|
||||
local repeatJobKey = args[9]
|
||||
local parent = args[8]
|
||||
local parentData
|
||||
-- Includes
|
||||
--[[
|
||||
Function to store a job
|
||||
]]
|
||||
local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
|
||||
parentKey, parentData, repeatJobKey)
|
||||
local jsonOpts = cjson.encode(opts)
|
||||
local delay = opts['delay'] or 0
|
||||
local priority = opts['priority'] or 0
|
||||
local optionalValues = {}
|
||||
if parentKey ~= nil then
|
||||
table.insert(optionalValues, "parentKey")
|
||||
table.insert(optionalValues, parentKey)
|
||||
table.insert(optionalValues, "parent")
|
||||
table.insert(optionalValues, parentData)
|
||||
end
|
||||
if repeatJobKey ~= nil then
|
||||
table.insert(optionalValues, "rjk")
|
||||
table.insert(optionalValues, repeatJobKey)
|
||||
end
|
||||
rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
|
||||
"timestamp", timestamp, "delay", delay, "priority", priority,
|
||||
unpack(optionalValues))
|
||||
rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
|
||||
return delay, priority
|
||||
end
|
||||
--[[
|
||||
Validate and move or add dependencies to parent.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Validate and move parent to active if needed.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Add delay marker if needed.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Function to return the next delayed job timestamp.
|
||||
]]
|
||||
local function getNextDelayedTimestamp(delayedKey)
|
||||
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
|
||||
if #result then
|
||||
local nextTimestamp = tonumber(result[2])
|
||||
if (nextTimestamp ~= nil) then
|
||||
nextTimestamp = nextTimestamp / 0x1000
|
||||
end
|
||||
return nextTimestamp
|
||||
end
|
||||
end
|
||||
local function addDelayMarkerIfNeeded(targetKey, delayedKey)
|
||||
local waitLen = rcall("LLEN", targetKey)
|
||||
if waitLen <= 1 then
|
||||
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
|
||||
if nextTimestamp ~= nil then
|
||||
-- Check if there is already a marker with older timestamp
|
||||
-- if there is, we need to replace it.
|
||||
if waitLen == 1 then
|
||||
local marker = rcall("LINDEX", targetKey, 0)
|
||||
local oldTimestamp = tonumber(marker:sub(3))
|
||||
if oldTimestamp and oldTimestamp > nextTimestamp then
|
||||
rcall("LSET", targetKey, 0, "0:" .. nextTimestamp)
|
||||
end
|
||||
else
|
||||
-- if there is no marker, then we need to add one
|
||||
rcall("LPUSH", targetKey, "0:" .. nextTimestamp)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Function to add job considering priority.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Function priority marker to wait if needed
|
||||
in order to wake up our workers and to respect priority
|
||||
order as much as possible
|
||||
]]
|
||||
local function addPriorityMarkerIfNeeded(waitKey)
|
||||
local waitLen = rcall("LLEN", waitKey)
|
||||
if waitLen == 0 then
|
||||
rcall("LPUSH", waitKey, "0:0")
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Function to get priority score.
|
||||
]]
|
||||
local function getPriorityScore(priority, priorityCounterKey)
|
||||
local prioCounter = rcall("INCR", priorityCounterKey)
|
||||
return priority * 0x100000000 + prioCounter % 0x100000000
|
||||
end
|
||||
local function addJobWithPriority(waitKey, prioritizedKey, priority, paused, jobId, priorityCounterKey)
|
||||
local score = getPriorityScore(priority, priorityCounterKey)
|
||||
rcall("ZADD", prioritizedKey, score, jobId)
|
||||
if not paused then
|
||||
addPriorityMarkerIfNeeded(waitKey)
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Function to check for the meta.paused key to decide if we are paused or not
|
||||
(since an empty list and !EXISTS are not really the same).
|
||||
]]
|
||||
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
|
||||
if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
|
||||
return waitKey, false
|
||||
else
|
||||
return pausedKey, true
|
||||
end
|
||||
end
|
||||
local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
|
||||
local isParentActive = rcall("ZSCORE", parentQueueKey .. ":waiting-children", parentId)
|
||||
if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then
|
||||
rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
|
||||
local parentWaitKey = parentQueueKey .. ":wait"
|
||||
local parentTarget, paused = getTargetQueueList(parentQueueKey .. ":meta", parentWaitKey,
|
||||
parentQueueKey .. ":paused")
|
||||
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
||||
local priority = tonumber(jobAttributes[1]) or 0
|
||||
local delay = tonumber(jobAttributes[2]) or 0
|
||||
if delay > 0 then
|
||||
local delayedTimestamp = tonumber(timestamp) + delay
|
||||
local score = delayedTimestamp * 0x1000
|
||||
local parentDelayedKey = parentQueueKey .. ":delayed"
|
||||
rcall("ZADD", parentDelayedKey, score, parentId)
|
||||
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId,
|
||||
"delay", delayedTimestamp)
|
||||
addDelayMarkerIfNeeded(parentTarget, parentDelayedKey)
|
||||
else
|
||||
if priority == 0 then
|
||||
rcall("RPUSH", parentTarget, parentId)
|
||||
else
|
||||
addJobWithPriority(parentWaitKey, parentQueueKey .. ":prioritized", priority, paused,
|
||||
parentId, parentQueueKey .. ":pc")
|
||||
end
|
||||
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId,
|
||||
"prev", "waiting-children")
|
||||
end
|
||||
end
|
||||
end
|
||||
local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
|
||||
parentId, jobIdKey, returnvalue, timestamp )
|
||||
local processedSet = parentKey .. ":processed"
|
||||
rcall("HSET", processedSet, jobIdKey, returnvalue)
|
||||
moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
|
||||
end
|
||||
--[[
|
||||
This function is used to update the parent's dependencies if the job
|
||||
is already completed and about to be ignored. The parent must get its
|
||||
dependencies updated to avoid the parent job being stuck forever in
|
||||
the waiting-children state.
|
||||
]]
|
||||
local function updateExistingJobsParent(parentKey, parent, parentData,
|
||||
parentDependenciesKey, completedKey,
|
||||
jobIdKey, jobId, timestamp)
|
||||
if parentKey ~= nil then
|
||||
if rcall("ZSCORE", completedKey, jobId) ~= false then
|
||||
local returnvalue = rcall("HGET", jobIdKey, "returnvalue")
|
||||
updateParentDepsIfNeeded(parentKey, parent['queueKey'],
|
||||
parentDependenciesKey, parent['id'],
|
||||
jobIdKey, returnvalue, timestamp)
|
||||
else
|
||||
if parentDependenciesKey ~= nil then
|
||||
rcall("SADD", parentDependenciesKey, jobIdKey)
|
||||
end
|
||||
end
|
||||
rcall("HMSET", jobIdKey, "parentKey", parentKey, "parent", parentData)
|
||||
end
|
||||
end
|
||||
local function getOrSetMaxEvents(metaKey)
|
||||
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
||||
if not maxEvents then
|
||||
maxEvents = 10000
|
||||
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
|
||||
end
|
||||
return maxEvents
|
||||
end
|
||||
if parentKey ~= nil then
|
||||
if rcall("EXISTS", parentKey) ~= 1 then return -5 end
|
||||
parentData = cjson.encode(parent)
|
||||
end
|
||||
local jobCounter = rcall("INCR", KEYS[4])
|
||||
local metaKey = KEYS[3]
|
||||
local maxEvents = getOrSetMaxEvents(metaKey)
|
||||
local parentDependenciesKey = args[7]
|
||||
local timestamp = args[4]
|
||||
if args[2] == "" then
|
||||
jobId = jobCounter
|
||||
jobIdKey = args[1] .. jobId
|
||||
else
|
||||
jobId = args[2]
|
||||
jobIdKey = args[1] .. jobId
|
||||
if rcall("EXISTS", jobIdKey) == 1 then
|
||||
updateExistingJobsParent(parentKey, parent, parentData,
|
||||
parentDependenciesKey, KEYS[5], jobIdKey,
|
||||
jobId, timestamp)
|
||||
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
|
||||
"duplicated", "jobId", jobId)
|
||||
return jobId .. "" -- convert to string
|
||||
end
|
||||
end
|
||||
-- Store the job.
|
||||
storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2], opts, timestamp,
|
||||
parentKey, parentData, repeatJobKey)
|
||||
local target, paused = getTargetQueueList(metaKey, KEYS[1], KEYS[2])
|
||||
-- LIFO or FIFO
|
||||
local pushCmd = opts['lifo'] and 'RPUSH' or 'LPUSH'
|
||||
rcall(pushCmd, target, jobId)
|
||||
-- Emit waiting event
|
||||
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting",
|
||||
"jobId", jobId)
|
||||
-- Check if this job is a child of another job, if so add it to the parents dependencies
|
||||
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
|
||||
-- fail in this case.
|
||||
if parentDependenciesKey ~= nil then
|
||||
rcall("SADD", parentDependenciesKey, jobIdKey)
|
||||
end
|
||||
return jobId .. "" -- convert to string
|
||||
`;
|
||||
exports.addStandardJob = {
|
||||
name: 'addStandardJob',
|
||||
content,
|
||||
keys: 6,
|
||||
};
|
||||
//# sourceMappingURL=addStandardJob-6.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"addStandardJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addStandardJob-6.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2Qf,CAAC;AACW,QAAA,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.changeDelay = void 0;
|
||||
const content = `--[[
|
||||
Change job delay when it is in delayed set.
|
||||
Input:
|
||||
KEYS[1] delayed key
|
||||
KEYS[2] job key
|
||||
KEYS[3] events stream
|
||||
ARGV[1] delay
|
||||
ARGV[2] delayedTimestamp
|
||||
ARGV[3] the id of the job
|
||||
Output:
|
||||
0 - OK
|
||||
-1 - Missing job.
|
||||
-3 - Job not in delayed set.
|
||||
Events:
|
||||
- delayed key.
|
||||
]]
|
||||
local rcall = redis.call
|
||||
if rcall("EXISTS", KEYS[2]) == 1 then
|
||||
local jobId = ARGV[3]
|
||||
local score = tonumber(ARGV[2])
|
||||
local delayedTimestamp = (score / 0x1000)
|
||||
local numRemovedElements = rcall("ZREM", KEYS[1], jobId)
|
||||
if numRemovedElements < 1 then
|
||||
return -3
|
||||
end
|
||||
rcall("HSET", KEYS[2], "delay", tonumber(ARGV[1]))
|
||||
rcall("ZADD", KEYS[1], score, jobId)
|
||||
rcall("XADD", KEYS[3], "*", "event", "delayed", "jobId", jobId, "delay", delayedTimestamp)
|
||||
return 0
|
||||
else
|
||||
return -1
|
||||
end`;
|
||||
exports.changeDelay = {
|
||||
name: 'changeDelay',
|
||||
content,
|
||||
keys: 3,
|
||||
};
|
||||
//# sourceMappingURL=changeDelay-3.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"changeDelay-3.js","sourceRoot":"","sources":["../../../src/scripts/changeDelay-3.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IA+BZ,CAAC;AACQ,QAAA,WAAW,GAAG;IACzB,IAAI,EAAE,aAAa;IACnB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
||||
+93
@@ -0,0 +1,93 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.changePriority = void 0;
|
||||
const content = `--[[
|
||||
Change job priority
|
||||
Input:
|
||||
KEYS[1] 'wait',
|
||||
KEYS[2] 'paused'
|
||||
KEYS[3] 'meta'
|
||||
KEYS[4] 'prioritized'
|
||||
KEYS[5] 'pc' priority counter
|
||||
ARGV[1] priority value
|
||||
ARGV[2] job key
|
||||
ARGV[3] job id
|
||||
ARGV[4] lifo
|
||||
Output:
|
||||
0 - OK
|
||||
-1 - Missing job
|
||||
]]
|
||||
local jobKey = ARGV[2]
|
||||
local jobId = ARGV[3]
|
||||
local priority = tonumber(ARGV[1])
|
||||
local rcall = redis.call
|
||||
-- Includes
|
||||
--[[
|
||||
Function to add job considering priority.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Function priority marker to wait if needed
|
||||
in order to wake up our workers and to respect priority
|
||||
order as much as possible
|
||||
]]
|
||||
local function addPriorityMarkerIfNeeded(waitKey)
|
||||
local waitLen = rcall("LLEN", waitKey)
|
||||
if waitLen == 0 then
|
||||
rcall("LPUSH", waitKey, "0:0")
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Function to get priority score.
|
||||
]]
|
||||
local function getPriorityScore(priority, priorityCounterKey)
|
||||
local prioCounter = rcall("INCR", priorityCounterKey)
|
||||
return priority * 0x100000000 + prioCounter % 0x100000000
|
||||
end
|
||||
local function addJobWithPriority(waitKey, prioritizedKey, priority, paused, jobId, priorityCounterKey)
|
||||
local score = getPriorityScore(priority, priorityCounterKey)
|
||||
rcall("ZADD", prioritizedKey, score, jobId)
|
||||
if not paused then
|
||||
addPriorityMarkerIfNeeded(waitKey)
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Function to check for the meta.paused key to decide if we are paused or not
|
||||
(since an empty list and !EXISTS are not really the same).
|
||||
]]
|
||||
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
|
||||
if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
|
||||
return waitKey, false
|
||||
else
|
||||
return pausedKey, true
|
||||
end
|
||||
end
|
||||
if rcall("EXISTS", jobKey) == 1 then
|
||||
local target, paused = getTargetQueueList(KEYS[3], KEYS[1], KEYS[2])
|
||||
if rcall("ZREM", KEYS[4], jobId) > 0 then
|
||||
addJobWithPriority(KEYS[1], KEYS[4], priority, paused, jobId, KEYS[5])
|
||||
else
|
||||
local numRemovedElements = rcall("LREM", target, -1, jobId)
|
||||
if numRemovedElements > 0 then
|
||||
-- Standard or priority add
|
||||
if priority == 0 then
|
||||
-- LIFO or FIFO
|
||||
local pushCmd = ARGV[4] == '1' and 'RPUSH' or 'LPUSH';
|
||||
rcall(pushCmd, target, jobId)
|
||||
else
|
||||
addJobWithPriority(KEYS[1], KEYS[4], priority, paused, jobId, KEYS[5])
|
||||
end
|
||||
end
|
||||
end
|
||||
rcall("HSET", jobKey, "priority", priority)
|
||||
return 0
|
||||
else
|
||||
return -1
|
||||
end
|
||||
`;
|
||||
exports.changePriority = {
|
||||
name: 'changePriority',
|
||||
content,
|
||||
keys: 5,
|
||||
};
|
||||
//# sourceMappingURL=changePriority-5.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"changePriority-5.js","sourceRoot":"","sources":["../../../src/scripts/changePriority-5.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAmFf,CAAC;AACW,QAAA,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
||||
+271
@@ -0,0 +1,271 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.cleanJobsInSet = void 0;
|
||||
const content = `--[[
|
||||
Remove jobs from the specific set.
|
||||
Input:
|
||||
KEYS[1] set key,
|
||||
KEYS[2] events stream key
|
||||
ARGV[1] jobKey prefix
|
||||
ARGV[2] timestamp
|
||||
ARGV[3] limit the number of jobs to be removed. 0 is unlimited
|
||||
ARGV[4] set name, can be any of 'wait', 'active', 'paused', 'delayed', 'completed', or 'failed'
|
||||
]]
|
||||
local rcall = redis.call
|
||||
local rangeStart = 0
|
||||
local rangeEnd = -1
|
||||
local limit = tonumber(ARGV[3])
|
||||
-- If we're only deleting _n_ items, avoid retrieving all items
|
||||
-- for faster performance
|
||||
--
|
||||
-- Start from the tail of the list, since that's where oldest elements
|
||||
-- are generally added for FIFO lists
|
||||
if limit > 0 then
|
||||
rangeStart = -1 - limit + 1
|
||||
rangeEnd = -1
|
||||
end
|
||||
-- Includes
|
||||
--[[
|
||||
Function to clean job list.
|
||||
Returns jobIds and deleted count number.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Function to get the latest saved timestamp.
|
||||
]]
|
||||
local function getTimestamp(jobKey, attributes)
|
||||
if #attributes == 1 then
|
||||
return rcall("HGET", jobKey, attributes[1])
|
||||
end
|
||||
local jobTs
|
||||
for _, ts in ipairs(rcall("HMGET", jobKey, unpack(attributes))) do
|
||||
if (ts) then
|
||||
jobTs = ts
|
||||
break
|
||||
end
|
||||
end
|
||||
return jobTs
|
||||
end
|
||||
--[[
|
||||
Function to remove job.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Check if this job has a parent. If so we will just remove it from
|
||||
the parent child list, but if it is the last child we should move the parent to "wait/paused"
|
||||
which requires code from "moveToFinished"
|
||||
]]
|
||||
--[[
|
||||
Functions to destructure job key.
|
||||
Just a bit of warning, these functions may be a bit slow and affect performance significantly.
|
||||
]]
|
||||
local getJobIdFromKey = function (jobKey)
|
||||
return string.match(jobKey, ".*:(.*)")
|
||||
end
|
||||
local getJobKeyPrefix = function (jobKey, jobId)
|
||||
return string.sub(jobKey, 0, #jobKey - #jobId)
|
||||
end
|
||||
--[[
|
||||
Function to check for the meta.paused key to decide if we are paused or not
|
||||
(since an empty list and !EXISTS are not really the same).
|
||||
]]
|
||||
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
|
||||
if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
|
||||
return waitKey, false
|
||||
else
|
||||
return pausedKey, true
|
||||
end
|
||||
end
|
||||
local function moveParentToWait(parentPrefix, parentId, emitEvent)
|
||||
local parentTarget = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "wait", parentPrefix .. "paused")
|
||||
rcall("RPUSH", parentTarget, parentId)
|
||||
if emitEvent then
|
||||
local parentEventStream = parentPrefix .. "events"
|
||||
rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
|
||||
end
|
||||
end
|
||||
local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey)
|
||||
if parentKey then
|
||||
local parentDependenciesKey = parentKey .. ":dependencies"
|
||||
local result = rcall("SREM", parentDependenciesKey, jobKey)
|
||||
if result > 0 then
|
||||
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
|
||||
if pendingDependencies == 0 then
|
||||
local parentId = getJobIdFromKey(parentKey)
|
||||
local parentPrefix = getJobKeyPrefix(parentKey, parentId)
|
||||
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
|
||||
if numRemovedElements == 1 then
|
||||
if hard then
|
||||
if parentPrefix == baseKey then
|
||||
removeParentDependencyKey(parentKey, hard, nil, baseKey)
|
||||
rcall("DEL", parentKey, parentKey .. ':logs',
|
||||
parentKey .. ':dependencies', parentKey .. ':processed')
|
||||
else
|
||||
moveParentToWait(parentPrefix, parentId)
|
||||
end
|
||||
else
|
||||
moveParentToWait(parentPrefix, parentId, true)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
else
|
||||
local missedParentKey = rcall("HGET", jobKey, "parentKey")
|
||||
if( (type(missedParentKey) == "string") and missedParentKey ~= "" and (rcall("EXISTS", missedParentKey) == 1)) then
|
||||
local parentDependenciesKey = missedParentKey .. ":dependencies"
|
||||
local result = rcall("SREM", parentDependenciesKey, jobKey)
|
||||
if result > 0 then
|
||||
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
|
||||
if pendingDependencies == 0 then
|
||||
local parentId = getJobIdFromKey(missedParentKey)
|
||||
local parentPrefix = getJobKeyPrefix(missedParentKey, parentId)
|
||||
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
|
||||
if numRemovedElements == 1 then
|
||||
if hard then
|
||||
if parentPrefix == baseKey then
|
||||
removeParentDependencyKey(missedParentKey, hard, nil, baseKey)
|
||||
rcall("DEL", missedParentKey, missedParentKey .. ':logs',
|
||||
missedParentKey .. ':dependencies', missedParentKey .. ':processed')
|
||||
else
|
||||
moveParentToWait(parentPrefix, parentId)
|
||||
end
|
||||
else
|
||||
moveParentToWait(parentPrefix, parentId, true)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
local function removeJob(jobId, hard, baseKey)
|
||||
local jobKey = baseKey .. jobId
|
||||
removeParentDependencyKey(jobKey, hard, nil, baseKey)
|
||||
rcall("DEL", jobKey, jobKey .. ':logs',
|
||||
jobKey .. ':dependencies', jobKey .. ':processed')
|
||||
end
|
||||
local function cleanList(listKey, jobKeyPrefix, rangeStart, rangeEnd,
|
||||
timestamp, isWaiting)
|
||||
local jobs = rcall("LRANGE", listKey, rangeStart, rangeEnd)
|
||||
local deleted = {}
|
||||
local deletedCount = 0
|
||||
local jobTS
|
||||
local deletionMarker = ''
|
||||
local jobIdsLen = #jobs
|
||||
for i, job in ipairs(jobs) do
|
||||
if limit > 0 and deletedCount >= limit then
|
||||
break
|
||||
end
|
||||
local jobKey = jobKeyPrefix .. job
|
||||
if (isWaiting or rcall("EXISTS", jobKey .. ":lock") == 0) then
|
||||
-- Find the right timestamp of the job to compare to maxTimestamp:
|
||||
-- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed
|
||||
-- * processedOn represents when the job was last attempted, but it doesn't get populated until
|
||||
-- the job is first tried
|
||||
-- * timestamp is the original job submission time
|
||||
-- Fetch all three of these (in that order) and use the first one that is set so that we'll leave jobs
|
||||
-- that have been active within the grace period:
|
||||
jobTS = getTimestamp(jobKey, {"finishedOn", "processedOn", "timestamp"})
|
||||
if (not jobTS or jobTS <= timestamp) then
|
||||
-- replace the entry with a deletion marker; the actual deletion will
|
||||
-- occur at the end of the script
|
||||
rcall("LSET", listKey, rangeEnd - jobIdsLen + i, deletionMarker)
|
||||
removeJob(job, true, jobKeyPrefix)
|
||||
deletedCount = deletedCount + 1
|
||||
table.insert(deleted, job)
|
||||
end
|
||||
end
|
||||
end
|
||||
rcall("LREM", listKey, 0, deletionMarker)
|
||||
return {deleted, deletedCount}
|
||||
end
|
||||
--[[
|
||||
Function to clean job set.
|
||||
Returns jobIds and deleted count number.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Function to loop in batches.
|
||||
Just a bit of warning, some commands as ZREM
|
||||
could receive a maximum of 7000 parameters per call.
|
||||
]]
|
||||
local function batches(n, batchSize)
|
||||
local i = 0
|
||||
return function()
|
||||
local from = i * batchSize + 1
|
||||
i = i + 1
|
||||
if (from <= n) then
|
||||
local to = math.min(from + batchSize - 1, n)
|
||||
return from, to
|
||||
end
|
||||
end
|
||||
end
|
||||
-- We use ZRANGEBYSCORE to make the case where we're deleting a limited number
|
||||
-- of items in a sorted set only run a single iteration. If we simply used
|
||||
-- ZRANGE, we may take a long time traversing through jobs that are within the
|
||||
-- grace period.
|
||||
local function getJobsInZset(zsetKey, rangeEnd, limit)
|
||||
if limit > 0 then
|
||||
return rcall("ZRANGEBYSCORE", zsetKey, 0, rangeEnd, "LIMIT", 0, limit)
|
||||
else
|
||||
return rcall("ZRANGEBYSCORE", zsetKey, 0, rangeEnd)
|
||||
end
|
||||
end
|
||||
local function cleanSet(setKey, jobKeyPrefix, rangeEnd, timestamp, limit, attributes, isFinished)
|
||||
local jobs = getJobsInZset(setKey, rangeEnd, limit)
|
||||
local deleted = {}
|
||||
local deletedCount = 0
|
||||
local jobTS
|
||||
for i, job in ipairs(jobs) do
|
||||
if limit > 0 and deletedCount >= limit then
|
||||
break
|
||||
end
|
||||
local jobKey = jobKeyPrefix .. job
|
||||
if isFinished then
|
||||
removeJob(job, true, jobKeyPrefix)
|
||||
deletedCount = deletedCount + 1
|
||||
table.insert(deleted, job)
|
||||
else
|
||||
-- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed
|
||||
jobTS = getTimestamp(jobKey, attributes)
|
||||
if (not jobTS or jobTS <= timestamp) then
|
||||
removeJob(job, true, jobKeyPrefix)
|
||||
deletedCount = deletedCount + 1
|
||||
table.insert(deleted, job)
|
||||
end
|
||||
end
|
||||
end
|
||||
if(#deleted > 0) then
|
||||
for from, to in batches(#deleted, 7000) do
|
||||
rcall("ZREM", setKey, unpack(deleted, from, to))
|
||||
end
|
||||
end
|
||||
return {deleted, deletedCount}
|
||||
end
|
||||
local result
|
||||
if ARGV[4] == "active" then
|
||||
result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], false)
|
||||
elseif ARGV[4] == "delayed" then
|
||||
rangeEnd = "+inf"
|
||||
result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
|
||||
{"processedOn", "timestamp"}, false)
|
||||
elseif ARGV[4] == "prioritized" then
|
||||
rangeEnd = "+inf"
|
||||
result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
|
||||
{"timestamp"}, false)
|
||||
elseif ARGV[4] == "wait" or ARGV[4] == "paused" then
|
||||
result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], true)
|
||||
else
|
||||
rangeEnd = ARGV[2]
|
||||
result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
|
||||
{"finishedOn"}, true)
|
||||
end
|
||||
rcall("XADD", KEYS[2], "*", "event", "cleaned", "count", result[2])
|
||||
return result[1]
|
||||
`;
|
||||
exports.cleanJobsInSet = {
|
||||
name: 'cleanJobsInSet',
|
||||
content,
|
||||
keys: 2,
|
||||
};
|
||||
//# sourceMappingURL=cleanJobsInSet-2.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"cleanJobsInSet-2.js","sourceRoot":"","sources":["../../../src/scripts/cleanJobsInSet-2.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAqQf,CAAC;AACW,QAAA,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
||||
+182
@@ -0,0 +1,182 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.drain = void 0;
|
||||
const content = `--[[
|
||||
Drains the queue, removes all jobs that are waiting
|
||||
or delayed, but not active, completed or failed
|
||||
Input:
|
||||
KEYS[1] 'wait',
|
||||
KEYS[2] 'paused'
|
||||
KEYS[3] 'delayed'
|
||||
KEYS[4] 'prioritized'
|
||||
ARGV[1] queue key prefix
|
||||
]]
|
||||
local rcall = redis.call
|
||||
local queueBaseKey = ARGV[1]
|
||||
--[[
|
||||
Functions to remove jobs.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Functions to remove jobs.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Function to remove job.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Check if this job has a parent. If so we will just remove it from
|
||||
the parent child list, but if it is the last child we should move the parent to "wait/paused"
|
||||
which requires code from "moveToFinished"
|
||||
]]
|
||||
--[[
|
||||
Functions to destructure job key.
|
||||
Just a bit of warning, these functions may be a bit slow and affect performance significantly.
|
||||
]]
|
||||
local getJobIdFromKey = function (jobKey)
|
||||
return string.match(jobKey, ".*:(.*)")
|
||||
end
|
||||
local getJobKeyPrefix = function (jobKey, jobId)
|
||||
return string.sub(jobKey, 0, #jobKey - #jobId)
|
||||
end
|
||||
--[[
|
||||
Function to check for the meta.paused key to decide if we are paused or not
|
||||
(since an empty list and !EXISTS are not really the same).
|
||||
]]
|
||||
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
|
||||
if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
|
||||
return waitKey, false
|
||||
else
|
||||
return pausedKey, true
|
||||
end
|
||||
end
|
||||
local function moveParentToWait(parentPrefix, parentId, emitEvent)
|
||||
local parentTarget = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "wait", parentPrefix .. "paused")
|
||||
rcall("RPUSH", parentTarget, parentId)
|
||||
if emitEvent then
|
||||
local parentEventStream = parentPrefix .. "events"
|
||||
rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
|
||||
end
|
||||
end
|
||||
local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey)
|
||||
if parentKey then
|
||||
local parentDependenciesKey = parentKey .. ":dependencies"
|
||||
local result = rcall("SREM", parentDependenciesKey, jobKey)
|
||||
if result > 0 then
|
||||
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
|
||||
if pendingDependencies == 0 then
|
||||
local parentId = getJobIdFromKey(parentKey)
|
||||
local parentPrefix = getJobKeyPrefix(parentKey, parentId)
|
||||
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
|
||||
if numRemovedElements == 1 then
|
||||
if hard then
|
||||
if parentPrefix == baseKey then
|
||||
removeParentDependencyKey(parentKey, hard, nil, baseKey)
|
||||
rcall("DEL", parentKey, parentKey .. ':logs',
|
||||
parentKey .. ':dependencies', parentKey .. ':processed')
|
||||
else
|
||||
moveParentToWait(parentPrefix, parentId)
|
||||
end
|
||||
else
|
||||
moveParentToWait(parentPrefix, parentId, true)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
else
|
||||
local missedParentKey = rcall("HGET", jobKey, "parentKey")
|
||||
if( (type(missedParentKey) == "string") and missedParentKey ~= "" and (rcall("EXISTS", missedParentKey) == 1)) then
|
||||
local parentDependenciesKey = missedParentKey .. ":dependencies"
|
||||
local result = rcall("SREM", parentDependenciesKey, jobKey)
|
||||
if result > 0 then
|
||||
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
|
||||
if pendingDependencies == 0 then
|
||||
local parentId = getJobIdFromKey(missedParentKey)
|
||||
local parentPrefix = getJobKeyPrefix(missedParentKey, parentId)
|
||||
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
|
||||
if numRemovedElements == 1 then
|
||||
if hard then
|
||||
if parentPrefix == baseKey then
|
||||
removeParentDependencyKey(missedParentKey, hard, nil, baseKey)
|
||||
rcall("DEL", missedParentKey, missedParentKey .. ':logs',
|
||||
missedParentKey .. ':dependencies', missedParentKey .. ':processed')
|
||||
else
|
||||
moveParentToWait(parentPrefix, parentId)
|
||||
end
|
||||
else
|
||||
moveParentToWait(parentPrefix, parentId, true)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
local function removeJob(jobId, hard, baseKey)
|
||||
local jobKey = baseKey .. jobId
|
||||
removeParentDependencyKey(jobKey, hard, nil, baseKey)
|
||||
rcall("DEL", jobKey, jobKey .. ':logs',
|
||||
jobKey .. ':dependencies', jobKey .. ':processed')
|
||||
end
|
||||
local function removeJobs(keys, hard, baseKey, max)
|
||||
for i, key in ipairs(keys) do
|
||||
removeJob(key, hard, baseKey)
|
||||
end
|
||||
return max - #keys
|
||||
end
|
||||
local function getListItems(keyName, max)
|
||||
return rcall('LRANGE', keyName, 0, max - 1)
|
||||
end
|
||||
local function removeListJobs(keyName, hard, baseKey, max)
|
||||
local jobs = getListItems(keyName, max)
|
||||
local count = removeJobs(jobs, hard, baseKey, max)
|
||||
rcall("LTRIM", keyName, #jobs, -1)
|
||||
return count
|
||||
end
|
||||
-- Includes
|
||||
--[[
|
||||
Function to loop in batches.
|
||||
Just a bit of warning, some commands as ZREM
|
||||
could receive a maximum of 7000 parameters per call.
|
||||
]]
|
||||
local function batches(n, batchSize)
|
||||
local i = 0
|
||||
return function()
|
||||
local from = i * batchSize + 1
|
||||
i = i + 1
|
||||
if (from <= n) then
|
||||
local to = math.min(from + batchSize - 1, n)
|
||||
return from, to
|
||||
end
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Function to get ZSet items.
|
||||
]]
|
||||
local function getZSetItems(keyName, max)
|
||||
return rcall('ZRANGE', keyName, 0, max - 1)
|
||||
end
|
||||
local function removeZSetJobs(keyName, hard, baseKey, max)
|
||||
local jobs = getZSetItems(keyName, max)
|
||||
local count = removeJobs(jobs, hard, baseKey, max)
|
||||
if(#jobs > 0) then
|
||||
for from, to in batches(#jobs, 7000) do
|
||||
rcall("ZREM", keyName, unpack(jobs, from, to))
|
||||
end
|
||||
end
|
||||
return count
|
||||
end
|
||||
removeListJobs(KEYS[1], true, queueBaseKey, 0) --wait
|
||||
removeListJobs(KEYS[2], true, queueBaseKey, 0) --paused
|
||||
if KEYS[3] ~= "" then
|
||||
removeZSetJobs(KEYS[3], true, queueBaseKey, 0) --delayed
|
||||
end
|
||||
removeZSetJobs(KEYS[4], true, queueBaseKey, 0) --prioritized
|
||||
`;
|
||||
exports.drain = {
|
||||
name: 'drain',
|
||||
content,
|
||||
keys: 4,
|
||||
};
|
||||
//# sourceMappingURL=drain-4.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"drain-4.js","sourceRoot":"","sources":["../../../src/scripts/drain-4.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA4Kf,CAAC;AACW,QAAA,KAAK,GAAG;IACnB,IAAI,EAAE,OAAO;IACb,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.extendLock = void 0;
|
||||
const content = `--[[
|
||||
Extend lock and removes the job from the stalled set.
|
||||
Input:
|
||||
KEYS[1] 'lock',
|
||||
KEYS[2] 'stalled'
|
||||
ARGV[1] token
|
||||
ARGV[2] lock duration in milliseconds
|
||||
ARGV[3] jobid
|
||||
Output:
|
||||
"1" if lock extented succesfully.
|
||||
]]
|
||||
local rcall = redis.call
|
||||
if rcall("GET", KEYS[1]) == ARGV[1] then
|
||||
-- if rcall("SET", KEYS[1], ARGV[1], "PX", ARGV[2], "XX") then
|
||||
if rcall("SET", KEYS[1], ARGV[1], "PX", ARGV[2]) then
|
||||
rcall("SREM", KEYS[2], ARGV[3])
|
||||
return 1
|
||||
end
|
||||
end
|
||||
return 0
|
||||
`;
|
||||
exports.extendLock = {
|
||||
name: 'extendLock',
|
||||
content,
|
||||
keys: 2,
|
||||
};
|
||||
//# sourceMappingURL=extendLock-2.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"extendLock-2.js","sourceRoot":"","sources":["../../../src/scripts/extendLock-2.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;CAoBf,CAAC;AACW,QAAA,UAAU,GAAG;IACxB,IAAI,EAAE,YAAY;IAClB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.getCounts = void 0;
|
||||
const content = `--[[
|
||||
Get counts per provided states
|
||||
Input:
|
||||
KEYS[1] 'prefix'
|
||||
ARGV[1...] types
|
||||
]]
|
||||
local rcall = redis.call;
|
||||
local prefix = KEYS[1]
|
||||
local results = {}
|
||||
for i = 1, #ARGV do
|
||||
local stateKey = prefix .. ARGV[i]
|
||||
if ARGV[i] == "wait" or ARGV[i] == "paused" then
|
||||
local marker = rcall("LINDEX", stateKey, -1)
|
||||
if marker and string.sub(marker, 1, 2) == "0:" then
|
||||
local count = rcall("LLEN", stateKey)
|
||||
if count > 1 then
|
||||
rcall("RPOP", stateKey)
|
||||
results[#results+1] = count-1
|
||||
else
|
||||
results[#results+1] = 0
|
||||
end
|
||||
else
|
||||
results[#results+1] = rcall("LLEN", stateKey)
|
||||
end
|
||||
elseif ARGV[i] == "active" then
|
||||
results[#results+1] = rcall("LLEN", stateKey)
|
||||
else
|
||||
results[#results+1] = rcall("ZCARD", stateKey)
|
||||
end
|
||||
end
|
||||
return results
|
||||
`;
|
||||
exports.getCounts = {
|
||||
name: 'getCounts',
|
||||
content,
|
||||
keys: 1,
|
||||
};
|
||||
//# sourceMappingURL=getCounts-1.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"getCounts-1.js","sourceRoot":"","sources":["../../../src/scripts/getCounts-1.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+Bf,CAAC;AACW,QAAA,SAAS,GAAG;IACvB,IAAI,EAAE,WAAW;IACjB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.getRanges = void 0;
|
||||
const content = `--[[
|
||||
Get job ids per provided states
|
||||
Input:
|
||||
KEYS[1] 'prefix'
|
||||
ARGV[1] start
|
||||
ARGV[2] end
|
||||
ARGV[3] asc
|
||||
ARGV[4...] types
|
||||
]]
|
||||
local rcall = redis.call
|
||||
local prefix = KEYS[1]
|
||||
local rangeStart = tonumber(ARGV[1])
|
||||
local rangeEnd = tonumber(ARGV[2])
|
||||
local asc = ARGV[3]
|
||||
local results = {}
|
||||
local function getRangeInList(listKey, asc, rangeStart, rangeEnd, results)
|
||||
if asc == "1" then
|
||||
local modifiedRangeStart
|
||||
local modifiedRangeEnd
|
||||
if rangeStart == -1 then
|
||||
modifiedRangeStart = 0
|
||||
else
|
||||
modifiedRangeStart = -(rangeStart + 1)
|
||||
end
|
||||
if rangeEnd == -1 then
|
||||
modifiedRangeEnd = 0
|
||||
else
|
||||
modifiedRangeEnd = -(rangeEnd + 1)
|
||||
end
|
||||
results[#results+1] = rcall("LRANGE", listKey,
|
||||
modifiedRangeEnd,
|
||||
modifiedRangeStart)
|
||||
else
|
||||
results[#results+1] = rcall("LRANGE", listKey, rangeStart, rangeEnd)
|
||||
end
|
||||
end
|
||||
for i = 4, #ARGV do
|
||||
local stateKey = prefix .. ARGV[i]
|
||||
if ARGV[i] == "wait" or ARGV[i] == "paused" then
|
||||
local marker = rcall("LINDEX", stateKey, -1)
|
||||
if marker and string.sub(marker, 1, 2) == "0:" then
|
||||
local count = rcall("LLEN", stateKey)
|
||||
if count > 1 then
|
||||
rcall("RPOP", stateKey)
|
||||
getRangeInList(stateKey, asc, rangeStart, rangeEnd, results)
|
||||
else
|
||||
results[#results+1] = {}
|
||||
end
|
||||
else
|
||||
getRangeInList(stateKey, asc, rangeStart, rangeEnd, results)
|
||||
end
|
||||
elseif ARGV[i] == "active" then
|
||||
getRangeInList(stateKey, asc, rangeStart, rangeEnd, results)
|
||||
else
|
||||
if asc == "1" then
|
||||
results[#results+1] = rcall("ZRANGE", stateKey, rangeStart, rangeEnd)
|
||||
else
|
||||
results[#results+1] = rcall("ZREVRANGE", stateKey, rangeStart, rangeEnd)
|
||||
end
|
||||
end
|
||||
end
|
||||
return results
|
||||
`;
|
||||
exports.getRanges = {
|
||||
name: 'getRanges',
|
||||
content,
|
||||
keys: 1,
|
||||
};
|
||||
//# sourceMappingURL=getRanges-1.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"getRanges-1.js","sourceRoot":"","sources":["../../../src/scripts/getRanges-1.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8Df,CAAC;AACW,QAAA,SAAS,GAAG;IACvB,IAAI,EAAE,WAAW;IACjB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
||||
+73
@@ -0,0 +1,73 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.getState = void 0;
|
||||
const content = `--[[
|
||||
Get a job state
|
||||
Input:
|
||||
KEYS[1] 'completed' key,
|
||||
KEYS[2] 'failed' key
|
||||
KEYS[3] 'delayed' key
|
||||
KEYS[4] 'active' key
|
||||
KEYS[5] 'wait' key
|
||||
KEYS[6] 'paused' key
|
||||
KEYS[7] 'waiting-children' key
|
||||
KEYS[8] 'prioritized' key
|
||||
ARGV[1] job id
|
||||
Output:
|
||||
'completed'
|
||||
'failed'
|
||||
'delayed'
|
||||
'active'
|
||||
'prioritized'
|
||||
'waiting'
|
||||
'waiting-children'
|
||||
'unknown'
|
||||
]]
|
||||
local rcall = redis.call
|
||||
if rcall("ZSCORE", KEYS[1], ARGV[1]) ~= false then
|
||||
return "completed"
|
||||
end
|
||||
if rcall("ZSCORE", KEYS[2], ARGV[1]) ~= false then
|
||||
return "failed"
|
||||
end
|
||||
if rcall("ZSCORE", KEYS[3], ARGV[1]) ~= false then
|
||||
return "delayed"
|
||||
end
|
||||
if rcall("ZSCORE", KEYS[8], ARGV[1]) ~= false then
|
||||
return "prioritized"
|
||||
end
|
||||
-- Includes
|
||||
--[[
|
||||
Functions to check if a item belongs to a list.
|
||||
]]
|
||||
local function checkItemInList(list, item)
|
||||
for _, v in pairs(list) do
|
||||
if v == item then
|
||||
return 1
|
||||
end
|
||||
end
|
||||
return nil
|
||||
end
|
||||
local active_items = rcall("LRANGE", KEYS[4] , 0, -1)
|
||||
if checkItemInList(active_items, ARGV[1]) ~= nil then
|
||||
return "active"
|
||||
end
|
||||
local wait_items = rcall("LRANGE", KEYS[5] , 0, -1)
|
||||
if checkItemInList(wait_items, ARGV[1]) ~= nil then
|
||||
return "waiting"
|
||||
end
|
||||
local paused_items = rcall("LRANGE", KEYS[6] , 0, -1)
|
||||
if checkItemInList(paused_items, ARGV[1]) ~= nil then
|
||||
return "waiting"
|
||||
end
|
||||
if rcall("ZSCORE", KEYS[7], ARGV[1]) ~= false then
|
||||
return "waiting-children"
|
||||
end
|
||||
return "unknown"
|
||||
`;
|
||||
exports.getState = {
|
||||
name: 'getState',
|
||||
content,
|
||||
keys: 8,
|
||||
};
|
||||
//# sourceMappingURL=getState-8.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"getState-8.js","sourceRoot":"","sources":["../../../src/scripts/getState-8.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+Df,CAAC;AACW,QAAA,QAAQ,GAAG;IACtB,IAAI,EAAE,UAAU;IAChB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.getStateV2 = void 0;
|
||||
const content = `--[[
|
||||
Get a job state
|
||||
Input:
|
||||
KEYS[1] 'completed' key,
|
||||
KEYS[2] 'failed' key
|
||||
KEYS[3] 'delayed' key
|
||||
KEYS[4] 'active' key
|
||||
KEYS[5] 'wait' key
|
||||
KEYS[6] 'paused' key
|
||||
KEYS[7] 'waiting-children' key
|
||||
KEYS[8] 'prioritized' key
|
||||
ARGV[1] job id
|
||||
Output:
|
||||
'completed'
|
||||
'failed'
|
||||
'delayed'
|
||||
'active'
|
||||
'waiting'
|
||||
'waiting-children'
|
||||
'unknown'
|
||||
]]
|
||||
local rcall = redis.call
|
||||
if rcall("ZSCORE", KEYS[1], ARGV[1]) ~= false then
|
||||
return "completed"
|
||||
end
|
||||
if rcall("ZSCORE", KEYS[2], ARGV[1]) ~= false then
|
||||
return "failed"
|
||||
end
|
||||
if rcall("ZSCORE", KEYS[3], ARGV[1]) ~= false then
|
||||
return "delayed"
|
||||
end
|
||||
if rcall("ZSCORE", KEYS[8], ARGV[1]) ~= false then
|
||||
return "prioritized"
|
||||
end
|
||||
if rcall("LPOS", KEYS[4] , ARGV[1]) ~= false then
|
||||
return "active"
|
||||
end
|
||||
if rcall("LPOS", KEYS[5] , ARGV[1]) ~= false then
|
||||
return "waiting"
|
||||
end
|
||||
if rcall("LPOS", KEYS[6] , ARGV[1]) ~= false then
|
||||
return "waiting"
|
||||
end
|
||||
if rcall("ZSCORE", KEYS[7] , ARGV[1]) ~= false then
|
||||
return "waiting-children"
|
||||
end
|
||||
return "unknown"
|
||||
`;
|
||||
exports.getStateV2 = {
|
||||
name: 'getStateV2',
|
||||
content,
|
||||
keys: 8,
|
||||
};
|
||||
//# sourceMappingURL=getStateV2-8.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"getStateV2-8.js","sourceRoot":"","sources":["../../../src/scripts/getStateV2-8.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+Cf,CAAC;AACW,QAAA,UAAU,GAAG;IACxB,IAAI,EAAE,YAAY;IAClB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
const tslib_1 = require("tslib");
|
||||
tslib_1.__exportStar(require("./addDelayedJob-7"), exports);
|
||||
tslib_1.__exportStar(require("./addParentJob-4"), exports);
|
||||
tslib_1.__exportStar(require("./addPrioritizedJob-8"), exports);
|
||||
tslib_1.__exportStar(require("./addStandardJob-6"), exports);
|
||||
tslib_1.__exportStar(require("./changeDelay-3"), exports);
|
||||
tslib_1.__exportStar(require("./changePriority-5"), exports);
|
||||
tslib_1.__exportStar(require("./cleanJobsInSet-2"), exports);
|
||||
tslib_1.__exportStar(require("./drain-4"), exports);
|
||||
tslib_1.__exportStar(require("./extendLock-2"), exports);
|
||||
tslib_1.__exportStar(require("./getCounts-1"), exports);
|
||||
tslib_1.__exportStar(require("./getRanges-1"), exports);
|
||||
tslib_1.__exportStar(require("./getState-8"), exports);
|
||||
tslib_1.__exportStar(require("./getStateV2-8"), exports);
|
||||
tslib_1.__exportStar(require("./isFinished-3"), exports);
|
||||
tslib_1.__exportStar(require("./isJobInList-1"), exports);
|
||||
tslib_1.__exportStar(require("./moveJobFromActiveToWait-9"), exports);
|
||||
tslib_1.__exportStar(require("./moveJobsToWait-6"), exports);
|
||||
tslib_1.__exportStar(require("./moveStalledJobsToWait-8"), exports);
|
||||
tslib_1.__exportStar(require("./moveToActive-10"), exports);
|
||||
tslib_1.__exportStar(require("./moveToDelayed-8"), exports);
|
||||
tslib_1.__exportStar(require("./moveToFinished-13"), exports);
|
||||
tslib_1.__exportStar(require("./moveToWaitingChildren-4"), exports);
|
||||
tslib_1.__exportStar(require("./obliterate-2"), exports);
|
||||
tslib_1.__exportStar(require("./paginate-1"), exports);
|
||||
tslib_1.__exportStar(require("./pause-5"), exports);
|
||||
tslib_1.__exportStar(require("./promote-7"), exports);
|
||||
tslib_1.__exportStar(require("./releaseLock-1"), exports);
|
||||
tslib_1.__exportStar(require("./removeJob-1"), exports);
|
||||
tslib_1.__exportStar(require("./removeRepeatable-2"), exports);
|
||||
tslib_1.__exportStar(require("./reprocessJob-6"), exports);
|
||||
tslib_1.__exportStar(require("./retryJob-9"), exports);
|
||||
tslib_1.__exportStar(require("./saveStacktrace-1"), exports);
|
||||
tslib_1.__exportStar(require("./updateData-1"), exports);
|
||||
tslib_1.__exportStar(require("./updateProgress-3"), exports);
|
||||
//# sourceMappingURL=index.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/scripts/index.ts"],"names":[],"mappings":";;;AAAA,4DAAkC;AAClC,2DAAiC;AACjC,gEAAsC;AACtC,6DAAmC;AACnC,0DAAgC;AAChC,6DAAmC;AACnC,6DAAmC;AACnC,oDAA0B;AAC1B,yDAA+B;AAC/B,wDAA8B;AAC9B,wDAA8B;AAC9B,uDAA6B;AAC7B,yDAA+B;AAC/B,yDAA+B;AAC/B,0DAAgC;AAChC,sEAA4C;AAC5C,6DAAmC;AACnC,oEAA0C;AAC1C,4DAAkC;AAClC,4DAAkC;AAClC,8DAAoC;AACpC,oEAA0C;AAC1C,yDAA+B;AAC/B,uDAA6B;AAC7B,oDAA0B;AAC1B,sDAA4B;AAC5B,0DAAgC;AAChC,wDAA8B;AAC9B,+DAAqC;AACrC,2DAAiC;AACjC,uDAA6B;AAC7B,6DAAmC;AACnC,yDAA+B;AAC/B,6DAAmC"}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.isFinished = void 0;
|
||||
const content = `--[[
|
||||
Checks if a job is finished (.i.e. is in the completed or failed set)
|
||||
Input:
|
||||
KEYS[1] completed key
|
||||
KEYS[2] failed key
|
||||
KEYS[3] job key
|
||||
ARGV[1] job id
|
||||
ARGV[2] return value?
|
||||
Output:
|
||||
0 - Not finished.
|
||||
1 - Completed.
|
||||
2 - Failed.
|
||||
-1 - Missing job.
|
||||
]]
|
||||
local rcall = redis.call
|
||||
if rcall("EXISTS", KEYS[3]) ~= 1 then
|
||||
if ARGV[2] == "1" then
|
||||
return {-1,"Missing key for job " .. KEYS[3] .. ". isFinished"}
|
||||
end
|
||||
return -1
|
||||
end
|
||||
if rcall("ZSCORE", KEYS[1], ARGV[1]) ~= false then
|
||||
if ARGV[2] == "1" then
|
||||
local returnValue = rcall("HGET", KEYS[3], "returnvalue")
|
||||
return {1,returnValue}
|
||||
end
|
||||
return 1
|
||||
end
|
||||
if rcall("ZSCORE", KEYS[2], ARGV[1]) ~= false then
|
||||
if ARGV[2] == "1" then
|
||||
local failedReason = rcall("HGET", KEYS[3], "failedReason")
|
||||
return {2,failedReason}
|
||||
end
|
||||
return 2
|
||||
end
|
||||
if ARGV[2] == "1" then
|
||||
return {0}
|
||||
end
|
||||
return 0
|
||||
`;
|
||||
exports.isFinished = {
|
||||
name: 'isFinished',
|
||||
content,
|
||||
keys: 3,
|
||||
};
|
||||
//# sourceMappingURL=isFinished-3.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"isFinished-3.js","sourceRoot":"","sources":["../../../src/scripts/isFinished-3.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAuCf,CAAC;AACW,QAAA,UAAU,GAAG;IACxB,IAAI,EAAE,YAAY;IAClB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.isJobInList = void 0;
|
||||
const content = `--[[
|
||||
Checks if job is in a given list.
|
||||
Input:
|
||||
KEYS[1]
|
||||
ARGV[1]
|
||||
Output:
|
||||
1 if element found in the list.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Functions to check if a item belongs to a list.
|
||||
]]
|
||||
local function checkItemInList(list, item)
|
||||
for _, v in pairs(list) do
|
||||
if v == item then
|
||||
return 1
|
||||
end
|
||||
end
|
||||
return nil
|
||||
end
|
||||
local items = redis.call("LRANGE", KEYS[1] , 0, -1)
|
||||
return checkItemInList(items, ARGV[1])
|
||||
`;
|
||||
exports.isJobInList = {
|
||||
name: 'isJobInList',
|
||||
content,
|
||||
keys: 1,
|
||||
};
|
||||
//# sourceMappingURL=isJobInList-1.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"isJobInList-1.js","sourceRoot":"","sources":["../../../src/scripts/isJobInList-1.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;CAsBf,CAAC;AACW,QAAA,WAAW,GAAG;IACzB,IAAI,EAAE,aAAa;IACnB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
||||
+70
@@ -0,0 +1,70 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.moveJobFromActiveToWait = void 0;
|
||||
const content = `--[[
|
||||
Function to move job from active state to wait.
|
||||
Input:
|
||||
KEYS[1] active key
|
||||
KEYS[2] wait key
|
||||
KEYS[3] stalled key
|
||||
KEYS[4] job lock key
|
||||
KEYS[5] paused key
|
||||
KEYS[6] meta key
|
||||
KEYS[7] limiter key
|
||||
KEYS[8] prioritized key
|
||||
KEYS[9] event key
|
||||
ARGV[1] job id
|
||||
ARGV[2] lock token
|
||||
ARGV[3] job id key
|
||||
]]
|
||||
local rcall = redis.call
|
||||
-- Includes
|
||||
--[[
|
||||
Function to push back job considering priority in front of same prioritized jobs.
|
||||
]]
|
||||
local function pushBackJobWithPriority(prioritizedKey, priority, jobId)
|
||||
-- in order to put it at front of same prioritized jobs
|
||||
-- we consider prioritized counter as 0
|
||||
local score = priority * 0x100000000
|
||||
rcall("ZADD", prioritizedKey, score, jobId)
|
||||
end
|
||||
--[[
|
||||
Function to check for the meta.paused key to decide if we are paused or not
|
||||
(since an empty list and !EXISTS are not really the same).
|
||||
]]
|
||||
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
|
||||
if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
|
||||
return waitKey, false
|
||||
else
|
||||
return pausedKey, true
|
||||
end
|
||||
end
|
||||
local jobId = ARGV[1]
|
||||
local token = ARGV[2]
|
||||
local lockKey = KEYS[4]
|
||||
local lockToken = rcall("GET", lockKey)
|
||||
local pttl = rcall("PTTL", KEYS[7])
|
||||
if lockToken == token and pttl > 0 then
|
||||
local removed = rcall("LREM", KEYS[1], 1, jobId)
|
||||
if (removed > 0) then
|
||||
local target = getTargetQueueList(KEYS[6], KEYS[2], KEYS[5])
|
||||
rcall("SREM", KEYS[3], jobId)
|
||||
local priority = tonumber(rcall("HGET", ARGV[3], "priority")) or 0
|
||||
if priority > 0 then
|
||||
pushBackJobWithPriority(KEYS[8], priority, jobId)
|
||||
else
|
||||
rcall("RPUSH", target, jobId)
|
||||
end
|
||||
rcall("DEL", lockKey)
|
||||
-- Emit waiting event
|
||||
rcall("XADD", KEYS[9], "*", "event", "waiting", "jobId", jobId)
|
||||
end
|
||||
end
|
||||
return pttl
|
||||
`;
|
||||
exports.moveJobFromActiveToWait = {
|
||||
name: 'moveJobFromActiveToWait',
|
||||
content,
|
||||
keys: 9,
|
||||
};
|
||||
//# sourceMappingURL=moveJobFromActiveToWait-9.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"moveJobFromActiveToWait-9.js","sourceRoot":"","sources":["../../../src/scripts/moveJobFromActiveToWait-9.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA4Df,CAAC;AACW,QAAA,uBAAuB,GAAG;IACrC,IAAI,EAAE,yBAAyB;IAC/B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
||||
+87
@@ -0,0 +1,87 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.moveJobsToWait = void 0;
|
||||
const content = `--[[
|
||||
Move completed, failed or delayed jobs to wait.
|
||||
Note: Does not support jobs with priorities.
|
||||
Input:
|
||||
KEYS[1] base key
|
||||
KEYS[2] events stream
|
||||
KEYS[3] state key (failed, completed, delayed)
|
||||
KEYS[4] 'wait'
|
||||
KEYS[5] 'paused'
|
||||
KEYS[6] 'meta'
|
||||
ARGV[1] count
|
||||
ARGV[2] timestamp
|
||||
ARGV[3] prev state
|
||||
Output:
|
||||
1 means the operation is not completed
|
||||
0 means the operation is completed
|
||||
]]
|
||||
local maxCount = tonumber(ARGV[1])
|
||||
local timestamp = tonumber(ARGV[2])
|
||||
local rcall = redis.call;
|
||||
-- Includes
|
||||
--[[
|
||||
Function to loop in batches.
|
||||
Just a bit of warning, some commands as ZREM
|
||||
could receive a maximum of 7000 parameters per call.
|
||||
]]
|
||||
local function batches(n, batchSize)
|
||||
local i = 0
|
||||
return function()
|
||||
local from = i * batchSize + 1
|
||||
i = i + 1
|
||||
if (from <= n) then
|
||||
local to = math.min(from + batchSize - 1, n)
|
||||
return from, to
|
||||
end
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Function to check for the meta.paused key to decide if we are paused or not
|
||||
(since an empty list and !EXISTS are not really the same).
|
||||
]]
|
||||
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
|
||||
if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
|
||||
return waitKey, false
|
||||
else
|
||||
return pausedKey, true
|
||||
end
|
||||
end
|
||||
local metaKey = KEYS[6]
|
||||
local target = getTargetQueueList(metaKey, KEYS[4], KEYS[5])
|
||||
local jobs = rcall('ZRANGEBYSCORE', KEYS[3], 0, timestamp, 'LIMIT', 0, maxCount)
|
||||
if (#jobs > 0) then
|
||||
if ARGV[3] == "failed" then
|
||||
for i, key in ipairs(jobs) do
|
||||
local jobKey = KEYS[1] .. key
|
||||
rcall("HDEL", jobKey, "finishedOn", "processedOn", "failedReason")
|
||||
end
|
||||
elseif ARGV[3] == "completed" then
|
||||
for i, key in ipairs(jobs) do
|
||||
local jobKey = KEYS[1] .. key
|
||||
rcall("HDEL", jobKey, "finishedOn", "processedOn", "returnvalue")
|
||||
end
|
||||
end
|
||||
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents") or 10000
|
||||
for i, key in ipairs(jobs) do
|
||||
-- Emit waiting event
|
||||
rcall("XADD", KEYS[2], "MAXLEN", "~", maxEvents, "*", "event",
|
||||
"waiting", "jobId", key, "prev", ARGV[3]);
|
||||
end
|
||||
for from, to in batches(#jobs, 7000) do
|
||||
rcall("ZREM", KEYS[3], unpack(jobs, from, to))
|
||||
rcall("LPUSH", target, unpack(jobs, from, to))
|
||||
end
|
||||
end
|
||||
maxCount = maxCount - #jobs
|
||||
if (maxCount <= 0) then return 1 end
|
||||
return 0
|
||||
`;
|
||||
exports.moveJobsToWait = {
|
||||
name: 'moveJobsToWait',
|
||||
content,
|
||||
keys: 6,
|
||||
};
|
||||
//# sourceMappingURL=moveJobsToWait-6.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"moveJobsToWait-6.js","sourceRoot":"","sources":["../../../src/scripts/moveJobsToWait-6.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6Ef,CAAC;AACW,QAAA,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
||||
+289
@@ -0,0 +1,289 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.moveStalledJobsToWait = void 0;
|
||||
const content = `--[[
|
||||
Move stalled jobs to wait.
|
||||
Input:
|
||||
KEYS[1] 'stalled' (SET)
|
||||
KEYS[2] 'wait', (LIST)
|
||||
KEYS[3] 'active', (LIST)
|
||||
KEYS[4] 'failed', (ZSET)
|
||||
KEYS[5] 'stalled-check', (KEY)
|
||||
KEYS[6] 'meta', (KEY)
|
||||
KEYS[7] 'paused', (LIST)
|
||||
KEYS[8] 'event stream' (STREAM)
|
||||
ARGV[1] Max stalled job count
|
||||
ARGV[2] queue.toKey('')
|
||||
ARGV[3] timestamp
|
||||
ARGV[4] max check time
|
||||
Events:
|
||||
'stalled' with stalled job id.
|
||||
]] -- Includes
|
||||
--[[
|
||||
Move stalled jobs to wait.
|
||||
Input:
|
||||
stalledKey 'stalled' (SET)
|
||||
waitKey 'wait', (LIST)
|
||||
activeKey 'active', (LIST)
|
||||
failedKey 'failed', (ZSET)
|
||||
stalledCheckKey 'stalled-check', (KEY)
|
||||
metaKey 'meta', (KEY)
|
||||
pausedKey 'paused', (LIST)
|
||||
eventStreamKey 'event stream' (STREAM)
|
||||
maxStalledJobCount Max stalled job count
|
||||
queueKeyPrefix queue.toKey('')
|
||||
timestamp timestamp
|
||||
maxCheckTime max check time
|
||||
Events:
|
||||
'stalled' with stalled job id.
|
||||
]]
|
||||
local rcall = redis.call
|
||||
-- Includes
|
||||
--[[
|
||||
Function to loop in batches.
|
||||
Just a bit of warning, some commands as ZREM
|
||||
could receive a maximum of 7000 parameters per call.
|
||||
]]
|
||||
local function batches(n, batchSize)
|
||||
local i = 0
|
||||
return function()
|
||||
local from = i * batchSize + 1
|
||||
i = i + 1
|
||||
if (from <= n) then
|
||||
local to = math.min(from + batchSize - 1, n)
|
||||
return from, to
|
||||
end
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Function to check for the meta.paused key to decide if we are paused or not
|
||||
(since an empty list and !EXISTS are not really the same).
|
||||
]]
|
||||
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
|
||||
if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
|
||||
return waitKey, false
|
||||
else
|
||||
return pausedKey, true
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Function to remove job.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Check if this job has a parent. If so we will just remove it from
|
||||
the parent child list, but if it is the last child we should move the parent to "wait/paused"
|
||||
which requires code from "moveToFinished"
|
||||
]]
|
||||
--[[
|
||||
Functions to destructure job key.
|
||||
Just a bit of warning, these functions may be a bit slow and affect performance significantly.
|
||||
]]
|
||||
local getJobIdFromKey = function (jobKey)
|
||||
return string.match(jobKey, ".*:(.*)")
|
||||
end
|
||||
local getJobKeyPrefix = function (jobKey, jobId)
|
||||
return string.sub(jobKey, 0, #jobKey - #jobId)
|
||||
end
|
||||
local function moveParentToWait(parentPrefix, parentId, emitEvent)
|
||||
local parentTarget = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "wait", parentPrefix .. "paused")
|
||||
rcall("RPUSH", parentTarget, parentId)
|
||||
if emitEvent then
|
||||
local parentEventStream = parentPrefix .. "events"
|
||||
rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
|
||||
end
|
||||
end
|
||||
local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey)
|
||||
if parentKey then
|
||||
local parentDependenciesKey = parentKey .. ":dependencies"
|
||||
local result = rcall("SREM", parentDependenciesKey, jobKey)
|
||||
if result > 0 then
|
||||
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
|
||||
if pendingDependencies == 0 then
|
||||
local parentId = getJobIdFromKey(parentKey)
|
||||
local parentPrefix = getJobKeyPrefix(parentKey, parentId)
|
||||
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
|
||||
if numRemovedElements == 1 then
|
||||
if hard then
|
||||
if parentPrefix == baseKey then
|
||||
removeParentDependencyKey(parentKey, hard, nil, baseKey)
|
||||
rcall("DEL", parentKey, parentKey .. ':logs',
|
||||
parentKey .. ':dependencies', parentKey .. ':processed')
|
||||
else
|
||||
moveParentToWait(parentPrefix, parentId)
|
||||
end
|
||||
else
|
||||
moveParentToWait(parentPrefix, parentId, true)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
else
|
||||
local missedParentKey = rcall("HGET", jobKey, "parentKey")
|
||||
if( (type(missedParentKey) == "string") and missedParentKey ~= "" and (rcall("EXISTS", missedParentKey) == 1)) then
|
||||
local parentDependenciesKey = missedParentKey .. ":dependencies"
|
||||
local result = rcall("SREM", parentDependenciesKey, jobKey)
|
||||
if result > 0 then
|
||||
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
|
||||
if pendingDependencies == 0 then
|
||||
local parentId = getJobIdFromKey(missedParentKey)
|
||||
local parentPrefix = getJobKeyPrefix(missedParentKey, parentId)
|
||||
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
|
||||
if numRemovedElements == 1 then
|
||||
if hard then
|
||||
if parentPrefix == baseKey then
|
||||
removeParentDependencyKey(missedParentKey, hard, nil, baseKey)
|
||||
rcall("DEL", missedParentKey, missedParentKey .. ':logs',
|
||||
missedParentKey .. ':dependencies', missedParentKey .. ':processed')
|
||||
else
|
||||
moveParentToWait(parentPrefix, parentId)
|
||||
end
|
||||
else
|
||||
moveParentToWait(parentPrefix, parentId, true)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
local function removeJob(jobId, hard, baseKey)
|
||||
local jobKey = baseKey .. jobId
|
||||
removeParentDependencyKey(jobKey, hard, nil, baseKey)
|
||||
rcall("DEL", jobKey, jobKey .. ':logs',
|
||||
jobKey .. ':dependencies', jobKey .. ':processed')
|
||||
end
|
||||
--[[
|
||||
Functions to remove jobs by max age.
|
||||
]]
|
||||
-- Includes
|
||||
local function removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
|
||||
local start = timestamp - maxAge * 1000
|
||||
local jobIds = rcall("ZREVRANGEBYSCORE", targetSet, start, "-inf")
|
||||
for i, jobId in ipairs(jobIds) do
|
||||
removeJob(jobId, false, prefix)
|
||||
end
|
||||
rcall("ZREMRANGEBYSCORE", targetSet, "-inf", start)
|
||||
end
|
||||
--[[
|
||||
Functions to remove jobs by max count.
|
||||
]]
|
||||
-- Includes
|
||||
local function removeJobsByMaxCount(maxCount, targetSet, prefix)
|
||||
local start = maxCount
|
||||
local jobIds = rcall("ZREVRANGE", targetSet, start, -1)
|
||||
for i, jobId in ipairs(jobIds) do
|
||||
removeJob(jobId, false, prefix)
|
||||
end
|
||||
rcall("ZREMRANGEBYRANK", targetSet, 0, -(maxCount + 1))
|
||||
end
|
||||
--[[
|
||||
Function to trim events, default 10000.
|
||||
]]
|
||||
local function trimEvents(metaKey, eventStreamKey)
|
||||
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
||||
if maxEvents ~= false then
|
||||
rcall("XTRIM", eventStreamKey, "MAXLEN", "~", maxEvents)
|
||||
else
|
||||
rcall("XTRIM", eventStreamKey, "MAXLEN", "~", 10000)
|
||||
end
|
||||
end
|
||||
-- Check if we need to check for stalled jobs now.
|
||||
local function checkStalledJobs(stalledKey, waitKey, activeKey, failedKey,
|
||||
stalledCheckKey, metaKey, pausedKey,
|
||||
eventStreamKey, maxStalledJobCount,
|
||||
queueKeyPrefix, timestamp, maxCheckTime)
|
||||
if rcall("EXISTS", stalledCheckKey) == 1 then return {{}, {}} end
|
||||
rcall("SET", stalledCheckKey, timestamp, "PX", maxCheckTime)
|
||||
-- Trim events before emiting them to avoid trimming events emitted in this script
|
||||
trimEvents(metaKey, eventStreamKey)
|
||||
-- Move all stalled jobs to wait
|
||||
local stalling = rcall('SMEMBERS', stalledKey)
|
||||
local stalled = {}
|
||||
local failed = {}
|
||||
if (#stalling > 0) then
|
||||
rcall('DEL', stalledKey)
|
||||
local MAX_STALLED_JOB_COUNT = tonumber(maxStalledJobCount)
|
||||
-- Remove from active list
|
||||
for i, jobId in ipairs(stalling) do
|
||||
if string.sub(jobId, 1, 2) == "0:" then
|
||||
-- If the jobId is a delay marker ID we just remove it.
|
||||
rcall("LREM", activeKey, 1, jobId)
|
||||
else
|
||||
local jobKey = queueKeyPrefix .. jobId
|
||||
-- Check that the lock is also missing, then we can handle this job as really stalled.
|
||||
if (rcall("EXISTS", jobKey .. ":lock") == 0) then
|
||||
-- Remove from the active queue.
|
||||
local removed = rcall("LREM", activeKey, 1, jobId)
|
||||
if (removed > 0) then
|
||||
-- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
|
||||
local stalledCount =
|
||||
rcall("HINCRBY", jobKey, "stalledCounter", 1)
|
||||
if (stalledCount > MAX_STALLED_JOB_COUNT) then
|
||||
local rawOpts = rcall("HGET", jobKey, "opts")
|
||||
local opts = cjson.decode(rawOpts)
|
||||
local removeOnFailType = type(opts["removeOnFail"])
|
||||
rcall("ZADD", failedKey, timestamp, jobId)
|
||||
local failedReason =
|
||||
"job stalled more than allowable limit"
|
||||
rcall("HMSET", jobKey, "failedReason", failedReason,
|
||||
"finishedOn", timestamp)
|
||||
rcall("XADD", eventStreamKey, "*", "event",
|
||||
"failed", "jobId", jobId, 'prev', 'active',
|
||||
'failedReason', failedReason)
|
||||
if removeOnFailType == "number" then
|
||||
removeJobsByMaxCount(opts["removeOnFail"],
|
||||
failedKey, queueKeyPrefix)
|
||||
elseif removeOnFailType == "boolean" then
|
||||
if opts["removeOnFail"] then
|
||||
removeJob(jobId, false, queueKeyPrefix)
|
||||
rcall("ZREM", failedKey, jobId)
|
||||
end
|
||||
elseif removeOnFailType ~= "nil" then
|
||||
local maxAge = opts["removeOnFail"]["age"]
|
||||
local maxCount = opts["removeOnFail"]["count"]
|
||||
if maxAge ~= nil then
|
||||
removeJobsByMaxAge(timestamp, maxAge,
|
||||
failedKey, queueKeyPrefix)
|
||||
end
|
||||
if maxCount ~= nil and maxCount > 0 then
|
||||
removeJobsByMaxCount(maxCount, failedKey,
|
||||
queueKeyPrefix)
|
||||
end
|
||||
end
|
||||
table.insert(failed, jobId)
|
||||
else
|
||||
local target =
|
||||
getTargetQueueList(metaKey, waitKey, pausedKey)
|
||||
-- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
|
||||
rcall("RPUSH", target, jobId)
|
||||
rcall("XADD", eventStreamKey, "*", "event",
|
||||
"waiting", "jobId", jobId, 'prev', 'active')
|
||||
-- Emit the stalled event
|
||||
rcall("XADD", eventStreamKey, "*", "event",
|
||||
"stalled", "jobId", jobId)
|
||||
table.insert(stalled, jobId)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
-- Mark potentially stalled jobs
|
||||
local active = rcall('LRANGE', activeKey, 0, -1)
|
||||
if (#active > 0) then
|
||||
for from, to in batches(#active, 7000) do
|
||||
rcall('SADD', stalledKey, unpack(active, from, to))
|
||||
end
|
||||
end
|
||||
return {failed, stalled}
|
||||
end
|
||||
return checkStalledJobs(KEYS[1], KEYS[2], KEYS[3], KEYS[4], KEYS[5], KEYS[6],
|
||||
KEYS[7], KEYS[8], ARGV[1], ARGV[2], ARGV[3], ARGV[4])
|
||||
`;
|
||||
exports.moveStalledJobsToWait = {
|
||||
name: 'moveStalledJobsToWait',
|
||||
content,
|
||||
keys: 8,
|
||||
};
|
||||
//# sourceMappingURL=moveStalledJobsToWait-8.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"moveStalledJobsToWait-8.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-8.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAuRf,CAAC;AACW,QAAA,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
||||
+251
@@ -0,0 +1,251 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.moveToActive = void 0;
|
||||
const content = `--[[
|
||||
Move next job to be processed to active, lock it and fetch its data. The job
|
||||
may be delayed, in that case we need to move it to the delayed set instead.
|
||||
This operation guarantees that the worker owns the job during the lock
|
||||
expiration time. The worker is responsible of keeping the lock fresh
|
||||
so that no other worker picks this job again.
|
||||
Input:
|
||||
KEYS[1] wait key
|
||||
KEYS[2] active key
|
||||
KEYS[3] prioritized key
|
||||
KEYS[4] stream events key
|
||||
KEYS[5] stalled key
|
||||
-- Rate limiting
|
||||
KEYS[6] rate limiter key
|
||||
KEYS[7] delayed key
|
||||
-- Promote delayed jobs
|
||||
KEYS[8] paused key
|
||||
KEYS[9] meta key
|
||||
KEYS[10] pc priority counter
|
||||
-- Arguments
|
||||
ARGV[1] key prefix
|
||||
ARGV[2] timestamp
|
||||
ARGV[3] optional job ID
|
||||
ARGV[4] opts
|
||||
opts - token - lock token
|
||||
opts - lockDuration
|
||||
opts - limiter
|
||||
]]
|
||||
local rcall = redis.call
|
||||
local waitKey = KEYS[1]
|
||||
local activeKey = KEYS[2]
|
||||
local rateLimiterKey = KEYS[6]
|
||||
local delayedKey = KEYS[7]
|
||||
local opts = cmsgpack.unpack(ARGV[4])
|
||||
-- Includes
|
||||
--[[
|
||||
Function to return the next delayed job timestamp.
|
||||
]]
|
||||
local function getNextDelayedTimestamp(delayedKey)
|
||||
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
|
||||
if #result then
|
||||
local nextTimestamp = tonumber(result[2])
|
||||
if (nextTimestamp ~= nil) then
|
||||
nextTimestamp = nextTimestamp / 0x1000
|
||||
end
|
||||
return nextTimestamp
|
||||
end
|
||||
end
|
||||
local function getRateLimitTTL(maxJobs, rateLimiterKey)
|
||||
if maxJobs and maxJobs <= tonumber(rcall("GET", rateLimiterKey) or 0) then
|
||||
local pttl = rcall("PTTL", rateLimiterKey)
|
||||
if pttl == 0 then
|
||||
rcall("DEL", rateLimiterKey)
|
||||
end
|
||||
if pttl > 0 then
|
||||
return pttl
|
||||
end
|
||||
end
|
||||
return 0
|
||||
end
|
||||
--[[
|
||||
Function to check for the meta.paused key to decide if we are paused or not
|
||||
(since an empty list and !EXISTS are not really the same).
|
||||
]]
|
||||
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
|
||||
if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
|
||||
return waitKey, false
|
||||
else
|
||||
return pausedKey, true
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Function to move job from prioritized state to active.
|
||||
]]
|
||||
local function moveJobFromPriorityToActive(priorityKey, activeKey, priorityCounterKey)
|
||||
local prioritizedJob = rcall("ZPOPMIN", priorityKey)
|
||||
if #prioritizedJob > 0 then
|
||||
rcall("LPUSH", activeKey, prioritizedJob[1])
|
||||
return prioritizedJob[1]
|
||||
else
|
||||
rcall("DEL", priorityCounterKey)
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Function to move job from wait state to active.
|
||||
Input:
|
||||
keys[1] wait key
|
||||
keys[2] active key
|
||||
keys[3] prioritized key
|
||||
keys[4] stream events key
|
||||
keys[5] stalled key
|
||||
-- Rate limiting
|
||||
keys[6] rate limiter key
|
||||
keys[7] delayed key
|
||||
keys[8] paused key
|
||||
keys[9] meta key
|
||||
keys[10] pc priority counter
|
||||
opts - token - lock token
|
||||
opts - lockDuration
|
||||
opts - limiter
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Function to push back job considering priority in front of same prioritized jobs.
|
||||
]]
|
||||
local function pushBackJobWithPriority(prioritizedKey, priority, jobId)
|
||||
-- in order to put it at front of same prioritized jobs
|
||||
-- we consider prioritized counter as 0
|
||||
local score = priority * 0x100000000
|
||||
rcall("ZADD", prioritizedKey, score, jobId)
|
||||
end
|
||||
local function prepareJobForProcessing(keys, keyPrefix, targetKey, jobId, processedOn,
|
||||
maxJobs, expireTime, opts)
|
||||
local jobKey = keyPrefix .. jobId
|
||||
-- Check if we need to perform rate limiting.
|
||||
if maxJobs then
|
||||
local rateLimiterKey = keys[6];
|
||||
-- check if we exceeded rate limit, we need to remove the job and return expireTime
|
||||
if expireTime > 0 then
|
||||
-- remove from active queue and add back to the wait list
|
||||
rcall("LREM", keys[2], 1, jobId)
|
||||
local priority = tonumber(rcall("HGET", jobKey, "priority")) or 0
|
||||
if priority == 0 then
|
||||
rcall("RPUSH", targetKey, jobId)
|
||||
else
|
||||
pushBackJobWithPriority(keys[3], priority, jobId)
|
||||
end
|
||||
-- Return when we can process more jobs
|
||||
return {0, 0, expireTime, 0}
|
||||
end
|
||||
local jobCounter = tonumber(rcall("INCR", rateLimiterKey))
|
||||
if jobCounter == 1 then
|
||||
local limiterDuration = opts['limiter'] and opts['limiter']['duration']
|
||||
local integerDuration = math.floor(math.abs(limiterDuration))
|
||||
rcall("PEXPIRE", rateLimiterKey, integerDuration)
|
||||
end
|
||||
end
|
||||
local lockKey = jobKey .. ':lock'
|
||||
-- get a lock
|
||||
if opts['token'] ~= "0" then
|
||||
rcall("SET", lockKey, opts['token'], "PX", opts['lockDuration'])
|
||||
end
|
||||
rcall("XADD", keys[4], "*", "event", "active", "jobId", jobId, "prev", "waiting")
|
||||
rcall("HSET", jobKey, "processedOn", processedOn)
|
||||
rcall("HINCRBY", jobKey, "attemptsMade", 1)
|
||||
return {rcall("HGETALL", jobKey), jobId, 0, 0} -- get job data
|
||||
end
|
||||
--[[
|
||||
Updates the delay set, by moving delayed jobs that should
|
||||
be processed now to "wait".
|
||||
Events:
|
||||
'waiting'
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Function priority marker to wait if needed
|
||||
in order to wake up our workers and to respect priority
|
||||
order as much as possible
|
||||
]]
|
||||
local function addPriorityMarkerIfNeeded(waitKey)
|
||||
local waitLen = rcall("LLEN", waitKey)
|
||||
if waitLen == 0 then
|
||||
rcall("LPUSH", waitKey, "0:0")
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Function to get priority score.
|
||||
]]
|
||||
local function getPriorityScore(priority, priorityCounterKey)
|
||||
local prioCounter = rcall("INCR", priorityCounterKey)
|
||||
return priority * 0x100000000 + prioCounter % 0x100000000
|
||||
end
|
||||
-- Try to get as much as 1000 jobs at once
|
||||
local function promoteDelayedJobs(delayedKey, waitKey, targetKey, prioritizedKey,
|
||||
eventStreamKey, prefix, timestamp, paused, priorityCounterKey)
|
||||
local jobs = rcall("ZRANGEBYSCORE", delayedKey, 0, (timestamp + 1) * 0x1000, "LIMIT", 0, 1000)
|
||||
if (#jobs > 0) then
|
||||
rcall("ZREM", delayedKey, unpack(jobs))
|
||||
for _, jobId in ipairs(jobs) do
|
||||
local jobKey = prefix .. jobId
|
||||
local priority =
|
||||
tonumber(rcall("HGET", jobKey, "priority")) or 0
|
||||
if priority == 0 then
|
||||
-- LIFO or FIFO
|
||||
rcall("LPUSH", targetKey, jobId)
|
||||
else
|
||||
local score = getPriorityScore(priority, priorityCounterKey)
|
||||
rcall("ZADD", prioritizedKey, score, jobId)
|
||||
end
|
||||
-- Emit waiting event
|
||||
rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId",
|
||||
jobId, "prev", "delayed")
|
||||
rcall("HSET", jobKey, "delay", 0)
|
||||
end
|
||||
if not paused then
|
||||
addPriorityMarkerIfNeeded(targetKey)
|
||||
end
|
||||
end
|
||||
end
|
||||
local target, paused = getTargetQueueList(KEYS[9], waitKey, KEYS[8])
|
||||
-- Check if there are delayed jobs that we can move to wait.
|
||||
promoteDelayedJobs(delayedKey, waitKey, target, KEYS[3], KEYS[4], ARGV[1],
|
||||
ARGV[2], paused, KEYS[10])
|
||||
local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
|
||||
local expireTime = getRateLimitTTL(maxJobs, rateLimiterKey)
|
||||
local jobId = nil
|
||||
if ARGV[3] ~= "" then
|
||||
jobId = ARGV[3]
|
||||
-- clean stalled key
|
||||
rcall("SREM", KEYS[5], jobId)
|
||||
end
|
||||
if not jobId or (jobId and string.sub(jobId, 1, 2) == "0:") then
|
||||
-- If jobId is special ID 0:delay, then there is no job to process
|
||||
if jobId then rcall("LREM", activeKey, 1, jobId) end
|
||||
-- Check if we are rate limited first.
|
||||
if expireTime > 0 then return {0, 0, expireTime, 0} end
|
||||
-- paused queue
|
||||
if paused then return {0, 0, 0, 0} end
|
||||
-- no job ID, try non-blocking move from wait to active
|
||||
jobId = rcall("RPOPLPUSH", waitKey, activeKey)
|
||||
-- Since it is possible that between a call to BRPOPLPUSH and moveToActive
|
||||
-- another script puts a new maker in wait, we need to check again.
|
||||
if jobId and string.sub(jobId, 1, 2) == "0:" then
|
||||
rcall("LREM", activeKey, 1, jobId)
|
||||
jobId = rcall("RPOPLPUSH", waitKey, activeKey)
|
||||
end
|
||||
end
|
||||
if jobId then
|
||||
return prepareJobForProcessing(KEYS, ARGV[1], target, jobId, ARGV[2],
|
||||
maxJobs, expireTime, opts)
|
||||
else
|
||||
jobId = moveJobFromPriorityToActive(KEYS[3], activeKey, KEYS[10])
|
||||
if jobId then
|
||||
return prepareJobForProcessing(KEYS, ARGV[1], target, jobId, ARGV[2],
|
||||
maxJobs, expireTime, opts)
|
||||
end
|
||||
end
|
||||
-- Return the timestamp for the next delayed job if any.
|
||||
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
|
||||
if (nextTimestamp ~= nil) then return {0, 0, 0, nextTimestamp} end
|
||||
return {0, 0, 0, 0}
|
||||
`;
|
||||
exports.moveToActive = {
|
||||
name: 'moveToActive',
|
||||
content,
|
||||
keys: 10,
|
||||
};
|
||||
//# sourceMappingURL=moveToActive-10.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"moveToActive-10.js","sourceRoot":"","sources":["../../../src/scripts/moveToActive-10.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAiPf,CAAC;AACW,QAAA,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
|
||||
+166
@@ -0,0 +1,166 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.moveToDelayed = void 0;
|
||||
const content = `--[[
|
||||
Moves job from active to delayed set.
|
||||
Input:
|
||||
KEYS[1] wait key
|
||||
KEYS[2] active key
|
||||
KEYS[3] prioritized key
|
||||
KEYS[4] delayed key
|
||||
KEYS[5] job key
|
||||
KEYS[6] events stream
|
||||
KEYS[7] paused key
|
||||
KEYS[8] meta key
|
||||
ARGV[1] key prefix
|
||||
ARGV[2] timestamp
|
||||
ARGV[3] delayedTimestamp
|
||||
ARGV[4] the id of the job
|
||||
ARGV[5] queue token
|
||||
ARGV[6] delay value
|
||||
Output:
|
||||
0 - OK
|
||||
-1 - Missing job.
|
||||
-3 - Job not in active set.
|
||||
Events:
|
||||
- delayed key.
|
||||
]]
|
||||
local rcall = redis.call
|
||||
-- Includes
|
||||
--[[
|
||||
Add delay marker if needed.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Function to return the next delayed job timestamp.
|
||||
]]
|
||||
local function getNextDelayedTimestamp(delayedKey)
|
||||
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
|
||||
if #result then
|
||||
local nextTimestamp = tonumber(result[2])
|
||||
if (nextTimestamp ~= nil) then
|
||||
nextTimestamp = nextTimestamp / 0x1000
|
||||
end
|
||||
return nextTimestamp
|
||||
end
|
||||
end
|
||||
local function addDelayMarkerIfNeeded(targetKey, delayedKey)
|
||||
local waitLen = rcall("LLEN", targetKey)
|
||||
if waitLen <= 1 then
|
||||
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
|
||||
if nextTimestamp ~= nil then
|
||||
-- Check if there is already a marker with older timestamp
|
||||
-- if there is, we need to replace it.
|
||||
if waitLen == 1 then
|
||||
local marker = rcall("LINDEX", targetKey, 0)
|
||||
local oldTimestamp = tonumber(marker:sub(3))
|
||||
if oldTimestamp and oldTimestamp > nextTimestamp then
|
||||
rcall("LSET", targetKey, 0, "0:" .. nextTimestamp)
|
||||
end
|
||||
else
|
||||
-- if there is no marker, then we need to add one
|
||||
rcall("LPUSH", targetKey, "0:" .. nextTimestamp)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Function to check for the meta.paused key to decide if we are paused or not
|
||||
(since an empty list and !EXISTS are not really the same).
|
||||
]]
|
||||
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
|
||||
if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
|
||||
return waitKey, false
|
||||
else
|
||||
return pausedKey, true
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Updates the delay set, by moving delayed jobs that should
|
||||
be processed now to "wait".
|
||||
Events:
|
||||
'waiting'
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Function priority marker to wait if needed
|
||||
in order to wake up our workers and to respect priority
|
||||
order as much as possible
|
||||
]]
|
||||
local function addPriorityMarkerIfNeeded(waitKey)
|
||||
local waitLen = rcall("LLEN", waitKey)
|
||||
if waitLen == 0 then
|
||||
rcall("LPUSH", waitKey, "0:0")
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Function to get priority score.
|
||||
]]
|
||||
local function getPriorityScore(priority, priorityCounterKey)
|
||||
local prioCounter = rcall("INCR", priorityCounterKey)
|
||||
return priority * 0x100000000 + prioCounter % 0x100000000
|
||||
end
|
||||
-- Try to get as much as 1000 jobs at once
|
||||
local function promoteDelayedJobs(delayedKey, waitKey, targetKey, prioritizedKey,
|
||||
eventStreamKey, prefix, timestamp, paused, priorityCounterKey)
|
||||
local jobs = rcall("ZRANGEBYSCORE", delayedKey, 0, (timestamp + 1) * 0x1000, "LIMIT", 0, 1000)
|
||||
if (#jobs > 0) then
|
||||
rcall("ZREM", delayedKey, unpack(jobs))
|
||||
for _, jobId in ipairs(jobs) do
|
||||
local jobKey = prefix .. jobId
|
||||
local priority =
|
||||
tonumber(rcall("HGET", jobKey, "priority")) or 0
|
||||
if priority == 0 then
|
||||
-- LIFO or FIFO
|
||||
rcall("LPUSH", targetKey, jobId)
|
||||
else
|
||||
local score = getPriorityScore(priority, priorityCounterKey)
|
||||
rcall("ZADD", prioritizedKey, score, jobId)
|
||||
end
|
||||
-- Emit waiting event
|
||||
rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId",
|
||||
jobId, "prev", "delayed")
|
||||
rcall("HSET", jobKey, "delay", 0)
|
||||
end
|
||||
if not paused then
|
||||
addPriorityMarkerIfNeeded(targetKey)
|
||||
end
|
||||
end
|
||||
end
|
||||
local jobKey = KEYS[5]
|
||||
if rcall("EXISTS", jobKey) == 1 then
|
||||
local delayedKey = KEYS[4]
|
||||
if ARGV[5] ~= "0" then
|
||||
local lockKey = jobKey .. ':lock'
|
||||
if rcall("GET", lockKey) == ARGV[5] then
|
||||
rcall("DEL", lockKey)
|
||||
else
|
||||
return -2
|
||||
end
|
||||
end
|
||||
local jobId = ARGV[4]
|
||||
local score = tonumber(ARGV[3])
|
||||
local delayedTimestamp = (score / 0x1000)
|
||||
local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
|
||||
if numRemovedElements < 1 then
|
||||
return -3
|
||||
end
|
||||
rcall("HSET", jobKey, "delay", ARGV[6])
|
||||
local maxEvents = rcall("HGET", KEYS[8], "opts.maxLenEvents") or 10000
|
||||
rcall("ZADD", delayedKey, score, jobId)
|
||||
rcall("XADD", KEYS[6], "MAXLEN", "~", maxEvents, "*", "event", "delayed",
|
||||
"jobId", jobId, "delay", delayedTimestamp)
|
||||
-- Check if we need to push a marker job to wake up sleeping workers.
|
||||
local target = getTargetQueueList(KEYS[8], KEYS[1], KEYS[7])
|
||||
addDelayMarkerIfNeeded(target, delayedKey)
|
||||
return 0
|
||||
else
|
||||
return -1
|
||||
end
|
||||
`;
|
||||
exports.moveToDelayed = {
|
||||
name: 'moveToDelayed',
|
||||
content,
|
||||
keys: 8,
|
||||
};
|
||||
//# sourceMappingURL=moveToDelayed-8.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"moveToDelayed-8.js","sourceRoot":"","sources":["../../../src/scripts/moveToDelayed-8.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA4Jf,CAAC;AACW,QAAA,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
||||
+675
@@ -0,0 +1,675 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.moveToFinished = void 0;
|
||||
const content = `--[[
|
||||
Move job from active to a finished status (completed o failed)
|
||||
A job can only be moved to completed if it was active.
|
||||
The job must be locked before it can be moved to a finished status,
|
||||
and the lock must be released in this script.
|
||||
Input:
|
||||
KEYS[1] wait key
|
||||
KEYS[2] active key
|
||||
KEYS[3] prioritized key
|
||||
KEYS[4] event stream key
|
||||
KEYS[5] stalled key
|
||||
-- Rate limiting
|
||||
KEYS[6] rate limiter key
|
||||
KEYS[7] delayed key
|
||||
KEYS[8] paused key
|
||||
KEYS[9] meta key
|
||||
KEYS[10] pc priority counter
|
||||
KEYS[11] completed/failed key
|
||||
KEYS[12] jobId key
|
||||
KEYS[13] metrics key
|
||||
ARGV[1] jobId
|
||||
ARGV[2] timestamp
|
||||
ARGV[3] msg property returnvalue / failedReason
|
||||
ARGV[4] return value / failed reason
|
||||
ARGV[5] target (completed/failed)
|
||||
ARGV[6] event data (? maybe just send jobid).
|
||||
ARGV[7] fetch next?
|
||||
ARGV[8] keys prefix
|
||||
ARGV[9] opts
|
||||
opts - token - lock token
|
||||
opts - keepJobs
|
||||
opts - lockDuration - lock duration in milliseconds
|
||||
opts - attempts max attempts
|
||||
opts - attemptsMade
|
||||
opts - maxMetricsSize
|
||||
opts - fpof - fail parent on fail
|
||||
opts - rdof - remove dependency on fail
|
||||
Output:
|
||||
0 OK
|
||||
-1 Missing key.
|
||||
-2 Missing lock.
|
||||
-3 Job not in active set
|
||||
-4 Job has pending dependencies
|
||||
-6 Lock is not owned by this client
|
||||
Events:
|
||||
'completed/failed'
|
||||
]]
|
||||
local rcall = redis.call
|
||||
--- Includes
|
||||
--[[
|
||||
Functions to collect metrics based on a current and previous count of jobs.
|
||||
Granualarity is fixed at 1 minute.
|
||||
]]
|
||||
--[[
|
||||
Function to loop in batches.
|
||||
Just a bit of warning, some commands as ZREM
|
||||
could receive a maximum of 7000 parameters per call.
|
||||
]]
|
||||
local function batches(n, batchSize)
|
||||
local i = 0
|
||||
return function()
|
||||
local from = i * batchSize + 1
|
||||
i = i + 1
|
||||
if (from <= n) then
|
||||
local to = math.min(from + batchSize - 1, n)
|
||||
return from, to
|
||||
end
|
||||
end
|
||||
end
|
||||
local function collectMetrics(metaKey, dataPointsList, maxDataPoints,
|
||||
timestamp)
|
||||
-- Increment current count
|
||||
local count = rcall("HINCRBY", metaKey, "count", 1) - 1
|
||||
-- Compute how many data points we need to add to the list, N.
|
||||
local prevTS = rcall("HGET", metaKey, "prevTS")
|
||||
if not prevTS then
|
||||
-- If prevTS is nil, set it to the current timestamp
|
||||
rcall("HSET", metaKey, "prevTS", timestamp, "prevCount", 0)
|
||||
return
|
||||
end
|
||||
local N = math.floor((timestamp - prevTS) / 60000)
|
||||
if N > 0 then
|
||||
local delta = count - rcall("HGET", metaKey, "prevCount")
|
||||
-- If N > 1, add N-1 zeros to the list
|
||||
if N > 1 then
|
||||
local points = {}
|
||||
points[1] = delta
|
||||
for i = 2, N do
|
||||
points[i] = 0
|
||||
end
|
||||
for from, to in batches(#points, 7000) do
|
||||
rcall("LPUSH", dataPointsList, unpack(points, from, to))
|
||||
end
|
||||
else
|
||||
-- LPUSH delta to the list
|
||||
rcall("LPUSH", dataPointsList, delta)
|
||||
end
|
||||
-- LTRIM to keep list to its max size
|
||||
rcall("LTRIM", dataPointsList, 0, maxDataPoints - 1)
|
||||
-- update prev count with current count
|
||||
rcall("HSET", metaKey, "prevCount", count, "prevTS", timestamp)
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Function to return the next delayed job timestamp.
|
||||
]]
|
||||
local function getNextDelayedTimestamp(delayedKey)
|
||||
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
|
||||
if #result then
|
||||
local nextTimestamp = tonumber(result[2])
|
||||
if (nextTimestamp ~= nil) then
|
||||
nextTimestamp = nextTimestamp / 0x1000
|
||||
end
|
||||
return nextTimestamp
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Function to move job from prioritized state to active.
|
||||
]]
|
||||
local function moveJobFromPriorityToActive(priorityKey, activeKey, priorityCounterKey)
|
||||
local prioritizedJob = rcall("ZPOPMIN", priorityKey)
|
||||
if #prioritizedJob > 0 then
|
||||
rcall("LPUSH", activeKey, prioritizedJob[1])
|
||||
return prioritizedJob[1]
|
||||
else
|
||||
rcall("DEL", priorityCounterKey)
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Function to move job from wait state to active.
|
||||
Input:
|
||||
keys[1] wait key
|
||||
keys[2] active key
|
||||
keys[3] prioritized key
|
||||
keys[4] stream events key
|
||||
keys[5] stalled key
|
||||
-- Rate limiting
|
||||
keys[6] rate limiter key
|
||||
keys[7] delayed key
|
||||
keys[8] paused key
|
||||
keys[9] meta key
|
||||
keys[10] pc priority counter
|
||||
opts - token - lock token
|
||||
opts - lockDuration
|
||||
opts - limiter
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Function to push back job considering priority in front of same prioritized jobs.
|
||||
]]
|
||||
local function pushBackJobWithPriority(prioritizedKey, priority, jobId)
|
||||
-- in order to put it at front of same prioritized jobs
|
||||
-- we consider prioritized counter as 0
|
||||
local score = priority * 0x100000000
|
||||
rcall("ZADD", prioritizedKey, score, jobId)
|
||||
end
|
||||
local function prepareJobForProcessing(keys, keyPrefix, targetKey, jobId, processedOn,
|
||||
maxJobs, expireTime, opts)
|
||||
local jobKey = keyPrefix .. jobId
|
||||
-- Check if we need to perform rate limiting.
|
||||
if maxJobs then
|
||||
local rateLimiterKey = keys[6];
|
||||
-- check if we exceeded rate limit, we need to remove the job and return expireTime
|
||||
if expireTime > 0 then
|
||||
-- remove from active queue and add back to the wait list
|
||||
rcall("LREM", keys[2], 1, jobId)
|
||||
local priority = tonumber(rcall("HGET", jobKey, "priority")) or 0
|
||||
if priority == 0 then
|
||||
rcall("RPUSH", targetKey, jobId)
|
||||
else
|
||||
pushBackJobWithPriority(keys[3], priority, jobId)
|
||||
end
|
||||
-- Return when we can process more jobs
|
||||
return {0, 0, expireTime, 0}
|
||||
end
|
||||
local jobCounter = tonumber(rcall("INCR", rateLimiterKey))
|
||||
if jobCounter == 1 then
|
||||
local limiterDuration = opts['limiter'] and opts['limiter']['duration']
|
||||
local integerDuration = math.floor(math.abs(limiterDuration))
|
||||
rcall("PEXPIRE", rateLimiterKey, integerDuration)
|
||||
end
|
||||
end
|
||||
local lockKey = jobKey .. ':lock'
|
||||
-- get a lock
|
||||
if opts['token'] ~= "0" then
|
||||
rcall("SET", lockKey, opts['token'], "PX", opts['lockDuration'])
|
||||
end
|
||||
rcall("XADD", keys[4], "*", "event", "active", "jobId", jobId, "prev", "waiting")
|
||||
rcall("HSET", jobKey, "processedOn", processedOn)
|
||||
rcall("HINCRBY", jobKey, "attemptsMade", 1)
|
||||
return {rcall("HGETALL", jobKey), jobId, 0, 0} -- get job data
|
||||
end
|
||||
--[[
|
||||
Function to recursively move from waitingChildren to failed.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Validate and move parent to active if needed.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Add delay marker if needed.
|
||||
]]
|
||||
-- Includes
|
||||
local function addDelayMarkerIfNeeded(targetKey, delayedKey)
|
||||
local waitLen = rcall("LLEN", targetKey)
|
||||
if waitLen <= 1 then
|
||||
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
|
||||
if nextTimestamp ~= nil then
|
||||
-- Check if there is already a marker with older timestamp
|
||||
-- if there is, we need to replace it.
|
||||
if waitLen == 1 then
|
||||
local marker = rcall("LINDEX", targetKey, 0)
|
||||
local oldTimestamp = tonumber(marker:sub(3))
|
||||
if oldTimestamp and oldTimestamp > nextTimestamp then
|
||||
rcall("LSET", targetKey, 0, "0:" .. nextTimestamp)
|
||||
end
|
||||
else
|
||||
-- if there is no marker, then we need to add one
|
||||
rcall("LPUSH", targetKey, "0:" .. nextTimestamp)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Function to add job considering priority.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Function priority marker to wait if needed
|
||||
in order to wake up our workers and to respect priority
|
||||
order as much as possible
|
||||
]]
|
||||
local function addPriorityMarkerIfNeeded(waitKey)
|
||||
local waitLen = rcall("LLEN", waitKey)
|
||||
if waitLen == 0 then
|
||||
rcall("LPUSH", waitKey, "0:0")
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Function to get priority score.
|
||||
]]
|
||||
local function getPriorityScore(priority, priorityCounterKey)
|
||||
local prioCounter = rcall("INCR", priorityCounterKey)
|
||||
return priority * 0x100000000 + prioCounter % 0x100000000
|
||||
end
|
||||
local function addJobWithPriority(waitKey, prioritizedKey, priority, paused, jobId, priorityCounterKey)
|
||||
local score = getPriorityScore(priority, priorityCounterKey)
|
||||
rcall("ZADD", prioritizedKey, score, jobId)
|
||||
if not paused then
|
||||
addPriorityMarkerIfNeeded(waitKey)
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Function to check for the meta.paused key to decide if we are paused or not
|
||||
(since an empty list and !EXISTS are not really the same).
|
||||
]]
|
||||
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
|
||||
if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
|
||||
return waitKey, false
|
||||
else
|
||||
return pausedKey, true
|
||||
end
|
||||
end
|
||||
local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
|
||||
local isParentActive = rcall("ZSCORE", parentQueueKey .. ":waiting-children", parentId)
|
||||
if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then
|
||||
rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
|
||||
local parentWaitKey = parentQueueKey .. ":wait"
|
||||
local parentTarget, paused = getTargetQueueList(parentQueueKey .. ":meta", parentWaitKey,
|
||||
parentQueueKey .. ":paused")
|
||||
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
||||
local priority = tonumber(jobAttributes[1]) or 0
|
||||
local delay = tonumber(jobAttributes[2]) or 0
|
||||
if delay > 0 then
|
||||
local delayedTimestamp = tonumber(timestamp) + delay
|
||||
local score = delayedTimestamp * 0x1000
|
||||
local parentDelayedKey = parentQueueKey .. ":delayed"
|
||||
rcall("ZADD", parentDelayedKey, score, parentId)
|
||||
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId,
|
||||
"delay", delayedTimestamp)
|
||||
addDelayMarkerIfNeeded(parentTarget, parentDelayedKey)
|
||||
else
|
||||
if priority == 0 then
|
||||
rcall("RPUSH", parentTarget, parentId)
|
||||
else
|
||||
addJobWithPriority(parentWaitKey, parentQueueKey .. ":prioritized", priority, paused,
|
||||
parentId, parentQueueKey .. ":pc")
|
||||
end
|
||||
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId,
|
||||
"prev", "waiting-children")
|
||||
end
|
||||
end
|
||||
end
|
||||
local function moveParentFromWaitingChildrenToFailed( parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
|
||||
if rcall("ZREM", parentQueueKey .. ":waiting-children", parentId) == 1 then
|
||||
rcall("ZADD", parentQueueKey .. ":failed", timestamp, parentId)
|
||||
local failedReason = "child " .. jobIdKey .. " failed"
|
||||
rcall("HMSET", parentKey, "failedReason", failedReason, "finishedOn", timestamp)
|
||||
rcall("XADD", parentQueueKey .. ":events", "*", "event", "failed", "jobId", parentId, "failedReason",
|
||||
failedReason, "prev", "waiting-children")
|
||||
local rawParentData = rcall("HGET", parentKey, "parent")
|
||||
if rawParentData ~= false then
|
||||
local parentData = cjson.decode(rawParentData)
|
||||
if parentData['fpof'] then
|
||||
moveParentFromWaitingChildrenToFailed(
|
||||
parentData['queueKey'],
|
||||
parentData['queueKey'] .. ':' .. parentData['id'],
|
||||
parentData['id'],
|
||||
parentKey,
|
||||
timestamp
|
||||
)
|
||||
elseif parentData['rdof'] then
|
||||
local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
||||
local grandParentDependenciesSet = grandParentKey .. ":dependencies"
|
||||
if rcall("SREM", grandParentDependenciesSet, parentKey) == 1 then
|
||||
moveParentToWaitIfNeeded(parentData['queueKey'], grandParentDependenciesSet,
|
||||
grandParentKey, parentData['id'], timestamp)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Updates the delay set, by moving delayed jobs that should
|
||||
be processed now to "wait".
|
||||
Events:
|
||||
'waiting'
|
||||
]]
|
||||
-- Includes
|
||||
-- Try to get as much as 1000 jobs at once
|
||||
local function promoteDelayedJobs(delayedKey, waitKey, targetKey, prioritizedKey,
|
||||
eventStreamKey, prefix, timestamp, paused, priorityCounterKey)
|
||||
local jobs = rcall("ZRANGEBYSCORE", delayedKey, 0, (timestamp + 1) * 0x1000, "LIMIT", 0, 1000)
|
||||
if (#jobs > 0) then
|
||||
rcall("ZREM", delayedKey, unpack(jobs))
|
||||
for _, jobId in ipairs(jobs) do
|
||||
local jobKey = prefix .. jobId
|
||||
local priority =
|
||||
tonumber(rcall("HGET", jobKey, "priority")) or 0
|
||||
if priority == 0 then
|
||||
-- LIFO or FIFO
|
||||
rcall("LPUSH", targetKey, jobId)
|
||||
else
|
||||
local score = getPriorityScore(priority, priorityCounterKey)
|
||||
rcall("ZADD", prioritizedKey, score, jobId)
|
||||
end
|
||||
-- Emit waiting event
|
||||
rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId",
|
||||
jobId, "prev", "delayed")
|
||||
rcall("HSET", jobKey, "delay", 0)
|
||||
end
|
||||
if not paused then
|
||||
addPriorityMarkerIfNeeded(targetKey)
|
||||
end
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Functions to remove jobs by max age.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Function to remove job.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Check if this job has a parent. If so we will just remove it from
|
||||
the parent child list, but if it is the last child we should move the parent to "wait/paused"
|
||||
which requires code from "moveToFinished"
|
||||
]]
|
||||
--[[
|
||||
Functions to destructure job key.
|
||||
Just a bit of warning, these functions may be a bit slow and affect performance significantly.
|
||||
]]
|
||||
local getJobIdFromKey = function (jobKey)
|
||||
return string.match(jobKey, ".*:(.*)")
|
||||
end
|
||||
local getJobKeyPrefix = function (jobKey, jobId)
|
||||
return string.sub(jobKey, 0, #jobKey - #jobId)
|
||||
end
|
||||
local function moveParentToWait(parentPrefix, parentId, emitEvent)
|
||||
local parentTarget = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "wait", parentPrefix .. "paused")
|
||||
rcall("RPUSH", parentTarget, parentId)
|
||||
if emitEvent then
|
||||
local parentEventStream = parentPrefix .. "events"
|
||||
rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
|
||||
end
|
||||
end
|
||||
local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey)
|
||||
if parentKey then
|
||||
local parentDependenciesKey = parentKey .. ":dependencies"
|
||||
local result = rcall("SREM", parentDependenciesKey, jobKey)
|
||||
if result > 0 then
|
||||
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
|
||||
if pendingDependencies == 0 then
|
||||
local parentId = getJobIdFromKey(parentKey)
|
||||
local parentPrefix = getJobKeyPrefix(parentKey, parentId)
|
||||
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
|
||||
if numRemovedElements == 1 then
|
||||
if hard then
|
||||
if parentPrefix == baseKey then
|
||||
removeParentDependencyKey(parentKey, hard, nil, baseKey)
|
||||
rcall("DEL", parentKey, parentKey .. ':logs',
|
||||
parentKey .. ':dependencies', parentKey .. ':processed')
|
||||
else
|
||||
moveParentToWait(parentPrefix, parentId)
|
||||
end
|
||||
else
|
||||
moveParentToWait(parentPrefix, parentId, true)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
else
|
||||
local missedParentKey = rcall("HGET", jobKey, "parentKey")
|
||||
if( (type(missedParentKey) == "string") and missedParentKey ~= "" and (rcall("EXISTS", missedParentKey) == 1)) then
|
||||
local parentDependenciesKey = missedParentKey .. ":dependencies"
|
||||
local result = rcall("SREM", parentDependenciesKey, jobKey)
|
||||
if result > 0 then
|
||||
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
|
||||
if pendingDependencies == 0 then
|
||||
local parentId = getJobIdFromKey(missedParentKey)
|
||||
local parentPrefix = getJobKeyPrefix(missedParentKey, parentId)
|
||||
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
|
||||
if numRemovedElements == 1 then
|
||||
if hard then
|
||||
if parentPrefix == baseKey then
|
||||
removeParentDependencyKey(missedParentKey, hard, nil, baseKey)
|
||||
rcall("DEL", missedParentKey, missedParentKey .. ':logs',
|
||||
missedParentKey .. ':dependencies', missedParentKey .. ':processed')
|
||||
else
|
||||
moveParentToWait(parentPrefix, parentId)
|
||||
end
|
||||
else
|
||||
moveParentToWait(parentPrefix, parentId, true)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
local function removeJob(jobId, hard, baseKey)
|
||||
local jobKey = baseKey .. jobId
|
||||
removeParentDependencyKey(jobKey, hard, nil, baseKey)
|
||||
rcall("DEL", jobKey, jobKey .. ':logs',
|
||||
jobKey .. ':dependencies', jobKey .. ':processed')
|
||||
end
|
||||
local function removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
|
||||
local start = timestamp - maxAge * 1000
|
||||
local jobIds = rcall("ZREVRANGEBYSCORE", targetSet, start, "-inf")
|
||||
for i, jobId in ipairs(jobIds) do
|
||||
removeJob(jobId, false, prefix)
|
||||
end
|
||||
rcall("ZREMRANGEBYSCORE", targetSet, "-inf", start)
|
||||
end
|
||||
--[[
|
||||
Functions to remove jobs by max count.
|
||||
]]
|
||||
-- Includes
|
||||
local function removeJobsByMaxCount(maxCount, targetSet, prefix)
|
||||
local start = maxCount
|
||||
local jobIds = rcall("ZREVRANGE", targetSet, start, -1)
|
||||
for i, jobId in ipairs(jobIds) do
|
||||
removeJob(jobId, false, prefix)
|
||||
end
|
||||
rcall("ZREMRANGEBYRANK", targetSet, 0, -(maxCount + 1))
|
||||
end
|
||||
--[[
|
||||
Function to trim events, default 10000.
|
||||
]]
|
||||
local function trimEvents(metaKey, eventStreamKey)
|
||||
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
||||
if maxEvents ~= false then
|
||||
rcall("XTRIM", eventStreamKey, "MAXLEN", "~", maxEvents)
|
||||
else
|
||||
rcall("XTRIM", eventStreamKey, "MAXLEN", "~", 10000)
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Validate and move or add dependencies to parent.
|
||||
]]
|
||||
-- Includes
|
||||
local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
|
||||
parentId, jobIdKey, returnvalue, timestamp )
|
||||
local processedSet = parentKey .. ":processed"
|
||||
rcall("HSET", processedSet, jobIdKey, returnvalue)
|
||||
moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
|
||||
end
|
||||
local function getRateLimitTTL(maxJobs, rateLimiterKey)
|
||||
if maxJobs and maxJobs <= tonumber(rcall("GET", rateLimiterKey) or 0) then
|
||||
local pttl = rcall("PTTL", rateLimiterKey)
|
||||
if pttl == 0 then
|
||||
rcall("DEL", rateLimiterKey)
|
||||
end
|
||||
if pttl > 0 then
|
||||
return pttl
|
||||
end
|
||||
end
|
||||
return 0
|
||||
end
|
||||
local jobIdKey = KEYS[12]
|
||||
if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
||||
local opts = cmsgpack.unpack(ARGV[9])
|
||||
local token = opts['token']
|
||||
local attempts = opts['attempts']
|
||||
local attemptsMade = opts['attemptsMade']
|
||||
local maxMetricsSize = opts['maxMetricsSize']
|
||||
local maxCount = opts['keepJobs']['count']
|
||||
local maxAge = opts['keepJobs']['age']
|
||||
if token ~= "0" then
|
||||
local lockKey = jobIdKey .. ':lock'
|
||||
local lockToken = rcall("GET", lockKey)
|
||||
if lockToken == token then
|
||||
rcall("DEL", lockKey)
|
||||
rcall("SREM", KEYS[5], ARGV[1])
|
||||
else
|
||||
if lockToken then
|
||||
-- Lock exists but token does not match
|
||||
return -6
|
||||
else
|
||||
-- Lock is missing completely
|
||||
return -2
|
||||
end
|
||||
end
|
||||
end
|
||||
if rcall("SCARD", jobIdKey .. ":dependencies") ~= 0 then -- // Make sure it does not have pending dependencies
|
||||
return -4
|
||||
end
|
||||
local parentReferences = rcall("HMGET", jobIdKey, "parentKey", "parent")
|
||||
local parentKey = parentReferences[1] or ""
|
||||
local parentId = ""
|
||||
local parentQueueKey = ""
|
||||
if parentReferences[2] ~= false then
|
||||
local jsonDecodedParent = cjson.decode(parentReferences[2])
|
||||
parentId = jsonDecodedParent['id']
|
||||
parentQueueKey = jsonDecodedParent['queueKey']
|
||||
end
|
||||
local jobId = ARGV[1]
|
||||
local timestamp = ARGV[2]
|
||||
-- Remove from active list (if not active we shall return error)
|
||||
local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
|
||||
if (numRemovedElements < 1) then return -3 end
|
||||
-- Trim events before emiting them to avoid trimming events emitted in this script
|
||||
trimEvents(KEYS[9], KEYS[4])
|
||||
-- If job has a parent we need to
|
||||
-- 1) remove this job id from parents dependencies
|
||||
-- 2) move the job Id to parent "processed" set
|
||||
-- 3) push the results into parent "results" list
|
||||
-- 4) if parent's dependencies is empty, then move parent to "wait/paused". Note it may be a different queue!.
|
||||
if parentId == "" and parentKey ~= "" then
|
||||
parentId = getJobIdFromKey(parentKey)
|
||||
parentQueueKey = getJobKeyPrefix(parentKey, ":" .. parentId)
|
||||
end
|
||||
if parentId ~= "" then
|
||||
if ARGV[5] == "completed" then
|
||||
local dependenciesSet = parentKey .. ":dependencies"
|
||||
if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
|
||||
updateParentDepsIfNeeded(parentKey, parentQueueKey,
|
||||
dependenciesSet, parentId, jobIdKey,
|
||||
ARGV[4], timestamp)
|
||||
end
|
||||
else
|
||||
if opts['fpof'] then
|
||||
moveParentFromWaitingChildrenToFailed(parentQueueKey, parentKey,
|
||||
parentId, jobIdKey, timestamp)
|
||||
elseif opts['rdof'] then
|
||||
local dependenciesSet = parentKey .. ":dependencies"
|
||||
if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
|
||||
moveParentToWaitIfNeeded(parentQueueKey, dependenciesSet,
|
||||
parentKey, parentId, timestamp)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
-- Remove job?
|
||||
if maxCount ~= 0 then
|
||||
local targetSet = KEYS[11]
|
||||
-- Add to complete/failed set
|
||||
rcall("ZADD", targetSet, timestamp, jobId)
|
||||
rcall("HMSET", jobIdKey, ARGV[3], ARGV[4], "finishedOn", timestamp)
|
||||
-- "returnvalue" / "failedReason" and "finishedOn"
|
||||
-- Remove old jobs?
|
||||
local prefix = ARGV[8]
|
||||
if maxAge ~= nil then
|
||||
removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
|
||||
end
|
||||
if maxCount ~= nil and maxCount > 0 then
|
||||
removeJobsByMaxCount(maxCount, targetSet, prefix)
|
||||
end
|
||||
else
|
||||
rcall("DEL", jobIdKey, jobIdKey .. ':logs', jobIdKey .. ':processed')
|
||||
if parentKey ~= "" then
|
||||
removeParentDependencyKey(jobIdKey, false, parentKey)
|
||||
end
|
||||
end
|
||||
rcall("XADD", KEYS[4], "*", "event", ARGV[5], "jobId", jobId, ARGV[3],
|
||||
ARGV[4])
|
||||
if ARGV[5] == "failed" then
|
||||
if tonumber(attemptsMade) >= tonumber(attempts) then
|
||||
rcall("XADD", KEYS[4], "*", "event", "retries-exhausted", "jobId",
|
||||
jobId, "attemptsMade", attemptsMade)
|
||||
end
|
||||
end
|
||||
-- Collect metrics
|
||||
if maxMetricsSize ~= "" then
|
||||
collectMetrics(KEYS[13], KEYS[13] .. ':data', maxMetricsSize, timestamp)
|
||||
end
|
||||
-- Try to get next job to avoid an extra roundtrip if the queue is not closing,
|
||||
-- and not rate limited.
|
||||
if (ARGV[7] == "1") then
|
||||
local target, paused = getTargetQueueList(KEYS[9], KEYS[1], KEYS[8])
|
||||
-- Check if there are delayed jobs that can be promoted
|
||||
promoteDelayedJobs(KEYS[7], KEYS[1], target, KEYS[3],
|
||||
KEYS[4], ARGV[8], timestamp, paused, KEYS[10])
|
||||
local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
|
||||
-- Check if we are rate limited first.
|
||||
local expireTime = getRateLimitTTL(maxJobs, KEYS[6])
|
||||
if expireTime > 0 then return {0, 0, expireTime, 0} end
|
||||
-- paused queue
|
||||
if paused then return {0, 0, 0, 0} end
|
||||
jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
|
||||
if jobId then
|
||||
if string.sub(jobId, 1, 2) == "0:" then
|
||||
rcall("LREM", KEYS[2], 1, jobId)
|
||||
-- If jobId is special ID 0:delay (delay greater than 0), then there is no job to process
|
||||
-- but if ID is 0:0, then there is at least 1 prioritized job to process
|
||||
if jobId == "0:0" then
|
||||
jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2], KEYS[10])
|
||||
return prepareJobForProcessing(KEYS, ARGV[8], target, jobId, timestamp,
|
||||
maxJobs, expireTime, opts)
|
||||
end
|
||||
else
|
||||
return prepareJobForProcessing(KEYS, ARGV[8], target, jobId, timestamp, maxJobs,
|
||||
expireTime, opts)
|
||||
end
|
||||
else
|
||||
jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2], KEYS[10])
|
||||
if jobId then
|
||||
return prepareJobForProcessing(KEYS, ARGV[8], target, jobId, timestamp, maxJobs,
|
||||
expireTime, opts)
|
||||
end
|
||||
end
|
||||
-- Return the timestamp for the next delayed job if any.
|
||||
local nextTimestamp = getNextDelayedTimestamp(KEYS[7])
|
||||
if nextTimestamp ~= nil then
|
||||
-- The result is guaranteed to be positive, since the
|
||||
-- ZRANGEBYSCORE command would have return a job otherwise.
|
||||
return {0, 0, 0, nextTimestamp}
|
||||
end
|
||||
end
|
||||
local waitLen = rcall("LLEN", KEYS[1])
|
||||
if waitLen == 0 then
|
||||
local activeLen = rcall("LLEN", KEYS[2])
|
||||
if activeLen == 0 then
|
||||
local prioritizedLen = rcall("ZCARD", KEYS[3])
|
||||
if prioritizedLen == 0 then
|
||||
rcall("XADD", KEYS[4], "*", "event", "drained")
|
||||
end
|
||||
end
|
||||
end
|
||||
return 0
|
||||
else
|
||||
return -1
|
||||
end
|
||||
`;
|
||||
exports.moveToFinished = {
|
||||
name: 'moveToFinished',
|
||||
content,
|
||||
keys: 13,
|
||||
};
|
||||
//# sourceMappingURL=moveToFinished-13.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"moveToFinished-13.js","sourceRoot":"","sources":["../../../src/scripts/moveToFinished-13.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAypBf,CAAC;AACW,QAAA,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.moveToWaitingChildren = void 0;
|
||||
const content = `--[[
|
||||
Moves job from active to waiting children set.
|
||||
Input:
|
||||
KEYS[1] lock key
|
||||
KEYS[2] active key
|
||||
KEYS[3] waitChildrenKey key
|
||||
KEYS[4] job key
|
||||
ARGV[1] token
|
||||
ARGV[2] child key
|
||||
ARGV[3] timestamp
|
||||
ARGV[4] the id of the job
|
||||
Output:
|
||||
0 - OK
|
||||
1 - There are not pending dependencies.
|
||||
-1 - Missing job.
|
||||
-2 - Missing lock
|
||||
-3 - Job not in active set
|
||||
]]
|
||||
local rcall = redis.call
|
||||
local function moveToWaitingChildren (activeKey, waitingChildrenKey, jobId, timestamp, lockKey, token)
|
||||
if token ~= "0" then
|
||||
if rcall("GET", lockKey) == token then
|
||||
rcall("DEL", lockKey)
|
||||
else
|
||||
return -2
|
||||
end
|
||||
end
|
||||
local score = tonumber(timestamp)
|
||||
local numRemovedElements = rcall("LREM", activeKey, -1, jobId)
|
||||
if(numRemovedElements < 1) then
|
||||
return -3
|
||||
end
|
||||
rcall("ZADD", waitingChildrenKey, score, jobId)
|
||||
return 0
|
||||
end
|
||||
if rcall("EXISTS", KEYS[4]) == 1 then
|
||||
if ARGV[2] ~= "" then
|
||||
if rcall("SISMEMBER", KEYS[4] .. ":dependencies", ARGV[2]) ~= 0 then
|
||||
return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1], ARGV[1])
|
||||
end
|
||||
return 1
|
||||
else
|
||||
if rcall("SCARD", KEYS[4] .. ":dependencies") ~= 0 then
|
||||
return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1], ARGV[1])
|
||||
end
|
||||
return 1
|
||||
end
|
||||
end
|
||||
return -1
|
||||
`;
|
||||
exports.moveToWaitingChildren = {
|
||||
name: 'moveToWaitingChildren',
|
||||
content,
|
||||
keys: 4,
|
||||
};
|
||||
//# sourceMappingURL=moveToWaitingChildren-4.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"moveToWaitingChildren-4.js","sourceRoot":"","sources":["../../../src/scripts/moveToWaitingChildren-4.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAiDf,CAAC;AACW,QAAA,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
||||
+248
@@ -0,0 +1,248 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.obliterate = void 0;
|
||||
const content = `--[[
|
||||
Completely obliterates a queue and all of its contents
|
||||
Input:
|
||||
KEYS[1] meta
|
||||
KEYS[2] base
|
||||
ARGV[1] count
|
||||
ARGV[2] force
|
||||
]]
|
||||
-- This command completely destroys a queue including all of its jobs, current or past
|
||||
-- leaving no trace of its existence. Since this script needs to iterate to find all the job
|
||||
-- keys, consider that this call may be slow for very large queues.
|
||||
-- The queue needs to be "paused" or it will return an error
|
||||
-- If the queue has currently active jobs then the script by default will return error,
|
||||
-- however this behaviour can be overrided using the 'force' option.
|
||||
local maxCount = tonumber(ARGV[1])
|
||||
local baseKey = KEYS[2]
|
||||
local rcall = redis.call
|
||||
-- Includes
|
||||
--[[
|
||||
Functions to remove jobs.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Function to remove job.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Check if this job has a parent. If so we will just remove it from
|
||||
the parent child list, but if it is the last child we should move the parent to "wait/paused"
|
||||
which requires code from "moveToFinished"
|
||||
]]
|
||||
--[[
|
||||
Functions to destructure job key.
|
||||
Just a bit of warning, these functions may be a bit slow and affect performance significantly.
|
||||
]]
|
||||
local getJobIdFromKey = function (jobKey)
|
||||
return string.match(jobKey, ".*:(.*)")
|
||||
end
|
||||
local getJobKeyPrefix = function (jobKey, jobId)
|
||||
return string.sub(jobKey, 0, #jobKey - #jobId)
|
||||
end
|
||||
--[[
|
||||
Function to check for the meta.paused key to decide if we are paused or not
|
||||
(since an empty list and !EXISTS are not really the same).
|
||||
]]
|
||||
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
|
||||
if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
|
||||
return waitKey, false
|
||||
else
|
||||
return pausedKey, true
|
||||
end
|
||||
end
|
||||
local function moveParentToWait(parentPrefix, parentId, emitEvent)
|
||||
local parentTarget = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "wait", parentPrefix .. "paused")
|
||||
rcall("RPUSH", parentTarget, parentId)
|
||||
if emitEvent then
|
||||
local parentEventStream = parentPrefix .. "events"
|
||||
rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
|
||||
end
|
||||
end
|
||||
local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey)
|
||||
if parentKey then
|
||||
local parentDependenciesKey = parentKey .. ":dependencies"
|
||||
local result = rcall("SREM", parentDependenciesKey, jobKey)
|
||||
if result > 0 then
|
||||
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
|
||||
if pendingDependencies == 0 then
|
||||
local parentId = getJobIdFromKey(parentKey)
|
||||
local parentPrefix = getJobKeyPrefix(parentKey, parentId)
|
||||
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
|
||||
if numRemovedElements == 1 then
|
||||
if hard then
|
||||
if parentPrefix == baseKey then
|
||||
removeParentDependencyKey(parentKey, hard, nil, baseKey)
|
||||
rcall("DEL", parentKey, parentKey .. ':logs',
|
||||
parentKey .. ':dependencies', parentKey .. ':processed')
|
||||
else
|
||||
moveParentToWait(parentPrefix, parentId)
|
||||
end
|
||||
else
|
||||
moveParentToWait(parentPrefix, parentId, true)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
else
|
||||
local missedParentKey = rcall("HGET", jobKey, "parentKey")
|
||||
if( (type(missedParentKey) == "string") and missedParentKey ~= "" and (rcall("EXISTS", missedParentKey) == 1)) then
|
||||
local parentDependenciesKey = missedParentKey .. ":dependencies"
|
||||
local result = rcall("SREM", parentDependenciesKey, jobKey)
|
||||
if result > 0 then
|
||||
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
|
||||
if pendingDependencies == 0 then
|
||||
local parentId = getJobIdFromKey(missedParentKey)
|
||||
local parentPrefix = getJobKeyPrefix(missedParentKey, parentId)
|
||||
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
|
||||
if numRemovedElements == 1 then
|
||||
if hard then
|
||||
if parentPrefix == baseKey then
|
||||
removeParentDependencyKey(missedParentKey, hard, nil, baseKey)
|
||||
rcall("DEL", missedParentKey, missedParentKey .. ':logs',
|
||||
missedParentKey .. ':dependencies', missedParentKey .. ':processed')
|
||||
else
|
||||
moveParentToWait(parentPrefix, parentId)
|
||||
end
|
||||
else
|
||||
moveParentToWait(parentPrefix, parentId, true)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
local function removeJob(jobId, hard, baseKey)
|
||||
local jobKey = baseKey .. jobId
|
||||
removeParentDependencyKey(jobKey, hard, nil, baseKey)
|
||||
rcall("DEL", jobKey, jobKey .. ':logs',
|
||||
jobKey .. ':dependencies', jobKey .. ':processed')
|
||||
end
|
||||
local function removeJobs(keys, hard, baseKey, max)
|
||||
for i, key in ipairs(keys) do
|
||||
removeJob(key, hard, baseKey)
|
||||
end
|
||||
return max - #keys
|
||||
end
|
||||
--[[
|
||||
Functions to remove jobs.
|
||||
]]
|
||||
-- Includes
|
||||
local function getListItems(keyName, max)
|
||||
return rcall('LRANGE', keyName, 0, max - 1)
|
||||
end
|
||||
local function removeListJobs(keyName, hard, baseKey, max)
|
||||
local jobs = getListItems(keyName, max)
|
||||
local count = removeJobs(jobs, hard, baseKey, max)
|
||||
rcall("LTRIM", keyName, #jobs, -1)
|
||||
return count
|
||||
end
|
||||
-- Includes
|
||||
--[[
|
||||
Function to loop in batches.
|
||||
Just a bit of warning, some commands as ZREM
|
||||
could receive a maximum of 7000 parameters per call.
|
||||
]]
|
||||
local function batches(n, batchSize)
|
||||
local i = 0
|
||||
return function()
|
||||
local from = i * batchSize + 1
|
||||
i = i + 1
|
||||
if (from <= n) then
|
||||
local to = math.min(from + batchSize - 1, n)
|
||||
return from, to
|
||||
end
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Function to get ZSet items.
|
||||
]]
|
||||
local function getZSetItems(keyName, max)
|
||||
return rcall('ZRANGE', keyName, 0, max - 1)
|
||||
end
|
||||
local function removeZSetJobs(keyName, hard, baseKey, max)
|
||||
local jobs = getZSetItems(keyName, max)
|
||||
local count = removeJobs(jobs, hard, baseKey, max)
|
||||
if(#jobs > 0) then
|
||||
for from, to in batches(#jobs, 7000) do
|
||||
rcall("ZREM", keyName, unpack(jobs, from, to))
|
||||
end
|
||||
end
|
||||
return count
|
||||
end
|
||||
local function removeLockKeys(keys)
|
||||
for i, key in ipairs(keys) do
|
||||
rcall("DEL", baseKey .. key .. ':lock')
|
||||
end
|
||||
end
|
||||
-- 1) Check if paused, if not return with error.
|
||||
if rcall("HEXISTS", KEYS[1], "paused") ~= 1 then
|
||||
return -1 -- Error, NotPaused
|
||||
end
|
||||
-- 2) Check if there are active jobs, if there are and not "force" return error.
|
||||
local activeKey = baseKey .. 'active'
|
||||
local activeJobs = getListItems(activeKey, maxCount)
|
||||
if (#activeJobs > 0) then
|
||||
if(ARGV[2] == "") then
|
||||
return -2 -- Error, ExistActiveJobs
|
||||
end
|
||||
end
|
||||
removeLockKeys(activeJobs)
|
||||
maxCount = removeJobs(activeJobs, true, baseKey, maxCount)
|
||||
rcall("LTRIM", activeKey, #activeJobs, -1)
|
||||
if(maxCount <= 0) then
|
||||
return 1
|
||||
end
|
||||
local delayedKey = baseKey .. 'delayed'
|
||||
maxCount = removeZSetJobs(delayedKey, true, baseKey, maxCount)
|
||||
if(maxCount <= 0) then
|
||||
return 1
|
||||
end
|
||||
local completedKey = baseKey .. 'completed'
|
||||
maxCount = removeZSetJobs(completedKey, true, baseKey, maxCount)
|
||||
if(maxCount <= 0) then
|
||||
return 1
|
||||
end
|
||||
local waitKey = baseKey .. 'paused'
|
||||
maxCount = removeListJobs(waitKey, true, baseKey, maxCount)
|
||||
if(maxCount <= 0) then
|
||||
return 1
|
||||
end
|
||||
local prioritizedKey = baseKey .. 'prioritized'
|
||||
maxCount = removeZSetJobs(prioritizedKey, true, baseKey, maxCount)
|
||||
if(maxCount <= 0) then
|
||||
return 1
|
||||
end
|
||||
local failedKey = baseKey .. 'failed'
|
||||
maxCount = removeZSetJobs(failedKey, true, baseKey, maxCount)
|
||||
if(maxCount <= 0) then
|
||||
return 1
|
||||
end
|
||||
if(maxCount > 0) then
|
||||
rcall("DEL",
|
||||
baseKey .. 'events',
|
||||
baseKey .. 'delay',
|
||||
baseKey .. 'stalled-check',
|
||||
baseKey .. 'stalled',
|
||||
baseKey .. 'id',
|
||||
baseKey .. 'pc',
|
||||
baseKey .. 'meta',
|
||||
baseKey .. 'repeat',
|
||||
baseKey .. 'metrics:completed',
|
||||
baseKey .. 'metrics:completed:data',
|
||||
baseKey .. 'metrics:failed',
|
||||
baseKey .. 'metrics:failed:data')
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
end
|
||||
`;
|
||||
exports.obliterate = {
|
||||
name: 'obliterate',
|
||||
content,
|
||||
keys: 2,
|
||||
};
|
||||
//# sourceMappingURL=obliterate-2.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"obliterate-2.js","sourceRoot":"","sources":["../../../src/scripts/obliterate-2.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8Of,CAAC;AACW,QAAA,UAAU,GAAG;IACxB,IAAI,EAAE,YAAY;IAClB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
||||
+110
@@ -0,0 +1,110 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.paginate = void 0;
|
||||
const content = `--[[
|
||||
Paginate a set or hash
|
||||
Input:
|
||||
KEYS[1] key pointing to the set or hash to be paginated.
|
||||
ARGV[1] page start offset
|
||||
ARGV[2] page end offset (-1 for all the elements)
|
||||
ARGV[3] cursor
|
||||
ARGV[4] offset
|
||||
ARGV[5] max iterations
|
||||
ARGV[6] fetch jobs?
|
||||
Output:
|
||||
[cursor, offset, items, numItems]
|
||||
]]
|
||||
local rcall = redis.call
|
||||
-- Includes
|
||||
--[[
|
||||
Function to achieve pagination for a set or hash.
|
||||
This function simulates pagination in the most efficient way possible
|
||||
for a set using sscan or hscan.
|
||||
The main limitation is that sets are not order preserving, so the
|
||||
pagination is not stable. This means that if the set is modified
|
||||
between pages, the same element may appear in different pages.
|
||||
]] -- Maximum number of elements to be returned by sscan per iteration.
|
||||
local maxCount = 100
|
||||
-- Finds the cursor, and returns the first elements available for the requested page.
|
||||
local function findPage(key, command, pageStart, pageSize, cursor, offset,
|
||||
maxIterations, fetchJobs)
|
||||
local items = {}
|
||||
local jobs = {}
|
||||
local iterations = 0
|
||||
repeat
|
||||
-- Iterate over the set using sscan/hscan.
|
||||
local result = rcall(command, key, cursor, "COUNT", maxCount)
|
||||
cursor = result[1]
|
||||
local members = result[2]
|
||||
local step = 1
|
||||
if command == "HSCAN" then
|
||||
step = 2
|
||||
end
|
||||
if #members == 0 then
|
||||
-- If the result is empty, we can return the result.
|
||||
return cursor, offset, items, jobs
|
||||
end
|
||||
local chunkStart = offset
|
||||
local chunkEnd = offset + #members / step
|
||||
local pageEnd = pageStart + pageSize
|
||||
if chunkEnd < pageStart then
|
||||
-- If the chunk is before the page, we can skip it.
|
||||
offset = chunkEnd
|
||||
elseif chunkStart > pageEnd then
|
||||
-- If the chunk is after the page, we can return the result.
|
||||
return cursor, offset, items, jobs
|
||||
else
|
||||
-- If the chunk is overlapping the page, we need to add the elements to the result.
|
||||
for i = 1, #members, step do
|
||||
if offset >= pageEnd then
|
||||
return cursor, offset, items, jobs
|
||||
end
|
||||
if offset >= pageStart then
|
||||
local index = #items + 1
|
||||
if fetchJobs ~= nil then
|
||||
jobs[#jobs+1] = rcall("HGETALL", members[i])
|
||||
end
|
||||
if step == 2 then
|
||||
items[index] = {members[i], members[i + 1]}
|
||||
else
|
||||
items[index] = members[i]
|
||||
end
|
||||
end
|
||||
offset = offset + 1
|
||||
end
|
||||
end
|
||||
iterations = iterations + 1
|
||||
until cursor == "0" or iterations >= maxIterations
|
||||
return cursor, offset, items, jobs
|
||||
end
|
||||
local key = KEYS[1]
|
||||
local scanCommand = "SSCAN"
|
||||
local countCommand = "SCARD"
|
||||
local type = rcall("TYPE", key)["ok"]
|
||||
if type == "none" then
|
||||
return {0, 0, {}, 0}
|
||||
elseif type == "hash" then
|
||||
scanCommand = "HSCAN"
|
||||
countCommand = "HLEN"
|
||||
elseif type ~= "set" then
|
||||
return
|
||||
redis.error_reply("Pagination is only supported for sets and hashes.")
|
||||
end
|
||||
local numItems = rcall(countCommand, key)
|
||||
local startOffset = tonumber(ARGV[1])
|
||||
local endOffset = tonumber(ARGV[2])
|
||||
if endOffset == -1 then
|
||||
endOffset = numItems
|
||||
end
|
||||
local pageSize = (endOffset - startOffset) + 1
|
||||
local cursor, offset, items, jobs = findPage(key, scanCommand, startOffset,
|
||||
pageSize, ARGV[3], tonumber(ARGV[4]),
|
||||
tonumber(ARGV[5]), ARGV[6])
|
||||
return {cursor, offset, items, numItems, jobs}
|
||||
`;
|
||||
exports.paginate = {
|
||||
name: 'paginate',
|
||||
content,
|
||||
keys: 1,
|
||||
};
|
||||
//# sourceMappingURL=paginate-1.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"paginate-1.js","sourceRoot":"","sources":["../../../src/scripts/paginate-1.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAoGf,CAAC;AACW,QAAA,QAAQ,GAAG;IACtB,IAAI,EAAE,UAAU;IAChB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.pause = void 0;
|
||||
const content = `--[[
|
||||
Pauses or resumes a queue globably.
|
||||
Input:
|
||||
KEYS[1] 'wait' or 'paused''
|
||||
KEYS[2] 'paused' or 'wait'
|
||||
KEYS[3] 'meta'
|
||||
KEYS[4] 'prioritized'
|
||||
KEYS[5] events stream key
|
||||
ARGV[1] 'paused' or 'resumed'
|
||||
Event:
|
||||
publish paused or resumed event.
|
||||
]]
|
||||
local rcall = redis.call
|
||||
-- Includes
|
||||
--[[
|
||||
Function priority marker to wait if needed
|
||||
in order to wake up our workers and to respect priority
|
||||
order as much as possible
|
||||
]]
|
||||
local function addPriorityMarkerIfNeeded(waitKey)
|
||||
local waitLen = rcall("LLEN", waitKey)
|
||||
if waitLen == 0 then
|
||||
rcall("LPUSH", waitKey, "0:0")
|
||||
end
|
||||
end
|
||||
if rcall("EXISTS", KEYS[1]) == 1 then
|
||||
rcall("RENAME", KEYS[1], KEYS[2])
|
||||
end
|
||||
if ARGV[1] == "paused" then
|
||||
rcall("HSET", KEYS[3], "paused", 1)
|
||||
else
|
||||
rcall("HDEL", KEYS[3], "paused")
|
||||
local priorityCount = rcall("ZCARD", KEYS[4])
|
||||
if priorityCount > 0 then
|
||||
addPriorityMarkerIfNeeded(KEYS[2])
|
||||
end
|
||||
end
|
||||
rcall("XADD", KEYS[5], "*", "event", ARGV[1]);
|
||||
`;
|
||||
exports.pause = {
|
||||
name: 'pause',
|
||||
content,
|
||||
keys: 5,
|
||||
};
|
||||
//# sourceMappingURL=pause-5.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"pause-5.js","sourceRoot":"","sources":["../../../src/scripts/pause-5.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAsCf,CAAC;AACW,QAAA,KAAK,GAAG;IACnB,IAAI,EAAE,OAAO;IACb,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
||||
+93
@@ -0,0 +1,93 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.promote = void 0;
|
||||
const content = `--[[
|
||||
Promotes a job that is currently "delayed" to the "waiting" state
|
||||
Input:
|
||||
KEYS[1] 'delayed'
|
||||
KEYS[2] 'wait'
|
||||
KEYS[3] 'paused'
|
||||
KEYS[4] 'meta'
|
||||
KEYS[5] 'prioritized'
|
||||
KEYS[6] 'pc' priority counter
|
||||
KEYS[7] 'event stream'
|
||||
ARGV[1] queue.toKey('')
|
||||
ARGV[2] jobId
|
||||
Output:
|
||||
0 - OK
|
||||
-3 - Job not in delayed zset.
|
||||
Events:
|
||||
'waiting'
|
||||
]]
|
||||
local rcall = redis.call
|
||||
local jobId = ARGV[2]
|
||||
-- Includes
|
||||
--[[
|
||||
Function to add job considering priority.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Function priority marker to wait if needed
|
||||
in order to wake up our workers and to respect priority
|
||||
order as much as possible
|
||||
]]
|
||||
local function addPriorityMarkerIfNeeded(waitKey)
|
||||
local waitLen = rcall("LLEN", waitKey)
|
||||
if waitLen == 0 then
|
||||
rcall("LPUSH", waitKey, "0:0")
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Function to get priority score.
|
||||
]]
|
||||
local function getPriorityScore(priority, priorityCounterKey)
|
||||
local prioCounter = rcall("INCR", priorityCounterKey)
|
||||
return priority * 0x100000000 + prioCounter % 0x100000000
|
||||
end
|
||||
local function addJobWithPriority(waitKey, prioritizedKey, priority, paused, jobId, priorityCounterKey)
|
||||
local score = getPriorityScore(priority, priorityCounterKey)
|
||||
rcall("ZADD", prioritizedKey, score, jobId)
|
||||
if not paused then
|
||||
addPriorityMarkerIfNeeded(waitKey)
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Function to check for the meta.paused key to decide if we are paused or not
|
||||
(since an empty list and !EXISTS are not really the same).
|
||||
]]
|
||||
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
|
||||
if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
|
||||
return waitKey, false
|
||||
else
|
||||
return pausedKey, true
|
||||
end
|
||||
end
|
||||
if rcall("ZREM", KEYS[1], jobId) == 1 then
|
||||
local jobKey = ARGV[1] .. jobId
|
||||
local priority = tonumber(rcall("HGET", jobKey, "priority")) or 0
|
||||
local target, paused = getTargetQueueList(KEYS[4], KEYS[2], KEYS[3])
|
||||
-- Remove delayed "marker" from the wait list if there is any.
|
||||
-- Since we are adding a job we do not need the marker anymore.
|
||||
local marker = rcall("LINDEX", target, 0)
|
||||
if marker and string.sub(marker, 1, 2) == "0:" then
|
||||
rcall("LPOP", target)
|
||||
end
|
||||
if priority == 0 then
|
||||
-- LIFO or FIFO
|
||||
rcall("LPUSH", target, jobId)
|
||||
else
|
||||
addJobWithPriority(KEYS[2], KEYS[5], priority, paused, jobId, KEYS[6])
|
||||
end
|
||||
-- Emit waiting event (wait..ing@token)
|
||||
rcall("XADD", KEYS[7], "*", "event", "waiting", "jobId", jobId, "prev", "delayed");
|
||||
rcall("HSET", jobKey, "delay", 0)
|
||||
return 0
|
||||
else
|
||||
return -3
|
||||
end`;
|
||||
exports.promote = {
|
||||
name: 'promote',
|
||||
content,
|
||||
keys: 7,
|
||||
};
|
||||
//# sourceMappingURL=promote-7.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"promote-7.js","sourceRoot":"","sources":["../../../src/scripts/promote-7.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAmFZ,CAAC;AACQ,QAAA,OAAO,GAAG;IACrB,IAAI,EAAE,SAAS;IACf,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.releaseLock = void 0;
|
||||
const content = `--[[
|
||||
Release lock
|
||||
Input:
|
||||
KEYS[1] 'lock',
|
||||
ARGV[1] token
|
||||
ARGV[2] lock duration in milliseconds
|
||||
Output:
|
||||
"OK" if lock extented succesfully.
|
||||
]]
|
||||
local rcall = redis.call
|
||||
if rcall("GET", KEYS[1]) == ARGV[1] then
|
||||
return rcall("DEL", KEYS[1])
|
||||
else
|
||||
return 0
|
||||
end
|
||||
`;
|
||||
exports.releaseLock = {
|
||||
name: 'releaseLock',
|
||||
content,
|
||||
keys: 1,
|
||||
};
|
||||
//# sourceMappingURL=releaseLock-1.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"releaseLock-1.js","sourceRoot":"","sources":["../../../src/scripts/releaseLock-1.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;CAef,CAAC;AACW,QAAA,WAAW,GAAG;IACzB,IAAI,EAAE,aAAa;IACnB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
||||
+211
@@ -0,0 +1,211 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.removeJob = void 0;
|
||||
const content = `--[[
|
||||
Remove a job from all the queues it may be in as well as all its data.
|
||||
In order to be able to remove a job, it cannot be active.
|
||||
Input:
|
||||
KEYS[1] queue prefix
|
||||
ARGV[1] jobId
|
||||
ARGV[2] remove children
|
||||
Events:
|
||||
'removed'
|
||||
]]
|
||||
local rcall = redis.call
|
||||
-- Includes
|
||||
--[[
|
||||
Functions to destructure job key.
|
||||
Just a bit of warning, these functions may be a bit slow and affect performance significantly.
|
||||
]]
|
||||
local getJobIdFromKey = function (jobKey)
|
||||
return string.match(jobKey, ".*:(.*)")
|
||||
end
|
||||
local getJobKeyPrefix = function (jobKey, jobId)
|
||||
return string.sub(jobKey, 0, #jobKey - #jobId)
|
||||
end
|
||||
--[[
|
||||
Function to recursively check if there are no locks
|
||||
on the jobs to be removed.
|
||||
returns:
|
||||
boolean
|
||||
]]
|
||||
local function isLocked( prefix, jobId, removeChildren)
|
||||
local jobKey = prefix .. jobId;
|
||||
-- Check if this job is locked
|
||||
local lockKey = jobKey .. ':lock'
|
||||
local lock = rcall("GET", lockKey)
|
||||
if not lock then
|
||||
if removeChildren == "1" then
|
||||
local dependencies = rcall("SMEMBERS", jobKey .. ":dependencies")
|
||||
if (#dependencies > 0) then
|
||||
for i, childJobKey in ipairs(dependencies) do
|
||||
-- We need to get the jobId for this job.
|
||||
local childJobId = getJobIdFromKey(childJobKey)
|
||||
local childJobPrefix = getJobKeyPrefix(childJobKey, childJobId)
|
||||
local result = isLocked( childJobPrefix, childJobId, removeChildren )
|
||||
if result then
|
||||
return true
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
return false
|
||||
end
|
||||
return true
|
||||
end
|
||||
--[[
|
||||
Function to remove from any state.
|
||||
returns:
|
||||
prev state
|
||||
]]
|
||||
local function removeJobFromAnyState( prefix, jobId)
|
||||
-- We start with the ZSCORE checks, since they have O(1) complexity
|
||||
if rcall("ZSCORE", prefix .. "completed", jobId) then
|
||||
rcall("ZREM", prefix .. "completed", jobId)
|
||||
return "completed"
|
||||
elseif rcall("ZSCORE", prefix .. "waiting-children", jobId) then
|
||||
rcall("ZREM", prefix .. "waiting-children", jobId)
|
||||
return "waiting-children"
|
||||
elseif rcall("ZSCORE", prefix .. "delayed", jobId) then
|
||||
rcall("ZREM", prefix .. "delayed", jobId)
|
||||
return "delayed"
|
||||
elseif rcall("ZSCORE", prefix .. "failed", jobId) then
|
||||
rcall("ZREM", prefix .. "failed", jobId)
|
||||
return "failed"
|
||||
elseif rcall("ZSCORE", prefix .. "prioritized", jobId) then
|
||||
rcall("ZREM", prefix .. "prioritized", jobId)
|
||||
return "prioritized"
|
||||
-- We remove only 1 element from the list, since we assume they are not added multiple times
|
||||
elseif rcall("LREM", prefix .. "wait", 1, jobId) == 1 then
|
||||
return "wait"
|
||||
elseif rcall("LREM", prefix .. "paused", 1, jobId) == 1 then
|
||||
return "paused"
|
||||
elseif rcall("LREM", prefix .. "active", 1, jobId) == 1 then
|
||||
return "active"
|
||||
end
|
||||
return "unknown"
|
||||
end
|
||||
--[[
|
||||
Check if this job has a parent. If so we will just remove it from
|
||||
the parent child list, but if it is the last child we should move the parent to "wait/paused"
|
||||
which requires code from "moveToFinished"
|
||||
]]
|
||||
--[[
|
||||
Function to check for the meta.paused key to decide if we are paused or not
|
||||
(since an empty list and !EXISTS are not really the same).
|
||||
]]
|
||||
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
|
||||
if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
|
||||
return waitKey, false
|
||||
else
|
||||
return pausedKey, true
|
||||
end
|
||||
end
|
||||
local function moveParentToWait(parentPrefix, parentId, emitEvent)
|
||||
local parentTarget = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "wait", parentPrefix .. "paused")
|
||||
rcall("RPUSH", parentTarget, parentId)
|
||||
if emitEvent then
|
||||
local parentEventStream = parentPrefix .. "events"
|
||||
rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
|
||||
end
|
||||
end
|
||||
local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey)
|
||||
if parentKey then
|
||||
local parentDependenciesKey = parentKey .. ":dependencies"
|
||||
local result = rcall("SREM", parentDependenciesKey, jobKey)
|
||||
if result > 0 then
|
||||
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
|
||||
if pendingDependencies == 0 then
|
||||
local parentId = getJobIdFromKey(parentKey)
|
||||
local parentPrefix = getJobKeyPrefix(parentKey, parentId)
|
||||
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
|
||||
if numRemovedElements == 1 then
|
||||
if hard then
|
||||
if parentPrefix == baseKey then
|
||||
removeParentDependencyKey(parentKey, hard, nil, baseKey)
|
||||
rcall("DEL", parentKey, parentKey .. ':logs',
|
||||
parentKey .. ':dependencies', parentKey .. ':processed')
|
||||
else
|
||||
moveParentToWait(parentPrefix, parentId)
|
||||
end
|
||||
else
|
||||
moveParentToWait(parentPrefix, parentId, true)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
else
|
||||
local missedParentKey = rcall("HGET", jobKey, "parentKey")
|
||||
if( (type(missedParentKey) == "string") and missedParentKey ~= "" and (rcall("EXISTS", missedParentKey) == 1)) then
|
||||
local parentDependenciesKey = missedParentKey .. ":dependencies"
|
||||
local result = rcall("SREM", parentDependenciesKey, jobKey)
|
||||
if result > 0 then
|
||||
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
|
||||
if pendingDependencies == 0 then
|
||||
local parentId = getJobIdFromKey(missedParentKey)
|
||||
local parentPrefix = getJobKeyPrefix(missedParentKey, parentId)
|
||||
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
|
||||
if numRemovedElements == 1 then
|
||||
if hard then
|
||||
if parentPrefix == baseKey then
|
||||
removeParentDependencyKey(missedParentKey, hard, nil, baseKey)
|
||||
rcall("DEL", missedParentKey, missedParentKey .. ':logs',
|
||||
missedParentKey .. ':dependencies', missedParentKey .. ':processed')
|
||||
else
|
||||
moveParentToWait(parentPrefix, parentId)
|
||||
end
|
||||
else
|
||||
moveParentToWait(parentPrefix, parentId, true)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
local function removeJob( prefix, jobId, parentKey, removeChildren)
|
||||
local jobKey = prefix .. jobId;
|
||||
removeParentDependencyKey(jobKey, false, parentKey)
|
||||
if removeChildren == "1" then
|
||||
-- Check if this job has children
|
||||
-- If so, we are going to try to remove the children recursively in deep first way because
|
||||
-- if some job is locked we must exit with and error.
|
||||
--local countProcessed = rcall("HLEN", jobKey .. ":processed")
|
||||
local processed = rcall("HGETALL", jobKey .. ":processed")
|
||||
if (#processed > 0) then
|
||||
for i = 1, #processed, 2 do
|
||||
local childJobId = getJobIdFromKey(processed[i])
|
||||
local childJobPrefix = getJobKeyPrefix(processed[i], childJobId)
|
||||
removeJob( childJobPrefix, childJobId, jobKey, removeChildren )
|
||||
end
|
||||
end
|
||||
local dependencies = rcall("SMEMBERS", jobKey .. ":dependencies")
|
||||
if (#dependencies > 0) then
|
||||
for i, childJobKey in ipairs(dependencies) do
|
||||
-- We need to get the jobId for this job.
|
||||
local childJobId = getJobIdFromKey(childJobKey)
|
||||
local childJobPrefix = getJobKeyPrefix(childJobKey, childJobId)
|
||||
removeJob( childJobPrefix, childJobId, jobKey, removeChildren )
|
||||
end
|
||||
end
|
||||
end
|
||||
local prev = removeJobFromAnyState(prefix, jobId)
|
||||
if rcall("DEL", jobKey, jobKey .. ":logs", jobKey .. ":dependencies", jobKey .. ":processed") > 0 then
|
||||
local maxEvents = rcall("HGET", prefix .. "meta", "opts.maxLenEvents") or 10000
|
||||
rcall("XADD", prefix .. "events", "MAXLEN", "~", maxEvents, "*", "event", "removed",
|
||||
"jobId", jobId, "prev", prev)
|
||||
end
|
||||
end
|
||||
local prefix = KEYS[1]
|
||||
if not isLocked(prefix, ARGV[1], ARGV[2]) then
|
||||
removeJob(prefix, ARGV[1], nil, ARGV[2])
|
||||
return 1
|
||||
end
|
||||
return 0
|
||||
`;
|
||||
exports.removeJob = {
|
||||
name: 'removeJob',
|
||||
content,
|
||||
keys: 1,
|
||||
};
|
||||
//# sourceMappingURL=removeJob-1.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"removeJob-1.js","sourceRoot":"","sources":["../../../src/scripts/removeJob-1.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyMf,CAAC;AACW,QAAA,SAAS,GAAG;IACvB,IAAI,EAAE,WAAW;IACjB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.removeRepeatable = void 0;
|
||||
const content = `--[[
|
||||
Removes a repeatable job
|
||||
Input:
|
||||
KEYS[1] repeat jobs key
|
||||
KEYS[2] delayed jobs key
|
||||
ARGV[1] repeat job id
|
||||
ARGV[2] repeat job key
|
||||
ARGV[3] queue key
|
||||
Output:
|
||||
0 - OK
|
||||
1 - Missing repeat job
|
||||
Events:
|
||||
'removed'
|
||||
]]
|
||||
local rcall = redis.call
|
||||
local millis = rcall("ZSCORE", KEYS[1], ARGV[2])
|
||||
if(millis) then
|
||||
-- Delete next programmed job.
|
||||
local repeatJobId = ARGV[1] .. millis
|
||||
if(rcall("ZREM", KEYS[2], repeatJobId) == 1) then
|
||||
rcall("DEL", ARGV[3] .. repeatJobId)
|
||||
rcall("XADD", ARGV[3] .. "events", "*", "event", "removed", "jobId", repeatJobId, "prev", "delayed");
|
||||
end
|
||||
end
|
||||
if(rcall("ZREM", KEYS[1], ARGV[2]) == 1) then
|
||||
return 0
|
||||
end
|
||||
return 1
|
||||
`;
|
||||
exports.removeRepeatable = {
|
||||
name: 'removeRepeatable',
|
||||
content,
|
||||
keys: 2,
|
||||
};
|
||||
//# sourceMappingURL=removeRepeatable-2.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"removeRepeatable-2.js","sourceRoot":"","sources":["../../../src/scripts/removeRepeatable-2.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA4Bf,CAAC;AACW,QAAA,gBAAgB,GAAG;IAC9B,IAAI,EAAE,kBAAkB;IACxB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
||||
+56
@@ -0,0 +1,56 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.reprocessJob = void 0;
|
||||
const content = `--[[
|
||||
Attempts to reprocess a job
|
||||
Input:
|
||||
KEYS[1] job key
|
||||
KEYS[2] events stream
|
||||
KEYS[3] job state
|
||||
KEYS[4] wait key
|
||||
KEYS[5] meta
|
||||
KEYS[6] paused key
|
||||
ARGV[1] job.id
|
||||
ARGV[2] (job.opts.lifo ? 'R' : 'L') + 'PUSH'
|
||||
ARGV[3] propVal - failedReason/returnvalue
|
||||
ARGV[4] prev state - failed/completed
|
||||
Output:
|
||||
1 means the operation was a success
|
||||
-1 means the job does not exist
|
||||
-3 means the job was not found in the expected set.
|
||||
]]
|
||||
local rcall = redis.call;
|
||||
-- Includes
|
||||
--[[
|
||||
Function to check for the meta.paused key to decide if we are paused or not
|
||||
(since an empty list and !EXISTS are not really the same).
|
||||
]]
|
||||
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
|
||||
if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
|
||||
return waitKey, false
|
||||
else
|
||||
return pausedKey, true
|
||||
end
|
||||
end
|
||||
if (rcall("EXISTS", KEYS[1]) == 1) then
|
||||
local jobId = ARGV[1]
|
||||
if (rcall("ZREM", KEYS[3], jobId) == 1) then
|
||||
rcall("HDEL", KEYS[1], "finishedOn", "processedOn", ARGV[3])
|
||||
local target = getTargetQueueList(KEYS[5], KEYS[4], KEYS[6])
|
||||
rcall(ARGV[2], target, jobId)
|
||||
-- Emit waiting event
|
||||
rcall("XADD", KEYS[2], "*", "event", "waiting", "jobId", jobId, "prev", ARGV[4]);
|
||||
return 1
|
||||
else
|
||||
return -3
|
||||
end
|
||||
else
|
||||
return -1
|
||||
end
|
||||
`;
|
||||
exports.reprocessJob = {
|
||||
name: 'reprocessJob',
|
||||
content,
|
||||
keys: 6,
|
||||
};
|
||||
//# sourceMappingURL=reprocessJob-6.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"reprocessJob-6.js","sourceRoot":"","sources":["../../../src/scripts/reprocessJob-6.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8Cf,CAAC;AACW,QAAA,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
||||
+139
@@ -0,0 +1,139 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.retryJob = void 0;
|
||||
const content = `--[[
|
||||
Retries a failed job by moving it back to the wait queue.
|
||||
Input:
|
||||
KEYS[1] 'active',
|
||||
KEYS[2] 'wait'
|
||||
KEYS[3] 'paused'
|
||||
KEYS[4] job key
|
||||
KEYS[5] 'meta'
|
||||
KEYS[6] events stream
|
||||
KEYS[7] delayed key
|
||||
KEYS[8] prioritized key
|
||||
KEYS[9] 'pc' priority counter
|
||||
ARGV[1] key prefix
|
||||
ARGV[2] timestamp
|
||||
ARGV[3] pushCmd
|
||||
ARGV[4] jobId
|
||||
ARGV[5] token
|
||||
Events:
|
||||
'waiting'
|
||||
Output:
|
||||
0 - OK
|
||||
-1 - Missing key
|
||||
-2 - Missing lock
|
||||
]]
|
||||
local rcall = redis.call
|
||||
-- Includes
|
||||
--[[
|
||||
Function to add job considering priority.
|
||||
]]
|
||||
-- Includes
|
||||
--[[
|
||||
Function priority marker to wait if needed
|
||||
in order to wake up our workers and to respect priority
|
||||
order as much as possible
|
||||
]]
|
||||
local function addPriorityMarkerIfNeeded(waitKey)
|
||||
local waitLen = rcall("LLEN", waitKey)
|
||||
if waitLen == 0 then
|
||||
rcall("LPUSH", waitKey, "0:0")
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Function to get priority score.
|
||||
]]
|
||||
local function getPriorityScore(priority, priorityCounterKey)
|
||||
local prioCounter = rcall("INCR", priorityCounterKey)
|
||||
return priority * 0x100000000 + prioCounter % 0x100000000
|
||||
end
|
||||
local function addJobWithPriority(waitKey, prioritizedKey, priority, paused, jobId, priorityCounterKey)
|
||||
local score = getPriorityScore(priority, priorityCounterKey)
|
||||
rcall("ZADD", prioritizedKey, score, jobId)
|
||||
if not paused then
|
||||
addPriorityMarkerIfNeeded(waitKey)
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Function to check for the meta.paused key to decide if we are paused or not
|
||||
(since an empty list and !EXISTS are not really the same).
|
||||
]]
|
||||
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
|
||||
if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
|
||||
return waitKey, false
|
||||
else
|
||||
return pausedKey, true
|
||||
end
|
||||
end
|
||||
--[[
|
||||
Updates the delay set, by moving delayed jobs that should
|
||||
be processed now to "wait".
|
||||
Events:
|
||||
'waiting'
|
||||
]]
|
||||
-- Includes
|
||||
-- Try to get as much as 1000 jobs at once
|
||||
local function promoteDelayedJobs(delayedKey, waitKey, targetKey, prioritizedKey,
|
||||
eventStreamKey, prefix, timestamp, paused, priorityCounterKey)
|
||||
local jobs = rcall("ZRANGEBYSCORE", delayedKey, 0, (timestamp + 1) * 0x1000, "LIMIT", 0, 1000)
|
||||
if (#jobs > 0) then
|
||||
rcall("ZREM", delayedKey, unpack(jobs))
|
||||
for _, jobId in ipairs(jobs) do
|
||||
local jobKey = prefix .. jobId
|
||||
local priority =
|
||||
tonumber(rcall("HGET", jobKey, "priority")) or 0
|
||||
if priority == 0 then
|
||||
-- LIFO or FIFO
|
||||
rcall("LPUSH", targetKey, jobId)
|
||||
else
|
||||
local score = getPriorityScore(priority, priorityCounterKey)
|
||||
rcall("ZADD", prioritizedKey, score, jobId)
|
||||
end
|
||||
-- Emit waiting event
|
||||
rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId",
|
||||
jobId, "prev", "delayed")
|
||||
rcall("HSET", jobKey, "delay", 0)
|
||||
end
|
||||
if not paused then
|
||||
addPriorityMarkerIfNeeded(targetKey)
|
||||
end
|
||||
end
|
||||
end
|
||||
local target, paused = getTargetQueueList(KEYS[5], KEYS[2], KEYS[3])
|
||||
-- Check if there are delayed jobs that we can move to wait.
|
||||
-- test example: when there are delayed jobs between retries
|
||||
promoteDelayedJobs(KEYS[7], KEYS[2], target, KEYS[8], KEYS[6], ARGV[1], ARGV[2], paused, KEYS[9])
|
||||
if rcall("EXISTS", KEYS[4]) == 1 then
|
||||
if ARGV[5] ~= "0" then
|
||||
local lockKey = KEYS[4] .. ':lock'
|
||||
if rcall("GET", lockKey) == ARGV[5] then
|
||||
rcall("DEL", lockKey)
|
||||
else
|
||||
return -2
|
||||
end
|
||||
end
|
||||
rcall("LREM", KEYS[1], 0, ARGV[4])
|
||||
local priority = tonumber(rcall("HGET", KEYS[4], "priority")) or 0
|
||||
-- Standard or priority add
|
||||
if priority == 0 then
|
||||
rcall(ARGV[3], target, ARGV[4])
|
||||
else
|
||||
addJobWithPriority(KEYS[2], KEYS[8], priority, paused, ARGV[4], KEYS[9])
|
||||
end
|
||||
local maxEvents = rcall("HGET", KEYS[5], "opts.maxLenEvents") or 10000
|
||||
-- Emit waiting event
|
||||
rcall("XADD", KEYS[6], "MAXLEN", "~", maxEvents, "*", "event", "waiting",
|
||||
"jobId", ARGV[4], "prev", "failed")
|
||||
return 0
|
||||
else
|
||||
return -1
|
||||
end
|
||||
`;
|
||||
exports.retryJob = {
|
||||
name: 'retryJob',
|
||||
content,
|
||||
keys: 9,
|
||||
};
|
||||
//# sourceMappingURL=retryJob-9.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"retryJob-9.js","sourceRoot":"","sources":["../../../src/scripts/retryJob-9.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAiIf,CAAC;AACW,QAAA,QAAQ,GAAG;IACtB,IAAI,EAAE,UAAU;IAChB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.saveStacktrace = void 0;
|
||||
const content = `--[[
|
||||
Save stacktrace and failedReason.
|
||||
Input:
|
||||
KEYS[1] job key
|
||||
ARGV[1] stacktrace
|
||||
ARGV[2] failedReason
|
||||
Output:
|
||||
0 - OK
|
||||
-1 - Missing key
|
||||
]]
|
||||
local rcall = redis.call
|
||||
if rcall("EXISTS", KEYS[1]) == 1 then
|
||||
rcall("HMSET", KEYS[1], "stacktrace", ARGV[1], "failedReason", ARGV[2])
|
||||
return 0
|
||||
else
|
||||
return -1
|
||||
end
|
||||
`;
|
||||
exports.saveStacktrace = {
|
||||
name: 'saveStacktrace',
|
||||
content,
|
||||
keys: 1,
|
||||
};
|
||||
//# sourceMappingURL=saveStacktrace-1.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"saveStacktrace-1.js","sourceRoot":"","sources":["../../../src/scripts/saveStacktrace-1.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;CAiBf,CAAC;AACW,QAAA,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.updateData = void 0;
|
||||
const content = `--[[
|
||||
Update job data
|
||||
Input:
|
||||
KEYS[1] Job id key
|
||||
ARGV[1] data
|
||||
Output:
|
||||
0 - OK
|
||||
-1 - Missing job.
|
||||
]]
|
||||
local rcall = redis.call
|
||||
if rcall("EXISTS",KEYS[1]) == 1 then -- // Make sure job exists
|
||||
rcall("HSET", KEYS[1], "data", ARGV[1])
|
||||
return 0
|
||||
else
|
||||
return -1
|
||||
end
|
||||
`;
|
||||
exports.updateData = {
|
||||
name: 'updateData',
|
||||
content,
|
||||
keys: 1,
|
||||
};
|
||||
//# sourceMappingURL=updateData-1.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"updateData-1.js","sourceRoot":"","sources":["../../../src/scripts/updateData-1.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;CAgBf,CAAC;AACW,QAAA,UAAU,GAAG;IACxB,IAAI,EAAE,YAAY;IAClB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.updateProgress = void 0;
|
||||
const content = `--[[
|
||||
Update job progress
|
||||
Input:
|
||||
KEYS[1] Job id key
|
||||
KEYS[2] event stream key
|
||||
KEYS[3] meta key
|
||||
ARGV[1] id
|
||||
ARGV[2] progress
|
||||
Output:
|
||||
0 - OK
|
||||
-1 - Missing job.
|
||||
Event:
|
||||
progress(jobId, progress)
|
||||
]]
|
||||
local rcall = redis.call
|
||||
if rcall("EXISTS", KEYS[1]) == 1 then -- // Make sure job exists
|
||||
local maxEvents = rcall("HGET", KEYS[3], "opts.maxLenEvents") or 10000
|
||||
rcall("HSET", KEYS[1], "progress", ARGV[2])
|
||||
rcall("XADD", KEYS[2], "MAXLEN", "~", maxEvents, "*", "event", "progress",
|
||||
"jobId", ARGV[1], "data", ARGV[2]);
|
||||
return 0
|
||||
else
|
||||
return -1
|
||||
end
|
||||
`;
|
||||
exports.updateProgress = {
|
||||
name: 'updateProgress',
|
||||
content,
|
||||
keys: 3,
|
||||
};
|
||||
//# sourceMappingURL=updateProgress-3.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"updateProgress-3.js","sourceRoot":"","sources":["../../../src/scripts/updateProgress-3.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;CAwBf,CAAC;AACW,QAAA,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
||||
Reference in New Issue
Block a user