Initial commit of working RSS Aggregator build
This commit is contained in:
+124
@@ -0,0 +1,124 @@
|
||||
--[[
|
||||
Adds a delayed job to the queue by doing the following:
|
||||
- Increases the job counter if needed.
|
||||
- Creates a new job key with the job data.
|
||||
|
||||
- computes timestamp.
|
||||
- adds to delayed zset.
|
||||
- Emits a global event 'delayed' if the job is delayed.
|
||||
|
||||
Input:
|
||||
KEYS[1] 'wait',
|
||||
KEYS[2] 'paused'
|
||||
KEYS[3] 'meta'
|
||||
KEYS[4] 'id'
|
||||
KEYS[5] 'delayed'
|
||||
KEYS[6] 'completed'
|
||||
KEYS[7] events stream key
|
||||
|
||||
ARGV[1] msgpacked arguments array
|
||||
[1] key prefix,
|
||||
[2] custom id (use custom instead of one generated automatically)
|
||||
[3] name
|
||||
[4] timestamp
|
||||
[5] parentKey?
|
||||
x [6] waitChildrenKey key.
|
||||
[7] parent dependencies key.
|
||||
[8] parent? {id, queueKey}
|
||||
[9] repeat job key
|
||||
|
||||
ARGV[2] Json stringified job data
|
||||
ARGV[3] msgpacked options
|
||||
|
||||
Output:
|
||||
jobId - OK
|
||||
-5 - Missing parent key
|
||||
]]
|
||||
local waitKey = KEYS[1]
|
||||
local pausedKey = KEYS[2]
|
||||
|
||||
local metaKey = KEYS[3]
|
||||
local idKey = KEYS[4]
|
||||
local delayedKey = KEYS[5]
|
||||
|
||||
local completedKey = KEYS[6]
|
||||
local eventsKey = KEYS[7]
|
||||
|
||||
local jobId
|
||||
local jobIdKey
|
||||
local rcall = redis.call
|
||||
|
||||
local args = cmsgpack.unpack(ARGV[1])
|
||||
|
||||
local data = ARGV[2]
|
||||
local opts = cmsgpack.unpack(ARGV[3])
|
||||
|
||||
local parentKey = args[5]
|
||||
local repeatJobKey = args[9]
|
||||
local parent = args[8]
|
||||
local parentData
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/storeJob"
|
||||
--- @include "includes/addDelayMarkerIfNeeded"
|
||||
--- @include "includes/getTargetQueueList"
|
||||
--- @include "includes/getNextDelayedTimestamp"
|
||||
--- @include "includes/updateExistingJobsParent"
|
||||
--- @include "includes/getOrSetMaxEvents"
|
||||
|
||||
if parentKey ~= nil then
|
||||
if rcall("EXISTS", parentKey) ~= 1 then return -5 end
|
||||
|
||||
parentData = cjson.encode(parent)
|
||||
end
|
||||
|
||||
local jobCounter = rcall("INCR", idKey)
|
||||
|
||||
local maxEvents = getOrSetMaxEvents(metaKey)
|
||||
|
||||
local parentDependenciesKey = args[7]
|
||||
local timestamp = args[4]
|
||||
if args[2] == "" then
|
||||
jobId = jobCounter
|
||||
jobIdKey = args[1] .. jobId
|
||||
else
|
||||
-- Refactor to: handleDuplicateJob.lua
|
||||
jobId = args[2]
|
||||
jobIdKey = args[1] .. jobId
|
||||
if rcall("EXISTS", jobIdKey) == 1 then
|
||||
updateExistingJobsParent(parentKey, parent, parentData,
|
||||
parentDependenciesKey, completedKey, jobIdKey,
|
||||
jobId, timestamp)
|
||||
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
|
||||
"duplicated", "jobId", jobId)
|
||||
|
||||
return jobId .. "" -- convert to string
|
||||
end
|
||||
end
|
||||
|
||||
-- Store the job.
|
||||
local delay, priority = storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2],
|
||||
opts, timestamp, parentKey, parentData,
|
||||
repeatJobKey)
|
||||
|
||||
-- Compute delayed timestamp and the score.
|
||||
local delayedTimestamp = (delay > 0 and (timestamp + delay)) or 0
|
||||
local score = delayedTimestamp * 0x1000 + bit.band(jobCounter, 0xfff)
|
||||
|
||||
rcall("ZADD", delayedKey, score, jobId)
|
||||
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
|
||||
"jobId", jobId, "delay", delayedTimestamp)
|
||||
|
||||
-- If wait list is empty, and this delayed job is the next one to be processed,
|
||||
-- then we need to signal the workers by adding a dummy job (jobId 0:delay) to the wait list.
|
||||
local target = getTargetQueueList(metaKey, KEYS[1], KEYS[2])
|
||||
addDelayMarkerIfNeeded(target, delayedKey)
|
||||
|
||||
-- Check if this job is a child of another job, if so add it to the parents dependencies
|
||||
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
|
||||
-- fail in this case.
|
||||
if parentDependenciesKey ~= nil then
|
||||
rcall("SADD", parentDependenciesKey, jobIdKey)
|
||||
end
|
||||
|
||||
return jobId .. "" -- convert to string
|
||||
+101
@@ -0,0 +1,101 @@
|
||||
--[[
|
||||
Adds a parent job to the queue by doing the following:
|
||||
- Increases the job counter if needed.
|
||||
- Creates a new job key with the job data.
|
||||
- adds the job to the waiting-children zset
|
||||
|
||||
Input:
|
||||
KEYS[1] 'meta'
|
||||
KEYS[2] 'id'
|
||||
KEYS[3] 'completed'
|
||||
KEYS[4] events stream key
|
||||
|
||||
ARGV[1] msgpacked arguments array
|
||||
[1] key prefix,
|
||||
[2] custom id (will not generate one automatically)
|
||||
[3] name
|
||||
[4] timestamp
|
||||
[5] parentKey?
|
||||
[6] waitChildrenKey key.
|
||||
[7] parent dependencies key.
|
||||
[8] parent? {id, queueKey}
|
||||
[9] repeat job key
|
||||
|
||||
ARGV[2] Json stringified job data
|
||||
ARGV[3] msgpacked options
|
||||
|
||||
Output:
|
||||
jobId - OK
|
||||
-5 - Missing parent key
|
||||
]]
|
||||
local metaKey = KEYS[1]
|
||||
local idKey = KEYS[2]
|
||||
|
||||
local completedKey = KEYS[3]
|
||||
local eventsKey = KEYS[4]
|
||||
|
||||
local jobId
|
||||
local jobIdKey
|
||||
local rcall = redis.call
|
||||
|
||||
local args = cmsgpack.unpack(ARGV[1])
|
||||
|
||||
local data = ARGV[2]
|
||||
local opts = cmsgpack.unpack(ARGV[3])
|
||||
|
||||
local parentKey = args[5]
|
||||
local repeatJobKey = args[9]
|
||||
local parent = args[8]
|
||||
local parentData
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/storeJob"
|
||||
--- @include "includes/updateExistingJobsParent"
|
||||
--- @include "includes/getOrSetMaxEvents"
|
||||
|
||||
if parentKey ~= nil then
|
||||
if rcall("EXISTS", parentKey) ~= 1 then return -5 end
|
||||
|
||||
parentData = cjson.encode(parent)
|
||||
end
|
||||
|
||||
local jobCounter = rcall("INCR", idKey)
|
||||
|
||||
local maxEvents = getOrSetMaxEvents(metaKey)
|
||||
|
||||
local parentDependenciesKey = args[7]
|
||||
local timestamp = args[4]
|
||||
if args[2] == "" then
|
||||
jobId = jobCounter
|
||||
jobIdKey = args[1] .. jobId
|
||||
else
|
||||
jobId = args[2]
|
||||
jobIdKey = args[1] .. jobId
|
||||
if rcall("EXISTS", jobIdKey) == 1 then
|
||||
updateExistingJobsParent(parentKey, parent, parentData,
|
||||
parentDependenciesKey, completedKey, jobIdKey,
|
||||
jobId, timestamp)
|
||||
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
|
||||
"duplicated", "jobId", jobId)
|
||||
|
||||
return jobId .. "" -- convert to string
|
||||
end
|
||||
end
|
||||
|
||||
-- Store the job.
|
||||
storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2], opts, timestamp,
|
||||
parentKey, parentData, repeatJobKey)
|
||||
|
||||
local waitChildrenKey = args[6]
|
||||
rcall("ZADD", waitChildrenKey, timestamp, jobId)
|
||||
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
|
||||
"waiting-children", "jobId", jobId)
|
||||
|
||||
-- Check if this job is a child of another job, if so add it to the parents dependencies
|
||||
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
|
||||
-- fail in this case.
|
||||
if parentDependenciesKey ~= nil then
|
||||
rcall("SADD", parentDependenciesKey, jobIdKey)
|
||||
end
|
||||
|
||||
return jobId .. "" -- convert to string
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
--[[
|
||||
Adds a priotitized job to the queue by doing the following:
|
||||
- Increases the job counter if needed.
|
||||
- Creates a new job key with the job data.
|
||||
- Adds the job to the "added" list so that workers gets notified.
|
||||
|
||||
Input:
|
||||
KEYS[1] 'wait',
|
||||
KEYS[2] 'paused'
|
||||
KEYS[3] 'meta'
|
||||
KEYS[4] 'id'
|
||||
KEYS[5] 'prioritized'
|
||||
KEYS[6] 'completed'
|
||||
KEYS[7] events stream key
|
||||
KEYS[8] 'pc' priority counter
|
||||
|
||||
ARGV[1] msgpacked arguments array
|
||||
[1] key prefix,
|
||||
[2] custom id (will not generate one automatically)
|
||||
[3] name
|
||||
[4] timestamp
|
||||
[5] parentKey?
|
||||
[6] waitChildrenKey key.
|
||||
[7] parent dependencies key.
|
||||
[8] parent? {id, queueKey}
|
||||
[9] repeat job key
|
||||
|
||||
ARGV[2] Json stringified job data
|
||||
ARGV[3] msgpacked options
|
||||
|
||||
Output:
|
||||
jobId - OK
|
||||
-5 - Missing parent key
|
||||
]]
|
||||
local waitKey = KEYS[1]
|
||||
local pausedKey = KEYS[2]
|
||||
|
||||
local metaKey = KEYS[3]
|
||||
local idKey = KEYS[4]
|
||||
local priorityKey = KEYS[5]
|
||||
|
||||
local completedKey = KEYS[6]
|
||||
local eventsKey = KEYS[7]
|
||||
local priorityCounterKey = KEYS[8]
|
||||
|
||||
local jobId
|
||||
local jobIdKey
|
||||
local rcall = redis.call
|
||||
|
||||
local args = cmsgpack.unpack(ARGV[1])
|
||||
|
||||
local data = ARGV[2]
|
||||
local opts = cmsgpack.unpack(ARGV[3])
|
||||
|
||||
local parentKey = args[5]
|
||||
local repeatJobKey = args[9]
|
||||
local parent = args[8]
|
||||
local parentData
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/storeJob"
|
||||
--- @include "includes/addJobWithPriority"
|
||||
--- @include "includes/getTargetQueueList"
|
||||
--- @include "includes/updateExistingJobsParent"
|
||||
--- @include "includes/getOrSetMaxEvents"
|
||||
|
||||
if parentKey ~= nil then
|
||||
if rcall("EXISTS", parentKey) ~= 1 then return -5 end
|
||||
|
||||
parentData = cjson.encode(parent)
|
||||
end
|
||||
|
||||
local jobCounter = rcall("INCR", idKey)
|
||||
|
||||
local maxEvents = getOrSetMaxEvents(metaKey)
|
||||
|
||||
local parentDependenciesKey = args[7]
|
||||
local timestamp = args[4]
|
||||
if args[2] == "" then
|
||||
jobId = jobCounter
|
||||
jobIdKey = args[1] .. jobId
|
||||
else
|
||||
jobId = args[2]
|
||||
jobIdKey = args[1] .. jobId
|
||||
if rcall("EXISTS", jobIdKey) == 1 then
|
||||
updateExistingJobsParent(parentKey, parent, parentData,
|
||||
parentDependenciesKey, completedKey, jobIdKey,
|
||||
jobId, timestamp)
|
||||
|
||||
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
|
||||
"duplicated", "jobId", jobId)
|
||||
|
||||
return jobId .. "" -- convert to string
|
||||
end
|
||||
end
|
||||
|
||||
-- Store the job.
|
||||
local delay, priority = storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2],
|
||||
opts, timestamp, parentKey, parentData,
|
||||
repeatJobKey)
|
||||
|
||||
local target, paused = getTargetQueueList(metaKey, waitKey, pausedKey)
|
||||
|
||||
addJobWithPriority(waitKey, priorityKey, priority, paused, jobId,
|
||||
priorityCounterKey)
|
||||
-- Emit waiting event
|
||||
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting",
|
||||
"jobId", jobId)
|
||||
|
||||
-- Check if this job is a child of another job, if so add it to the parents dependencies
|
||||
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
|
||||
-- fail in this case.
|
||||
if parentDependenciesKey ~= nil then
|
||||
rcall("SADD", parentDependenciesKey, jobIdKey)
|
||||
end
|
||||
|
||||
return jobId .. "" -- convert to string
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
--[[
|
||||
Adds a job to the queue by doing the following:
|
||||
- Increases the job counter if needed.
|
||||
- Creates a new job key with the job data.
|
||||
|
||||
- if delayed:
|
||||
- computes timestamp.
|
||||
- adds to delayed zset.
|
||||
- Emits a global event 'delayed' if the job is delayed.
|
||||
- if not delayed
|
||||
- Adds the jobId to the wait/paused list in one of three ways:
|
||||
- LIFO
|
||||
- FIFO
|
||||
- prioritized.
|
||||
- Adds the job to the "added" list so that workers gets notified.
|
||||
|
||||
Input:
|
||||
KEYS[1] 'wait',
|
||||
KEYS[2] 'paused'
|
||||
KEYS[3] 'meta'
|
||||
KEYS[4] 'id'
|
||||
KEYS[5] 'completed'
|
||||
KEYS[6] events stream key
|
||||
|
||||
ARGV[1] msgpacked arguments array
|
||||
[1] key prefix,
|
||||
[2] custom id (will not generate one automatically)
|
||||
[3] name
|
||||
[4] timestamp
|
||||
[5] parentKey?
|
||||
[6] waitChildrenKey key.
|
||||
[7] parent dependencies key.
|
||||
[8] parent? {id, queueKey}
|
||||
[9] repeat job key
|
||||
|
||||
ARGV[2] Json stringified job data
|
||||
ARGV[3] msgpacked options
|
||||
|
||||
Output:
|
||||
jobId - OK
|
||||
-5 - Missing parent key
|
||||
]]
|
||||
local eventsKey = KEYS[6]
|
||||
|
||||
local jobId
|
||||
local jobIdKey
|
||||
local rcall = redis.call
|
||||
|
||||
local args = cmsgpack.unpack(ARGV[1])
|
||||
|
||||
local data = ARGV[2]
|
||||
local opts = cmsgpack.unpack(ARGV[3])
|
||||
|
||||
local parentKey = args[5]
|
||||
local repeatJobKey = args[9]
|
||||
local parent = args[8]
|
||||
local parentData
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/storeJob"
|
||||
--- @include "includes/updateExistingJobsParent"
|
||||
--- @include "includes/getTargetQueueList"
|
||||
--- @include "includes/getOrSetMaxEvents"
|
||||
|
||||
if parentKey ~= nil then
|
||||
if rcall("EXISTS", parentKey) ~= 1 then return -5 end
|
||||
|
||||
parentData = cjson.encode(parent)
|
||||
end
|
||||
|
||||
local jobCounter = rcall("INCR", KEYS[4])
|
||||
|
||||
local metaKey = KEYS[3]
|
||||
local maxEvents = getOrSetMaxEvents(metaKey)
|
||||
|
||||
local parentDependenciesKey = args[7]
|
||||
local timestamp = args[4]
|
||||
if args[2] == "" then
|
||||
jobId = jobCounter
|
||||
jobIdKey = args[1] .. jobId
|
||||
else
|
||||
jobId = args[2]
|
||||
jobIdKey = args[1] .. jobId
|
||||
if rcall("EXISTS", jobIdKey) == 1 then
|
||||
updateExistingJobsParent(parentKey, parent, parentData,
|
||||
parentDependenciesKey, KEYS[5], jobIdKey,
|
||||
jobId, timestamp)
|
||||
|
||||
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
|
||||
"duplicated", "jobId", jobId)
|
||||
|
||||
return jobId .. "" -- convert to string
|
||||
end
|
||||
end
|
||||
|
||||
-- Store the job.
|
||||
storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2], opts, timestamp,
|
||||
parentKey, parentData, repeatJobKey)
|
||||
|
||||
local target, paused = getTargetQueueList(metaKey, KEYS[1], KEYS[2])
|
||||
|
||||
-- LIFO or FIFO
|
||||
local pushCmd = opts['lifo'] and 'RPUSH' or 'LPUSH'
|
||||
rcall(pushCmd, target, jobId)
|
||||
|
||||
-- Emit waiting event
|
||||
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting",
|
||||
"jobId", jobId)
|
||||
|
||||
-- Check if this job is a child of another job, if so add it to the parents dependencies
|
||||
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
|
||||
-- fail in this case.
|
||||
if parentDependenciesKey ~= nil then
|
||||
rcall("SADD", parentDependenciesKey, jobIdKey)
|
||||
end
|
||||
|
||||
return jobId .. "" -- convert to string
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
--[[
|
||||
Change job delay when it is in delayed set.
|
||||
Input:
|
||||
KEYS[1] delayed key
|
||||
KEYS[2] job key
|
||||
KEYS[3] events stream
|
||||
|
||||
ARGV[1] delay
|
||||
ARGV[2] delayedTimestamp
|
||||
ARGV[3] the id of the job
|
||||
Output:
|
||||
0 - OK
|
||||
-1 - Missing job.
|
||||
-3 - Job not in delayed set.
|
||||
|
||||
Events:
|
||||
- delayed key.
|
||||
]]
|
||||
local rcall = redis.call
|
||||
|
||||
if rcall("EXISTS", KEYS[2]) == 1 then
|
||||
local jobId = ARGV[3]
|
||||
local score = tonumber(ARGV[2])
|
||||
local delayedTimestamp = (score / 0x1000)
|
||||
|
||||
local numRemovedElements = rcall("ZREM", KEYS[1], jobId)
|
||||
|
||||
if numRemovedElements < 1 then
|
||||
return -3
|
||||
end
|
||||
|
||||
rcall("HSET", KEYS[2], "delay", tonumber(ARGV[1]))
|
||||
rcall("ZADD", KEYS[1], score, jobId)
|
||||
|
||||
rcall("XADD", KEYS[3], "*", "event", "delayed", "jobId", jobId, "delay", delayedTimestamp)
|
||||
|
||||
return 0
|
||||
else
|
||||
return -1
|
||||
end
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
--[[
|
||||
Change job priority
|
||||
Input:
|
||||
KEYS[1] 'wait',
|
||||
KEYS[2] 'paused'
|
||||
KEYS[3] 'meta'
|
||||
KEYS[4] 'prioritized'
|
||||
KEYS[5] 'pc' priority counter
|
||||
|
||||
ARGV[1] priority value
|
||||
ARGV[2] job key
|
||||
ARGV[3] job id
|
||||
ARGV[4] lifo
|
||||
|
||||
Output:
|
||||
0 - OK
|
||||
-1 - Missing job
|
||||
]]
|
||||
local jobKey = ARGV[2]
|
||||
local jobId = ARGV[3]
|
||||
local priority = tonumber(ARGV[1])
|
||||
local rcall = redis.call
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/addJobWithPriority"
|
||||
--- @include "includes/getTargetQueueList"
|
||||
|
||||
if rcall("EXISTS", jobKey) == 1 then
|
||||
local target, paused = getTargetQueueList(KEYS[3], KEYS[1], KEYS[2])
|
||||
|
||||
if rcall("ZREM", KEYS[4], jobId) > 0 then
|
||||
addJobWithPriority(KEYS[1], KEYS[4], priority, paused, jobId, KEYS[5])
|
||||
else
|
||||
local numRemovedElements = rcall("LREM", target, -1, jobId)
|
||||
if numRemovedElements > 0 then
|
||||
-- Standard or priority add
|
||||
if priority == 0 then
|
||||
-- LIFO or FIFO
|
||||
local pushCmd = ARGV[4] == '1' and 'RPUSH' or 'LPUSH';
|
||||
rcall(pushCmd, target, jobId)
|
||||
else
|
||||
addJobWithPriority(KEYS[1], KEYS[4], priority, paused, jobId, KEYS[5])
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
rcall("HSET", jobKey, "priority", priority)
|
||||
|
||||
return 0
|
||||
else
|
||||
return -1
|
||||
end
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
--[[
|
||||
Remove jobs from the specific set.
|
||||
|
||||
Input:
|
||||
KEYS[1] set key,
|
||||
KEYS[2] events stream key
|
||||
|
||||
ARGV[1] jobKey prefix
|
||||
ARGV[2] timestamp
|
||||
ARGV[3] limit the number of jobs to be removed. 0 is unlimited
|
||||
ARGV[4] set name, can be any of 'wait', 'active', 'paused', 'delayed', 'completed', or 'failed'
|
||||
]]
|
||||
local rcall = redis.call
|
||||
local rangeStart = 0
|
||||
local rangeEnd = -1
|
||||
|
||||
local limit = tonumber(ARGV[3])
|
||||
|
||||
-- If we're only deleting _n_ items, avoid retrieving all items
|
||||
-- for faster performance
|
||||
--
|
||||
-- Start from the tail of the list, since that's where oldest elements
|
||||
-- are generally added for FIFO lists
|
||||
if limit > 0 then
|
||||
rangeStart = -1 - limit + 1
|
||||
rangeEnd = -1
|
||||
end
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/cleanList"
|
||||
--- @include "includes/cleanSet"
|
||||
|
||||
local result
|
||||
if ARGV[4] == "active" then
|
||||
result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], false)
|
||||
elseif ARGV[4] == "delayed" then
|
||||
rangeEnd = "+inf"
|
||||
result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
|
||||
{"processedOn", "timestamp"}, false)
|
||||
elseif ARGV[4] == "prioritized" then
|
||||
rangeEnd = "+inf"
|
||||
result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
|
||||
{"timestamp"}, false)
|
||||
elseif ARGV[4] == "wait" or ARGV[4] == "paused" then
|
||||
result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], true)
|
||||
else
|
||||
rangeEnd = ARGV[2]
|
||||
result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
|
||||
{"finishedOn"}, true)
|
||||
end
|
||||
|
||||
rcall("XADD", KEYS[2], "*", "event", "cleaned", "count", result[2])
|
||||
|
||||
return result[1]
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
--[[
|
||||
Drains the queue, removes all jobs that are waiting
|
||||
or delayed, but not active, completed or failed
|
||||
|
||||
Input:
|
||||
KEYS[1] 'wait',
|
||||
KEYS[2] 'paused'
|
||||
KEYS[3] 'delayed'
|
||||
KEYS[4] 'prioritized'
|
||||
|
||||
ARGV[1] queue key prefix
|
||||
]]
|
||||
local rcall = redis.call
|
||||
local queueBaseKey = ARGV[1]
|
||||
|
||||
--- @include "includes/removeListJobs"
|
||||
--- @include "includes/removeZSetJobs"
|
||||
|
||||
removeListJobs(KEYS[1], true, queueBaseKey, 0) --wait
|
||||
removeListJobs(KEYS[2], true, queueBaseKey, 0) --paused
|
||||
|
||||
if KEYS[3] ~= "" then
|
||||
removeZSetJobs(KEYS[3], true, queueBaseKey, 0) --delayed
|
||||
end
|
||||
|
||||
removeZSetJobs(KEYS[4], true, queueBaseKey, 0) --prioritized
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
--[[
|
||||
Extend lock and removes the job from the stalled set.
|
||||
|
||||
Input:
|
||||
KEYS[1] 'lock',
|
||||
KEYS[2] 'stalled'
|
||||
|
||||
ARGV[1] token
|
||||
ARGV[2] lock duration in milliseconds
|
||||
ARGV[3] jobid
|
||||
|
||||
Output:
|
||||
"1" if lock extented succesfully.
|
||||
]]
|
||||
local rcall = redis.call
|
||||
if rcall("GET", KEYS[1]) == ARGV[1] then
|
||||
-- if rcall("SET", KEYS[1], ARGV[1], "PX", ARGV[2], "XX") then
|
||||
if rcall("SET", KEYS[1], ARGV[1], "PX", ARGV[2]) then
|
||||
rcall("SREM", KEYS[2], ARGV[3])
|
||||
return 1
|
||||
end
|
||||
end
|
||||
return 0
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
--[[
|
||||
Get counts per provided states
|
||||
|
||||
Input:
|
||||
KEYS[1] 'prefix'
|
||||
|
||||
ARGV[1...] types
|
||||
]]
|
||||
local rcall = redis.call;
|
||||
local prefix = KEYS[1]
|
||||
local results = {}
|
||||
|
||||
for i = 1, #ARGV do
|
||||
local stateKey = prefix .. ARGV[i]
|
||||
if ARGV[i] == "wait" or ARGV[i] == "paused" then
|
||||
local marker = rcall("LINDEX", stateKey, -1)
|
||||
if marker and string.sub(marker, 1, 2) == "0:" then
|
||||
local count = rcall("LLEN", stateKey)
|
||||
if count > 1 then
|
||||
rcall("RPOP", stateKey)
|
||||
results[#results+1] = count-1
|
||||
else
|
||||
results[#results+1] = 0
|
||||
end
|
||||
else
|
||||
results[#results+1] = rcall("LLEN", stateKey)
|
||||
end
|
||||
elseif ARGV[i] == "active" then
|
||||
results[#results+1] = rcall("LLEN", stateKey)
|
||||
else
|
||||
results[#results+1] = rcall("ZCARD", stateKey)
|
||||
end
|
||||
end
|
||||
|
||||
return results
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
--[[
|
||||
Get job ids per provided states
|
||||
|
||||
Input:
|
||||
KEYS[1] 'prefix'
|
||||
|
||||
ARGV[1] start
|
||||
ARGV[2] end
|
||||
ARGV[3] asc
|
||||
ARGV[4...] types
|
||||
]]
|
||||
local rcall = redis.call
|
||||
local prefix = KEYS[1]
|
||||
local rangeStart = tonumber(ARGV[1])
|
||||
local rangeEnd = tonumber(ARGV[2])
|
||||
local asc = ARGV[3]
|
||||
local results = {}
|
||||
|
||||
local function getRangeInList(listKey, asc, rangeStart, rangeEnd, results)
|
||||
if asc == "1" then
|
||||
local modifiedRangeStart
|
||||
local modifiedRangeEnd
|
||||
if rangeStart == -1 then
|
||||
modifiedRangeStart = 0
|
||||
else
|
||||
modifiedRangeStart = -(rangeStart + 1)
|
||||
end
|
||||
|
||||
if rangeEnd == -1 then
|
||||
modifiedRangeEnd = 0
|
||||
else
|
||||
modifiedRangeEnd = -(rangeEnd + 1)
|
||||
end
|
||||
|
||||
results[#results+1] = rcall("LRANGE", listKey,
|
||||
modifiedRangeEnd,
|
||||
modifiedRangeStart)
|
||||
else
|
||||
results[#results+1] = rcall("LRANGE", listKey, rangeStart, rangeEnd)
|
||||
end
|
||||
end
|
||||
|
||||
for i = 4, #ARGV do
|
||||
local stateKey = prefix .. ARGV[i]
|
||||
if ARGV[i] == "wait" or ARGV[i] == "paused" then
|
||||
local marker = rcall("LINDEX", stateKey, -1)
|
||||
if marker and string.sub(marker, 1, 2) == "0:" then
|
||||
local count = rcall("LLEN", stateKey)
|
||||
if count > 1 then
|
||||
rcall("RPOP", stateKey)
|
||||
getRangeInList(stateKey, asc, rangeStart, rangeEnd, results)
|
||||
else
|
||||
results[#results+1] = {}
|
||||
end
|
||||
else
|
||||
getRangeInList(stateKey, asc, rangeStart, rangeEnd, results)
|
||||
end
|
||||
elseif ARGV[i] == "active" then
|
||||
getRangeInList(stateKey, asc, rangeStart, rangeEnd, results)
|
||||
else
|
||||
if asc == "1" then
|
||||
results[#results+1] = rcall("ZRANGE", stateKey, rangeStart, rangeEnd)
|
||||
else
|
||||
results[#results+1] = rcall("ZREVRANGE", stateKey, rangeStart, rangeEnd)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
return results
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
--[[
|
||||
Get a job state
|
||||
|
||||
Input:
|
||||
KEYS[1] 'completed' key,
|
||||
KEYS[2] 'failed' key
|
||||
KEYS[3] 'delayed' key
|
||||
KEYS[4] 'active' key
|
||||
KEYS[5] 'wait' key
|
||||
KEYS[6] 'paused' key
|
||||
KEYS[7] 'waiting-children' key
|
||||
KEYS[8] 'prioritized' key
|
||||
|
||||
ARGV[1] job id
|
||||
Output:
|
||||
'completed'
|
||||
'failed'
|
||||
'delayed'
|
||||
'active'
|
||||
'prioritized'
|
||||
'waiting'
|
||||
'waiting-children'
|
||||
'unknown'
|
||||
]]
|
||||
local rcall = redis.call
|
||||
|
||||
if rcall("ZSCORE", KEYS[1], ARGV[1]) ~= false then
|
||||
return "completed"
|
||||
end
|
||||
|
||||
if rcall("ZSCORE", KEYS[2], ARGV[1]) ~= false then
|
||||
return "failed"
|
||||
end
|
||||
|
||||
if rcall("ZSCORE", KEYS[3], ARGV[1]) ~= false then
|
||||
return "delayed"
|
||||
end
|
||||
|
||||
if rcall("ZSCORE", KEYS[8], ARGV[1]) ~= false then
|
||||
return "prioritized"
|
||||
end
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/checkItemInList"
|
||||
|
||||
local active_items = rcall("LRANGE", KEYS[4] , 0, -1)
|
||||
if checkItemInList(active_items, ARGV[1]) ~= nil then
|
||||
return "active"
|
||||
end
|
||||
|
||||
local wait_items = rcall("LRANGE", KEYS[5] , 0, -1)
|
||||
if checkItemInList(wait_items, ARGV[1]) ~= nil then
|
||||
return "waiting"
|
||||
end
|
||||
|
||||
local paused_items = rcall("LRANGE", KEYS[6] , 0, -1)
|
||||
if checkItemInList(paused_items, ARGV[1]) ~= nil then
|
||||
return "waiting"
|
||||
end
|
||||
|
||||
if rcall("ZSCORE", KEYS[7], ARGV[1]) ~= false then
|
||||
return "waiting-children"
|
||||
end
|
||||
|
||||
return "unknown"
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
--[[
|
||||
Get a job state
|
||||
|
||||
Input:
|
||||
KEYS[1] 'completed' key,
|
||||
KEYS[2] 'failed' key
|
||||
KEYS[3] 'delayed' key
|
||||
KEYS[4] 'active' key
|
||||
KEYS[5] 'wait' key
|
||||
KEYS[6] 'paused' key
|
||||
KEYS[7] 'waiting-children' key
|
||||
KEYS[8] 'prioritized' key
|
||||
|
||||
ARGV[1] job id
|
||||
Output:
|
||||
'completed'
|
||||
'failed'
|
||||
'delayed'
|
||||
'active'
|
||||
'waiting'
|
||||
'waiting-children'
|
||||
'unknown'
|
||||
]]
|
||||
local rcall = redis.call
|
||||
|
||||
if rcall("ZSCORE", KEYS[1], ARGV[1]) ~= false then
|
||||
return "completed"
|
||||
end
|
||||
|
||||
if rcall("ZSCORE", KEYS[2], ARGV[1]) ~= false then
|
||||
return "failed"
|
||||
end
|
||||
|
||||
if rcall("ZSCORE", KEYS[3], ARGV[1]) ~= false then
|
||||
return "delayed"
|
||||
end
|
||||
|
||||
if rcall("ZSCORE", KEYS[8], ARGV[1]) ~= false then
|
||||
return "prioritized"
|
||||
end
|
||||
|
||||
if rcall("LPOS", KEYS[4] , ARGV[1]) ~= false then
|
||||
return "active"
|
||||
end
|
||||
|
||||
if rcall("LPOS", KEYS[5] , ARGV[1]) ~= false then
|
||||
return "waiting"
|
||||
end
|
||||
|
||||
if rcall("LPOS", KEYS[6] , ARGV[1]) ~= false then
|
||||
return "waiting"
|
||||
end
|
||||
|
||||
if rcall("ZSCORE", KEYS[7] , ARGV[1]) ~= false then
|
||||
return "waiting-children"
|
||||
end
|
||||
|
||||
return "unknown"
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
--[[
|
||||
Add delay marker if needed.
|
||||
]]
|
||||
|
||||
-- Includes
|
||||
--- @include "getNextDelayedTimestamp"
|
||||
|
||||
local function addDelayMarkerIfNeeded(targetKey, delayedKey)
|
||||
local waitLen = rcall("LLEN", targetKey)
|
||||
if waitLen <= 1 then
|
||||
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
|
||||
if nextTimestamp ~= nil then
|
||||
-- Check if there is already a marker with older timestamp
|
||||
-- if there is, we need to replace it.
|
||||
if waitLen == 1 then
|
||||
local marker = rcall("LINDEX", targetKey, 0)
|
||||
local oldTimestamp = tonumber(marker:sub(3))
|
||||
if oldTimestamp and oldTimestamp > nextTimestamp then
|
||||
rcall("LSET", targetKey, 0, "0:" .. nextTimestamp)
|
||||
end
|
||||
else
|
||||
-- if there is no marker, then we need to add one
|
||||
rcall("LPUSH", targetKey, "0:" .. nextTimestamp)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
--[[
|
||||
Function to add job considering priority.
|
||||
]]
|
||||
|
||||
-- Includes
|
||||
--- @include "addPriorityMarkerIfNeeded"
|
||||
--- @include "getPriorityScore"
|
||||
|
||||
local function addJobWithPriority(waitKey, prioritizedKey, priority, paused, jobId, priorityCounterKey)
|
||||
local score = getPriorityScore(priority, priorityCounterKey)
|
||||
rcall("ZADD", prioritizedKey, score, jobId)
|
||||
if not paused then
|
||||
addPriorityMarkerIfNeeded(waitKey)
|
||||
end
|
||||
end
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
--[[
|
||||
Function priority marker to wait if needed
|
||||
in order to wake up our workers and to respect priority
|
||||
order as much as possible
|
||||
]]
|
||||
local function addPriorityMarkerIfNeeded(waitKey)
|
||||
local waitLen = rcall("LLEN", waitKey)
|
||||
|
||||
if waitLen == 0 then
|
||||
rcall("LPUSH", waitKey, "0:0")
|
||||
end
|
||||
end
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
--[[
|
||||
Function to loop in batches.
|
||||
Just a bit of warning, some commands as ZREM
|
||||
could receive a maximum of 7000 parameters per call.
|
||||
]]
|
||||
|
||||
local function batches(n, batchSize)
|
||||
local i = 0
|
||||
|
||||
return function()
|
||||
local from = i * batchSize + 1
|
||||
i = i + 1
|
||||
if (from <= n) then
|
||||
local to = math.min(from + batchSize - 1, n)
|
||||
return from, to
|
||||
end
|
||||
end
|
||||
end
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
--[[
|
||||
Functions to check if a item belongs to a list.
|
||||
]]
|
||||
|
||||
local function checkItemInList(list, item)
|
||||
for _, v in pairs(list) do
|
||||
if v == item then
|
||||
return 1
|
||||
end
|
||||
end
|
||||
return nil
|
||||
end
|
||||
+139
@@ -0,0 +1,139 @@
|
||||
--[[
|
||||
Move stalled jobs to wait.
|
||||
|
||||
Input:
|
||||
stalledKey 'stalled' (SET)
|
||||
waitKey 'wait', (LIST)
|
||||
activeKey 'active', (LIST)
|
||||
failedKey 'failed', (ZSET)
|
||||
stalledCheckKey 'stalled-check', (KEY)
|
||||
metaKey 'meta', (KEY)
|
||||
pausedKey 'paused', (LIST)
|
||||
eventStreamKey 'event stream' (STREAM)
|
||||
|
||||
maxStalledJobCount Max stalled job count
|
||||
queueKeyPrefix queue.toKey('')
|
||||
timestamp timestamp
|
||||
maxCheckTime max check time
|
||||
|
||||
Events:
|
||||
'stalled' with stalled job id.
|
||||
]]
|
||||
local rcall = redis.call
|
||||
|
||||
-- Includes
|
||||
--- @include "batches"
|
||||
--- @include "getTargetQueueList"
|
||||
--- @include "removeJob"
|
||||
--- @include "removeJobsByMaxAge"
|
||||
--- @include "removeJobsByMaxCount"
|
||||
--- @include "trimEvents"
|
||||
|
||||
-- Check if we need to check for stalled jobs now.
|
||||
|
||||
local function checkStalledJobs(stalledKey, waitKey, activeKey, failedKey,
|
||||
stalledCheckKey, metaKey, pausedKey,
|
||||
eventStreamKey, maxStalledJobCount,
|
||||
queueKeyPrefix, timestamp, maxCheckTime)
|
||||
if rcall("EXISTS", stalledCheckKey) == 1 then return {{}, {}} end
|
||||
|
||||
rcall("SET", stalledCheckKey, timestamp, "PX", maxCheckTime)
|
||||
|
||||
-- Trim events before emiting them to avoid trimming events emitted in this script
|
||||
trimEvents(metaKey, eventStreamKey)
|
||||
|
||||
-- Move all stalled jobs to wait
|
||||
local stalling = rcall('SMEMBERS', stalledKey)
|
||||
local stalled = {}
|
||||
local failed = {}
|
||||
if (#stalling > 0) then
|
||||
rcall('DEL', stalledKey)
|
||||
|
||||
local MAX_STALLED_JOB_COUNT = tonumber(maxStalledJobCount)
|
||||
|
||||
-- Remove from active list
|
||||
for i, jobId in ipairs(stalling) do
|
||||
|
||||
if string.sub(jobId, 1, 2) == "0:" then
|
||||
-- If the jobId is a delay marker ID we just remove it.
|
||||
rcall("LREM", activeKey, 1, jobId)
|
||||
else
|
||||
local jobKey = queueKeyPrefix .. jobId
|
||||
|
||||
-- Check that the lock is also missing, then we can handle this job as really stalled.
|
||||
if (rcall("EXISTS", jobKey .. ":lock") == 0) then
|
||||
-- Remove from the active queue.
|
||||
local removed = rcall("LREM", activeKey, 1, jobId)
|
||||
|
||||
if (removed > 0) then
|
||||
-- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
|
||||
local stalledCount =
|
||||
rcall("HINCRBY", jobKey, "stalledCounter", 1)
|
||||
if (stalledCount > MAX_STALLED_JOB_COUNT) then
|
||||
local rawOpts = rcall("HGET", jobKey, "opts")
|
||||
local opts = cjson.decode(rawOpts)
|
||||
local removeOnFailType = type(opts["removeOnFail"])
|
||||
rcall("ZADD", failedKey, timestamp, jobId)
|
||||
local failedReason =
|
||||
"job stalled more than allowable limit"
|
||||
rcall("HMSET", jobKey, "failedReason", failedReason,
|
||||
"finishedOn", timestamp)
|
||||
rcall("XADD", eventStreamKey, "*", "event",
|
||||
"failed", "jobId", jobId, 'prev', 'active',
|
||||
'failedReason', failedReason)
|
||||
|
||||
if removeOnFailType == "number" then
|
||||
removeJobsByMaxCount(opts["removeOnFail"],
|
||||
failedKey, queueKeyPrefix)
|
||||
elseif removeOnFailType == "boolean" then
|
||||
if opts["removeOnFail"] then
|
||||
removeJob(jobId, false, queueKeyPrefix)
|
||||
rcall("ZREM", failedKey, jobId)
|
||||
end
|
||||
elseif removeOnFailType ~= "nil" then
|
||||
local maxAge = opts["removeOnFail"]["age"]
|
||||
local maxCount = opts["removeOnFail"]["count"]
|
||||
|
||||
if maxAge ~= nil then
|
||||
removeJobsByMaxAge(timestamp, maxAge,
|
||||
failedKey, queueKeyPrefix)
|
||||
end
|
||||
|
||||
if maxCount ~= nil and maxCount > 0 then
|
||||
removeJobsByMaxCount(maxCount, failedKey,
|
||||
queueKeyPrefix)
|
||||
end
|
||||
end
|
||||
|
||||
table.insert(failed, jobId)
|
||||
else
|
||||
local target =
|
||||
getTargetQueueList(metaKey, waitKey, pausedKey)
|
||||
|
||||
-- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
|
||||
rcall("RPUSH", target, jobId)
|
||||
rcall("XADD", eventStreamKey, "*", "event",
|
||||
"waiting", "jobId", jobId, 'prev', 'active')
|
||||
|
||||
-- Emit the stalled event
|
||||
rcall("XADD", eventStreamKey, "*", "event",
|
||||
"stalled", "jobId", jobId)
|
||||
table.insert(stalled, jobId)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
-- Mark potentially stalled jobs
|
||||
local active = rcall('LRANGE', activeKey, 0, -1)
|
||||
|
||||
if (#active > 0) then
|
||||
for from, to in batches(#active, 7000) do
|
||||
rcall('SADD', stalledKey, unpack(active, from, to))
|
||||
end
|
||||
end
|
||||
|
||||
return {failed, stalled}
|
||||
end
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
--[[
|
||||
Function to clean job list.
|
||||
Returns jobIds and deleted count number.
|
||||
]]
|
||||
|
||||
-- Includes
|
||||
--- @include "getTimestamp"
|
||||
--- @include "removeJob"
|
||||
|
||||
local function cleanList(listKey, jobKeyPrefix, rangeStart, rangeEnd,
|
||||
timestamp, isWaiting)
|
||||
local jobs = rcall("LRANGE", listKey, rangeStart, rangeEnd)
|
||||
local deleted = {}
|
||||
local deletedCount = 0
|
||||
local jobTS
|
||||
local deletionMarker = ''
|
||||
local jobIdsLen = #jobs
|
||||
for i, job in ipairs(jobs) do
|
||||
if limit > 0 and deletedCount >= limit then
|
||||
break
|
||||
end
|
||||
|
||||
local jobKey = jobKeyPrefix .. job
|
||||
if (isWaiting or rcall("EXISTS", jobKey .. ":lock") == 0) then
|
||||
-- Find the right timestamp of the job to compare to maxTimestamp:
|
||||
-- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed
|
||||
-- * processedOn represents when the job was last attempted, but it doesn't get populated until
|
||||
-- the job is first tried
|
||||
-- * timestamp is the original job submission time
|
||||
-- Fetch all three of these (in that order) and use the first one that is set so that we'll leave jobs
|
||||
-- that have been active within the grace period:
|
||||
jobTS = getTimestamp(jobKey, {"finishedOn", "processedOn", "timestamp"})
|
||||
if (not jobTS or jobTS <= timestamp) then
|
||||
-- replace the entry with a deletion marker; the actual deletion will
|
||||
-- occur at the end of the script
|
||||
rcall("LSET", listKey, rangeEnd - jobIdsLen + i, deletionMarker)
|
||||
removeJob(job, true, jobKeyPrefix)
|
||||
deletedCount = deletedCount + 1
|
||||
table.insert(deleted, job)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
rcall("LREM", listKey, 0, deletionMarker)
|
||||
|
||||
return {deleted, deletedCount}
|
||||
end
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
--[[
|
||||
Function to clean job set.
|
||||
Returns jobIds and deleted count number.
|
||||
]]
|
||||
|
||||
-- Includes
|
||||
--- @include "batches"
|
||||
--- @include "getJobsInZset"
|
||||
--- @include "getTimestamp"
|
||||
--- @include "removeJob"
|
||||
|
||||
local function cleanSet(setKey, jobKeyPrefix, rangeEnd, timestamp, limit, attributes, isFinished)
|
||||
local jobs = getJobsInZset(setKey, rangeEnd, limit)
|
||||
local deleted = {}
|
||||
local deletedCount = 0
|
||||
local jobTS
|
||||
for i, job in ipairs(jobs) do
|
||||
if limit > 0 and deletedCount >= limit then
|
||||
break
|
||||
end
|
||||
|
||||
local jobKey = jobKeyPrefix .. job
|
||||
if isFinished then
|
||||
removeJob(job, true, jobKeyPrefix)
|
||||
deletedCount = deletedCount + 1
|
||||
table.insert(deleted, job)
|
||||
else
|
||||
-- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed
|
||||
jobTS = getTimestamp(jobKey, attributes)
|
||||
if (not jobTS or jobTS <= timestamp) then
|
||||
removeJob(job, true, jobKeyPrefix)
|
||||
deletedCount = deletedCount + 1
|
||||
table.insert(deleted, job)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
if(#deleted > 0) then
|
||||
for from, to in batches(#deleted, 7000) do
|
||||
rcall("ZREM", setKey, unpack(deleted, from, to))
|
||||
end
|
||||
end
|
||||
|
||||
return {deleted, deletedCount}
|
||||
end
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
--[[
|
||||
Functions to collect metrics based on a current and previous count of jobs.
|
||||
Granualarity is fixed at 1 minute.
|
||||
]]
|
||||
--- @include "batches"
|
||||
local function collectMetrics(metaKey, dataPointsList, maxDataPoints,
|
||||
timestamp)
|
||||
-- Increment current count
|
||||
local count = rcall("HINCRBY", metaKey, "count", 1) - 1
|
||||
|
||||
-- Compute how many data points we need to add to the list, N.
|
||||
local prevTS = rcall("HGET", metaKey, "prevTS")
|
||||
|
||||
if not prevTS then
|
||||
-- If prevTS is nil, set it to the current timestamp
|
||||
rcall("HSET", metaKey, "prevTS", timestamp, "prevCount", 0)
|
||||
return
|
||||
end
|
||||
|
||||
local N = math.floor((timestamp - prevTS) / 60000)
|
||||
|
||||
if N > 0 then
|
||||
local delta = count - rcall("HGET", metaKey, "prevCount")
|
||||
-- If N > 1, add N-1 zeros to the list
|
||||
if N > 1 then
|
||||
local points = {}
|
||||
points[1] = delta
|
||||
for i = 2, N do
|
||||
points[i] = 0
|
||||
end
|
||||
|
||||
for from, to in batches(#points, 7000) do
|
||||
rcall("LPUSH", dataPointsList, unpack(points, from, to))
|
||||
end
|
||||
else
|
||||
-- LPUSH delta to the list
|
||||
rcall("LPUSH", dataPointsList, delta)
|
||||
end
|
||||
|
||||
-- LTRIM to keep list to its max size
|
||||
rcall("LTRIM", dataPointsList, 0, maxDataPoints - 1)
|
||||
|
||||
-- update prev count with current count
|
||||
rcall("HSET", metaKey, "prevCount", count, "prevTS", timestamp)
|
||||
end
|
||||
end
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
--[[
|
||||
Functions to destructure job key.
|
||||
Just a bit of warning, these functions may be a bit slow and affect performance significantly.
|
||||
]]
|
||||
|
||||
local getJobIdFromKey = function (jobKey)
|
||||
return string.match(jobKey, ".*:(.*)")
|
||||
end
|
||||
|
||||
local getJobKeyPrefix = function (jobKey, jobId)
|
||||
return string.sub(jobKey, 0, #jobKey - #jobId)
|
||||
end
|
||||
+70
@@ -0,0 +1,70 @@
|
||||
--[[
|
||||
Function to achieve pagination for a set or hash.
|
||||
This function simulates pagination in the most efficient way possible
|
||||
for a set using sscan or hscan.
|
||||
|
||||
The main limitation is that sets are not order preserving, so the
|
||||
pagination is not stable. This means that if the set is modified
|
||||
between pages, the same element may appear in different pages.
|
||||
]] -- Maximum number of elements to be returned by sscan per iteration.
|
||||
local maxCount = 100
|
||||
|
||||
-- Finds the cursor, and returns the first elements available for the requested page.
|
||||
local function findPage(key, command, pageStart, pageSize, cursor, offset,
|
||||
maxIterations, fetchJobs)
|
||||
local items = {}
|
||||
local jobs = {}
|
||||
local iterations = 0
|
||||
|
||||
repeat
|
||||
-- Iterate over the set using sscan/hscan.
|
||||
local result = rcall(command, key, cursor, "COUNT", maxCount)
|
||||
|
||||
cursor = result[1]
|
||||
local members = result[2]
|
||||
local step = 1
|
||||
if command == "HSCAN" then
|
||||
step = 2
|
||||
end
|
||||
|
||||
if #members == 0 then
|
||||
-- If the result is empty, we can return the result.
|
||||
return cursor, offset, items, jobs
|
||||
end
|
||||
|
||||
local chunkStart = offset
|
||||
local chunkEnd = offset + #members / step
|
||||
|
||||
local pageEnd = pageStart + pageSize
|
||||
|
||||
if chunkEnd < pageStart then
|
||||
-- If the chunk is before the page, we can skip it.
|
||||
offset = chunkEnd
|
||||
elseif chunkStart > pageEnd then
|
||||
-- If the chunk is after the page, we can return the result.
|
||||
return cursor, offset, items, jobs
|
||||
else
|
||||
-- If the chunk is overlapping the page, we need to add the elements to the result.
|
||||
for i = 1, #members, step do
|
||||
if offset >= pageEnd then
|
||||
return cursor, offset, items, jobs
|
||||
end
|
||||
if offset >= pageStart then
|
||||
local index = #items + 1
|
||||
if fetchJobs ~= nil then
|
||||
jobs[#jobs+1] = rcall("HGETALL", members[i])
|
||||
end
|
||||
if step == 2 then
|
||||
items[index] = {members[i], members[i + 1]}
|
||||
else
|
||||
items[index] = members[i]
|
||||
end
|
||||
end
|
||||
offset = offset + 1
|
||||
end
|
||||
end
|
||||
iterations = iterations + 1
|
||||
until cursor == "0" or iterations >= maxIterations
|
||||
|
||||
return cursor, offset, items, jobs
|
||||
end
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
-- We use ZRANGEBYSCORE to make the case where we're deleting a limited number
|
||||
-- of items in a sorted set only run a single iteration. If we simply used
|
||||
-- ZRANGE, we may take a long time traversing through jobs that are within the
|
||||
-- grace period.
|
||||
local function getJobsInZset(zsetKey, rangeEnd, limit)
|
||||
if limit > 0 then
|
||||
return rcall("ZRANGEBYSCORE", zsetKey, 0, rangeEnd, "LIMIT", 0, limit)
|
||||
else
|
||||
return rcall("ZRANGEBYSCORE", zsetKey, 0, rangeEnd)
|
||||
end
|
||||
end
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
--[[
|
||||
Function to return the next delayed job timestamp.
|
||||
]]
|
||||
local function getNextDelayedTimestamp(delayedKey)
|
||||
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
|
||||
if #result then
|
||||
local nextTimestamp = tonumber(result[2])
|
||||
if (nextTimestamp ~= nil) then
|
||||
nextTimestamp = nextTimestamp / 0x1000
|
||||
end
|
||||
return nextTimestamp
|
||||
end
|
||||
end
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
|
||||
local function getOrSetMaxEvents(metaKey)
|
||||
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
||||
if not maxEvents then
|
||||
maxEvents = 10000
|
||||
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
|
||||
end
|
||||
return maxEvents
|
||||
end
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
--[[
|
||||
Function to get priority score.
|
||||
]]
|
||||
local function getPriorityScore(priority, priorityCounterKey)
|
||||
local prioCounter = rcall("INCR", priorityCounterKey)
|
||||
return priority * 0x100000000 + prioCounter % 0x100000000
|
||||
end
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
local function getRateLimitTTL(maxJobs, rateLimiterKey)
|
||||
if maxJobs and maxJobs <= tonumber(rcall("GET", rateLimiterKey) or 0) then
|
||||
local pttl = rcall("PTTL", rateLimiterKey)
|
||||
|
||||
if pttl == 0 then
|
||||
rcall("DEL", rateLimiterKey)
|
||||
end
|
||||
|
||||
if pttl > 0 then
|
||||
return pttl
|
||||
end
|
||||
end
|
||||
return 0
|
||||
end
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
--[[
|
||||
Function to check for the meta.paused key to decide if we are paused or not
|
||||
(since an empty list and !EXISTS are not really the same).
|
||||
]]
|
||||
|
||||
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
|
||||
if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
|
||||
return waitKey, false
|
||||
else
|
||||
return pausedKey, true
|
||||
end
|
||||
end
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
--[[
|
||||
Function to get the latest saved timestamp.
|
||||
]]
|
||||
|
||||
local function getTimestamp(jobKey, attributes)
|
||||
if #attributes == 1 then
|
||||
return rcall("HGET", jobKey, attributes[1])
|
||||
end
|
||||
|
||||
local jobTs
|
||||
for _, ts in ipairs(rcall("HMGET", jobKey, unpack(attributes))) do
|
||||
if (ts) then
|
||||
jobTs = ts
|
||||
break
|
||||
end
|
||||
end
|
||||
|
||||
return jobTs
|
||||
end
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
--[[
|
||||
Function to get ZSet items.
|
||||
]]
|
||||
|
||||
local function getZSetItems(keyName, max)
|
||||
return rcall('ZRANGE', keyName, 0, max - 1)
|
||||
end
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
--[[
|
||||
Function to recursively check if there are no locks
|
||||
on the jobs to be removed.
|
||||
|
||||
returns:
|
||||
boolean
|
||||
]]
|
||||
|
||||
local function isLocked( prefix, jobId, removeChildren)
|
||||
local jobKey = prefix .. jobId;
|
||||
|
||||
-- Check if this job is locked
|
||||
local lockKey = jobKey .. ':lock'
|
||||
local lock = rcall("GET", lockKey)
|
||||
if not lock then
|
||||
if removeChildren == "1" then
|
||||
local dependencies = rcall("SMEMBERS", jobKey .. ":dependencies")
|
||||
if (#dependencies > 0) then
|
||||
for i, childJobKey in ipairs(dependencies) do
|
||||
-- We need to get the jobId for this job.
|
||||
local childJobId = getJobIdFromKey(childJobKey)
|
||||
local childJobPrefix = getJobKeyPrefix(childJobKey, childJobId)
|
||||
local result = isLocked( childJobPrefix, childJobId, removeChildren )
|
||||
if result then
|
||||
return true
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
return false
|
||||
end
|
||||
return true
|
||||
end
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
--[[
|
||||
Function to move job from prioritized state to active.
|
||||
]]
|
||||
|
||||
local function moveJobFromPriorityToActive(priorityKey, activeKey, priorityCounterKey)
|
||||
local prioritizedJob = rcall("ZPOPMIN", priorityKey)
|
||||
if #prioritizedJob > 0 then
|
||||
rcall("LPUSH", activeKey, prioritizedJob[1])
|
||||
return prioritizedJob[1]
|
||||
else
|
||||
rcall("DEL", priorityCounterKey)
|
||||
end
|
||||
end
|
||||
Generated
Vendored
+38
@@ -0,0 +1,38 @@
|
||||
--[[
|
||||
Function to recursively move from waitingChildren to failed.
|
||||
]]
|
||||
|
||||
-- Includes
|
||||
--- @include "moveParentToWaitIfNeeded"
|
||||
|
||||
local function moveParentFromWaitingChildrenToFailed( parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
|
||||
if rcall("ZREM", parentQueueKey .. ":waiting-children", parentId) == 1 then
|
||||
rcall("ZADD", parentQueueKey .. ":failed", timestamp, parentId)
|
||||
local failedReason = "child " .. jobIdKey .. " failed"
|
||||
rcall("HMSET", parentKey, "failedReason", failedReason, "finishedOn", timestamp)
|
||||
rcall("XADD", parentQueueKey .. ":events", "*", "event", "failed", "jobId", parentId, "failedReason",
|
||||
failedReason, "prev", "waiting-children")
|
||||
|
||||
local rawParentData = rcall("HGET", parentKey, "parent")
|
||||
|
||||
if rawParentData ~= false then
|
||||
local parentData = cjson.decode(rawParentData)
|
||||
if parentData['fpof'] then
|
||||
moveParentFromWaitingChildrenToFailed(
|
||||
parentData['queueKey'],
|
||||
parentData['queueKey'] .. ':' .. parentData['id'],
|
||||
parentData['id'],
|
||||
parentKey,
|
||||
timestamp
|
||||
)
|
||||
elseif parentData['rdof'] then
|
||||
local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
||||
local grandParentDependenciesSet = grandParentKey .. ":dependencies"
|
||||
if rcall("SREM", grandParentDependenciesSet, parentKey) == 1 then
|
||||
moveParentToWaitIfNeeded(parentData['queueKey'], grandParentDependenciesSet,
|
||||
grandParentKey, parentData['id'], timestamp)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
--[[
|
||||
Validate and move parent to active if needed.
|
||||
]]
|
||||
|
||||
-- Includes
|
||||
--- @include "addDelayMarkerIfNeeded"
|
||||
--- @include "addJobWithPriority"
|
||||
--- @include "getTargetQueueList"
|
||||
|
||||
local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
|
||||
local isParentActive = rcall("ZSCORE", parentQueueKey .. ":waiting-children", parentId)
|
||||
if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then
|
||||
rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
|
||||
local parentWaitKey = parentQueueKey .. ":wait"
|
||||
local parentTarget, paused = getTargetQueueList(parentQueueKey .. ":meta", parentWaitKey,
|
||||
parentQueueKey .. ":paused")
|
||||
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
||||
local priority = tonumber(jobAttributes[1]) or 0
|
||||
local delay = tonumber(jobAttributes[2]) or 0
|
||||
|
||||
if delay > 0 then
|
||||
local delayedTimestamp = tonumber(timestamp) + delay
|
||||
local score = delayedTimestamp * 0x1000
|
||||
local parentDelayedKey = parentQueueKey .. ":delayed"
|
||||
rcall("ZADD", parentDelayedKey, score, parentId)
|
||||
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId,
|
||||
"delay", delayedTimestamp)
|
||||
|
||||
addDelayMarkerIfNeeded(parentTarget, parentDelayedKey)
|
||||
else
|
||||
if priority == 0 then
|
||||
rcall("RPUSH", parentTarget, parentId)
|
||||
else
|
||||
addJobWithPriority(parentWaitKey, parentQueueKey .. ":prioritized", priority, paused,
|
||||
parentId, parentQueueKey .. ":pc")
|
||||
end
|
||||
|
||||
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId,
|
||||
"prev", "waiting-children")
|
||||
end
|
||||
end
|
||||
end
|
||||
+73
@@ -0,0 +1,73 @@
|
||||
|
||||
--[[
|
||||
Function to move job from wait state to active.
|
||||
Input:
|
||||
keys[1] wait key
|
||||
keys[2] active key
|
||||
keys[3] prioritized key
|
||||
keys[4] stream events key
|
||||
keys[5] stalled key
|
||||
|
||||
-- Rate limiting
|
||||
keys[6] rate limiter key
|
||||
keys[7] delayed key
|
||||
|
||||
keys[8] paused key
|
||||
keys[9] meta key
|
||||
keys[10] pc priority counter
|
||||
|
||||
opts - token - lock token
|
||||
opts - lockDuration
|
||||
opts - limiter
|
||||
]]
|
||||
|
||||
-- Includes
|
||||
--- @include "pushBackJobWithPriority"
|
||||
|
||||
local function prepareJobForProcessing(keys, keyPrefix, targetKey, jobId, processedOn,
|
||||
maxJobs, expireTime, opts)
|
||||
local jobKey = keyPrefix .. jobId
|
||||
|
||||
-- Check if we need to perform rate limiting.
|
||||
if maxJobs then
|
||||
local rateLimiterKey = keys[6];
|
||||
|
||||
-- check if we exceeded rate limit, we need to remove the job and return expireTime
|
||||
if expireTime > 0 then
|
||||
-- remove from active queue and add back to the wait list
|
||||
rcall("LREM", keys[2], 1, jobId)
|
||||
|
||||
local priority = tonumber(rcall("HGET", jobKey, "priority")) or 0
|
||||
|
||||
if priority == 0 then
|
||||
rcall("RPUSH", targetKey, jobId)
|
||||
else
|
||||
pushBackJobWithPriority(keys[3], priority, jobId)
|
||||
end
|
||||
|
||||
-- Return when we can process more jobs
|
||||
return {0, 0, expireTime, 0}
|
||||
end
|
||||
|
||||
local jobCounter = tonumber(rcall("INCR", rateLimiterKey))
|
||||
|
||||
if jobCounter == 1 then
|
||||
local limiterDuration = opts['limiter'] and opts['limiter']['duration']
|
||||
local integerDuration = math.floor(math.abs(limiterDuration))
|
||||
rcall("PEXPIRE", rateLimiterKey, integerDuration)
|
||||
end
|
||||
end
|
||||
|
||||
local lockKey = jobKey .. ':lock'
|
||||
|
||||
-- get a lock
|
||||
if opts['token'] ~= "0" then
|
||||
rcall("SET", lockKey, opts['token'], "PX", opts['lockDuration'])
|
||||
end
|
||||
|
||||
rcall("XADD", keys[4], "*", "event", "active", "jobId", jobId, "prev", "waiting")
|
||||
rcall("HSET", jobKey, "processedOn", processedOn)
|
||||
rcall("HINCRBY", jobKey, "attemptsMade", 1)
|
||||
|
||||
return {rcall("HGETALL", jobKey), jobId, 0, 0} -- get job data
|
||||
end
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
--[[
|
||||
Updates the delay set, by moving delayed jobs that should
|
||||
be processed now to "wait".
|
||||
|
||||
Events:
|
||||
'waiting'
|
||||
]]
|
||||
|
||||
-- Includes
|
||||
--- @include "addPriorityMarkerIfNeeded"
|
||||
--- @include "getPriorityScore"
|
||||
|
||||
-- Try to get as much as 1000 jobs at once
|
||||
local function promoteDelayedJobs(delayedKey, waitKey, targetKey, prioritizedKey,
|
||||
eventStreamKey, prefix, timestamp, paused, priorityCounterKey)
|
||||
local jobs = rcall("ZRANGEBYSCORE", delayedKey, 0, (timestamp + 1) * 0x1000, "LIMIT", 0, 1000)
|
||||
|
||||
if (#jobs > 0) then
|
||||
rcall("ZREM", delayedKey, unpack(jobs))
|
||||
|
||||
for _, jobId in ipairs(jobs) do
|
||||
local jobKey = prefix .. jobId
|
||||
local priority =
|
||||
tonumber(rcall("HGET", jobKey, "priority")) or 0
|
||||
|
||||
if priority == 0 then
|
||||
-- LIFO or FIFO
|
||||
rcall("LPUSH", targetKey, jobId)
|
||||
else
|
||||
local score = getPriorityScore(priority, priorityCounterKey)
|
||||
rcall("ZADD", prioritizedKey, score, jobId)
|
||||
end
|
||||
|
||||
-- Emit waiting event
|
||||
rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId",
|
||||
jobId, "prev", "delayed")
|
||||
rcall("HSET", jobKey, "delay", 0)
|
||||
end
|
||||
|
||||
if not paused then
|
||||
addPriorityMarkerIfNeeded(targetKey)
|
||||
end
|
||||
end
|
||||
end
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
--[[
|
||||
Function to push back job considering priority in front of same prioritized jobs.
|
||||
]]
|
||||
local function pushBackJobWithPriority(prioritizedKey, priority, jobId)
|
||||
-- in order to put it at front of same prioritized jobs
|
||||
-- we consider prioritized counter as 0
|
||||
local score = priority * 0x100000000
|
||||
rcall("ZADD", prioritizedKey, score, jobId)
|
||||
end
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
--[[
|
||||
Function to remove job.
|
||||
]]
|
||||
|
||||
-- Includes
|
||||
--- @include "removeParentDependencyKey"
|
||||
|
||||
local function removeJob(jobId, hard, baseKey)
|
||||
local jobKey = baseKey .. jobId
|
||||
removeParentDependencyKey(jobKey, hard, nil, baseKey)
|
||||
rcall("DEL", jobKey, jobKey .. ':logs',
|
||||
jobKey .. ':dependencies', jobKey .. ':processed')
|
||||
end
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
--[[
|
||||
Function to remove from any state.
|
||||
|
||||
returns:
|
||||
prev state
|
||||
]]
|
||||
|
||||
local function removeJobFromAnyState( prefix, jobId)
|
||||
-- We start with the ZSCORE checks, since they have O(1) complexity
|
||||
if rcall("ZSCORE", prefix .. "completed", jobId) then
|
||||
rcall("ZREM", prefix .. "completed", jobId)
|
||||
return "completed"
|
||||
elseif rcall("ZSCORE", prefix .. "waiting-children", jobId) then
|
||||
rcall("ZREM", prefix .. "waiting-children", jobId)
|
||||
return "waiting-children"
|
||||
elseif rcall("ZSCORE", prefix .. "delayed", jobId) then
|
||||
rcall("ZREM", prefix .. "delayed", jobId)
|
||||
return "delayed"
|
||||
elseif rcall("ZSCORE", prefix .. "failed", jobId) then
|
||||
rcall("ZREM", prefix .. "failed", jobId)
|
||||
return "failed"
|
||||
elseif rcall("ZSCORE", prefix .. "prioritized", jobId) then
|
||||
rcall("ZREM", prefix .. "prioritized", jobId)
|
||||
return "prioritized"
|
||||
-- We remove only 1 element from the list, since we assume they are not added multiple times
|
||||
elseif rcall("LREM", prefix .. "wait", 1, jobId) == 1 then
|
||||
return "wait"
|
||||
elseif rcall("LREM", prefix .. "paused", 1, jobId) == 1 then
|
||||
return "paused"
|
||||
elseif rcall("LREM", prefix .. "active", 1, jobId) == 1 then
|
||||
return "active"
|
||||
end
|
||||
|
||||
return "unknown"
|
||||
end
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
--[[
|
||||
Functions to remove jobs.
|
||||
]]
|
||||
|
||||
-- Includes
|
||||
--- @include "removeJob"
|
||||
|
||||
local function removeJobs(keys, hard, baseKey, max)
|
||||
for i, key in ipairs(keys) do
|
||||
removeJob(key, hard, baseKey)
|
||||
end
|
||||
return max - #keys
|
||||
end
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
--[[
|
||||
Functions to remove jobs by max age.
|
||||
]]
|
||||
|
||||
-- Includes
|
||||
--- @include "removeJob"
|
||||
|
||||
local function removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
|
||||
local start = timestamp - maxAge * 1000
|
||||
local jobIds = rcall("ZREVRANGEBYSCORE", targetSet, start, "-inf")
|
||||
for i, jobId in ipairs(jobIds) do
|
||||
removeJob(jobId, false, prefix)
|
||||
end
|
||||
rcall("ZREMRANGEBYSCORE", targetSet, "-inf", start)
|
||||
end
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
--[[
|
||||
Functions to remove jobs by max count.
|
||||
]]
|
||||
|
||||
-- Includes
|
||||
--- @include "removeJob"
|
||||
|
||||
local function removeJobsByMaxCount(maxCount, targetSet, prefix)
|
||||
local start = maxCount
|
||||
local jobIds = rcall("ZREVRANGE", targetSet, start, -1)
|
||||
for i, jobId in ipairs(jobIds) do
|
||||
removeJob(jobId, false, prefix)
|
||||
end
|
||||
rcall("ZREMRANGEBYRANK", targetSet, 0, -(maxCount + 1))
|
||||
end
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
--[[
|
||||
Functions to remove jobs.
|
||||
]]
|
||||
|
||||
-- Includes
|
||||
--- @include "removeJobs"
|
||||
|
||||
local function getListItems(keyName, max)
|
||||
return rcall('LRANGE', keyName, 0, max - 1)
|
||||
end
|
||||
|
||||
local function removeListJobs(keyName, hard, baseKey, max)
|
||||
local jobs = getListItems(keyName, max)
|
||||
local count = removeJobs(jobs, hard, baseKey, max)
|
||||
rcall("LTRIM", keyName, #jobs, -1)
|
||||
return count
|
||||
end
|
||||
+77
@@ -0,0 +1,77 @@
|
||||
--[[
|
||||
Check if this job has a parent. If so we will just remove it from
|
||||
the parent child list, but if it is the last child we should move the parent to "wait/paused"
|
||||
which requires code from "moveToFinished"
|
||||
]]
|
||||
|
||||
--- @include "destructureJobKey"
|
||||
--- @include "getTargetQueueList"
|
||||
|
||||
local function moveParentToWait(parentPrefix, parentId, emitEvent)
|
||||
local parentTarget = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "wait", parentPrefix .. "paused")
|
||||
rcall("RPUSH", parentTarget, parentId)
|
||||
|
||||
if emitEvent then
|
||||
local parentEventStream = parentPrefix .. "events"
|
||||
rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
|
||||
end
|
||||
end
|
||||
|
||||
local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey)
|
||||
if parentKey then
|
||||
local parentDependenciesKey = parentKey .. ":dependencies"
|
||||
local result = rcall("SREM", parentDependenciesKey, jobKey)
|
||||
if result > 0 then
|
||||
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
|
||||
if pendingDependencies == 0 then
|
||||
local parentId = getJobIdFromKey(parentKey)
|
||||
local parentPrefix = getJobKeyPrefix(parentKey, parentId)
|
||||
|
||||
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
|
||||
|
||||
if numRemovedElements == 1 then
|
||||
if hard then
|
||||
if parentPrefix == baseKey then
|
||||
removeParentDependencyKey(parentKey, hard, nil, baseKey)
|
||||
rcall("DEL", parentKey, parentKey .. ':logs',
|
||||
parentKey .. ':dependencies', parentKey .. ':processed')
|
||||
else
|
||||
moveParentToWait(parentPrefix, parentId)
|
||||
end
|
||||
else
|
||||
moveParentToWait(parentPrefix, parentId, true)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
else
|
||||
local missedParentKey = rcall("HGET", jobKey, "parentKey")
|
||||
if( (type(missedParentKey) == "string") and missedParentKey ~= "" and (rcall("EXISTS", missedParentKey) == 1)) then
|
||||
local parentDependenciesKey = missedParentKey .. ":dependencies"
|
||||
local result = rcall("SREM", parentDependenciesKey, jobKey)
|
||||
if result > 0 then
|
||||
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
|
||||
if pendingDependencies == 0 then
|
||||
local parentId = getJobIdFromKey(missedParentKey)
|
||||
local parentPrefix = getJobKeyPrefix(missedParentKey, parentId)
|
||||
|
||||
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
|
||||
|
||||
if numRemovedElements == 1 then
|
||||
if hard then
|
||||
if parentPrefix == baseKey then
|
||||
removeParentDependencyKey(missedParentKey, hard, nil, baseKey)
|
||||
rcall("DEL", missedParentKey, missedParentKey .. ':logs',
|
||||
missedParentKey .. ':dependencies', missedParentKey .. ':processed')
|
||||
else
|
||||
moveParentToWait(parentPrefix, parentId)
|
||||
end
|
||||
else
|
||||
moveParentToWait(parentPrefix, parentId, true)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
-- Includes
|
||||
--- @include "batches"
|
||||
--- @include "getZSetItems"
|
||||
--- @include "removeJobs"
|
||||
|
||||
local function removeZSetJobs(keyName, hard, baseKey, max)
|
||||
local jobs = getZSetItems(keyName, max)
|
||||
local count = removeJobs(jobs, hard, baseKey, max)
|
||||
if(#jobs > 0) then
|
||||
for from, to in batches(#jobs, 7000) do
|
||||
rcall("ZREM", keyName, unpack(jobs, from, to))
|
||||
end
|
||||
end
|
||||
return count
|
||||
end
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
--[[
|
||||
Function to store a job
|
||||
]]
|
||||
local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
|
||||
parentKey, parentData, repeatJobKey)
|
||||
local jsonOpts = cjson.encode(opts)
|
||||
local delay = opts['delay'] or 0
|
||||
local priority = opts['priority'] or 0
|
||||
|
||||
local optionalValues = {}
|
||||
if parentKey ~= nil then
|
||||
table.insert(optionalValues, "parentKey")
|
||||
table.insert(optionalValues, parentKey)
|
||||
table.insert(optionalValues, "parent")
|
||||
table.insert(optionalValues, parentData)
|
||||
end
|
||||
|
||||
if repeatJobKey ~= nil then
|
||||
table.insert(optionalValues, "rjk")
|
||||
table.insert(optionalValues, repeatJobKey)
|
||||
end
|
||||
|
||||
rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
|
||||
"timestamp", timestamp, "delay", delay, "priority", priority,
|
||||
unpack(optionalValues))
|
||||
|
||||
rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
|
||||
|
||||
return delay, priority
|
||||
end
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
--[[
|
||||
Function to trim events, default 10000.
|
||||
]]
|
||||
|
||||
local function trimEvents(metaKey, eventStreamKey)
|
||||
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
||||
if maxEvents ~= false then
|
||||
rcall("XTRIM", eventStreamKey, "MAXLEN", "~", maxEvents)
|
||||
else
|
||||
rcall("XTRIM", eventStreamKey, "MAXLEN", "~", 10000)
|
||||
end
|
||||
end
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
--- @include "updateParentDepsIfNeeded"
|
||||
|
||||
--[[
|
||||
This function is used to update the parent's dependencies if the job
|
||||
is already completed and about to be ignored. The parent must get its
|
||||
dependencies updated to avoid the parent job being stuck forever in
|
||||
the waiting-children state.
|
||||
]]
|
||||
local function updateExistingJobsParent(parentKey, parent, parentData,
|
||||
parentDependenciesKey, completedKey,
|
||||
jobIdKey, jobId, timestamp)
|
||||
if parentKey ~= nil then
|
||||
if rcall("ZSCORE", completedKey, jobId) ~= false then
|
||||
local returnvalue = rcall("HGET", jobIdKey, "returnvalue")
|
||||
updateParentDepsIfNeeded(parentKey, parent['queueKey'],
|
||||
parentDependenciesKey, parent['id'],
|
||||
jobIdKey, returnvalue, timestamp)
|
||||
else
|
||||
if parentDependenciesKey ~= nil then
|
||||
rcall("SADD", parentDependenciesKey, jobIdKey)
|
||||
end
|
||||
end
|
||||
rcall("HMSET", jobIdKey, "parentKey", parentKey, "parent", parentData)
|
||||
end
|
||||
end
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
--[[
|
||||
Validate and move or add dependencies to parent.
|
||||
]]
|
||||
|
||||
-- Includes
|
||||
--- @include "moveParentToWaitIfNeeded"
|
||||
|
||||
local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
|
||||
parentId, jobIdKey, returnvalue, timestamp )
|
||||
local processedSet = parentKey .. ":processed"
|
||||
rcall("HSET", processedSet, jobIdKey, returnvalue)
|
||||
moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
|
||||
end
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.scriptLoader = exports.ScriptLoader = exports.ScriptLoaderError = void 0;
|
||||
const script_loader_1 = require("./script-loader");
|
||||
Object.defineProperty(exports, "ScriptLoader", { enumerable: true, get: function () { return script_loader_1.ScriptLoader; } });
|
||||
var script_loader_2 = require("./script-loader");
|
||||
Object.defineProperty(exports, "ScriptLoaderError", { enumerable: true, get: function () { return script_loader_2.ScriptLoaderError; } });
|
||||
const scriptLoader = new script_loader_1.ScriptLoader();
|
||||
exports.scriptLoader = scriptLoader;
|
||||
//# sourceMappingURL=index.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/commands/index.ts"],"names":[],"mappings":";;;AAAA,mDAA+C;AAKtC,6FALA,4BAAY,OAKA;AAJrB,iDAA6E;AAA3C,kHAAA,iBAAiB,OAAA;AAEnD,MAAM,YAAY,GAAG,IAAI,4BAAY,EAAE,CAAC;AAEjB,oCAAY"}
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
--[[
|
||||
Checks if a job is finished (.i.e. is in the completed or failed set)
|
||||
|
||||
Input:
|
||||
KEYS[1] completed key
|
||||
KEYS[2] failed key
|
||||
KEYS[3] job key
|
||||
|
||||
ARGV[1] job id
|
||||
ARGV[2] return value?
|
||||
Output:
|
||||
0 - Not finished.
|
||||
1 - Completed.
|
||||
2 - Failed.
|
||||
-1 - Missing job.
|
||||
]]
|
||||
local rcall = redis.call
|
||||
if rcall("EXISTS", KEYS[3]) ~= 1 then
|
||||
if ARGV[2] == "1" then
|
||||
|
||||
return {-1,"Missing key for job " .. KEYS[3] .. ". isFinished"}
|
||||
end
|
||||
return -1
|
||||
end
|
||||
|
||||
if rcall("ZSCORE", KEYS[1], ARGV[1]) ~= false then
|
||||
if ARGV[2] == "1" then
|
||||
local returnValue = rcall("HGET", KEYS[3], "returnvalue")
|
||||
|
||||
return {1,returnValue}
|
||||
end
|
||||
return 1
|
||||
end
|
||||
|
||||
if rcall("ZSCORE", KEYS[2], ARGV[1]) ~= false then
|
||||
if ARGV[2] == "1" then
|
||||
local failedReason = rcall("HGET", KEYS[3], "failedReason")
|
||||
|
||||
return {2,failedReason}
|
||||
end
|
||||
return 2
|
||||
end
|
||||
|
||||
if ARGV[2] == "1" then
|
||||
return {0}
|
||||
end
|
||||
|
||||
return 0
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
--[[
|
||||
Checks if job is in a given list.
|
||||
|
||||
Input:
|
||||
KEYS[1]
|
||||
ARGV[1]
|
||||
|
||||
Output:
|
||||
1 if element found in the list.
|
||||
]]
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/checkItemInList"
|
||||
|
||||
local items = redis.call("LRANGE", KEYS[1] , 0, -1)
|
||||
return checkItemInList(items, ARGV[1])
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
--[[
|
||||
Function to move job from active state to wait.
|
||||
Input:
|
||||
KEYS[1] active key
|
||||
KEYS[2] wait key
|
||||
|
||||
KEYS[3] stalled key
|
||||
KEYS[4] job lock key
|
||||
KEYS[5] paused key
|
||||
KEYS[6] meta key
|
||||
KEYS[7] limiter key
|
||||
KEYS[8] prioritized key
|
||||
KEYS[9] event key
|
||||
|
||||
ARGV[1] job id
|
||||
ARGV[2] lock token
|
||||
ARGV[3] job id key
|
||||
]]
|
||||
local rcall = redis.call
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/pushBackJobWithPriority"
|
||||
--- @include "includes/getTargetQueueList"
|
||||
|
||||
local jobId = ARGV[1]
|
||||
local token = ARGV[2]
|
||||
local lockKey = KEYS[4]
|
||||
|
||||
local lockToken = rcall("GET", lockKey)
|
||||
local pttl = rcall("PTTL", KEYS[7])
|
||||
if lockToken == token and pttl > 0 then
|
||||
local removed = rcall("LREM", KEYS[1], 1, jobId)
|
||||
if (removed > 0) then
|
||||
local target = getTargetQueueList(KEYS[6], KEYS[2], KEYS[5])
|
||||
|
||||
rcall("SREM", KEYS[3], jobId)
|
||||
|
||||
local priority = tonumber(rcall("HGET", ARGV[3], "priority")) or 0
|
||||
|
||||
if priority > 0 then
|
||||
pushBackJobWithPriority(KEYS[8], priority, jobId)
|
||||
else
|
||||
rcall("RPUSH", target, jobId)
|
||||
end
|
||||
|
||||
rcall("DEL", lockKey)
|
||||
|
||||
-- Emit waiting event
|
||||
rcall("XADD", KEYS[9], "*", "event", "waiting", "jobId", jobId)
|
||||
end
|
||||
end
|
||||
|
||||
return pttl
|
||||
+67
@@ -0,0 +1,67 @@
|
||||
--[[
|
||||
Move completed, failed or delayed jobs to wait.
|
||||
|
||||
Note: Does not support jobs with priorities.
|
||||
|
||||
Input:
|
||||
KEYS[1] base key
|
||||
KEYS[2] events stream
|
||||
KEYS[3] state key (failed, completed, delayed)
|
||||
KEYS[4] 'wait'
|
||||
KEYS[5] 'paused'
|
||||
KEYS[6] 'meta'
|
||||
|
||||
ARGV[1] count
|
||||
ARGV[2] timestamp
|
||||
ARGV[3] prev state
|
||||
|
||||
Output:
|
||||
1 means the operation is not completed
|
||||
0 means the operation is completed
|
||||
]]
|
||||
local maxCount = tonumber(ARGV[1])
|
||||
local timestamp = tonumber(ARGV[2])
|
||||
|
||||
local rcall = redis.call;
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/batches"
|
||||
--- @include "includes/getTargetQueueList"
|
||||
|
||||
local metaKey = KEYS[6]
|
||||
local target = getTargetQueueList(metaKey, KEYS[4], KEYS[5])
|
||||
|
||||
local jobs = rcall('ZRANGEBYSCORE', KEYS[3], 0, timestamp, 'LIMIT', 0, maxCount)
|
||||
if (#jobs > 0) then
|
||||
|
||||
if ARGV[3] == "failed" then
|
||||
for i, key in ipairs(jobs) do
|
||||
local jobKey = KEYS[1] .. key
|
||||
rcall("HDEL", jobKey, "finishedOn", "processedOn", "failedReason")
|
||||
end
|
||||
elseif ARGV[3] == "completed" then
|
||||
for i, key in ipairs(jobs) do
|
||||
local jobKey = KEYS[1] .. key
|
||||
rcall("HDEL", jobKey, "finishedOn", "processedOn", "returnvalue")
|
||||
end
|
||||
end
|
||||
|
||||
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents") or 10000
|
||||
|
||||
for i, key in ipairs(jobs) do
|
||||
-- Emit waiting event
|
||||
rcall("XADD", KEYS[2], "MAXLEN", "~", maxEvents, "*", "event",
|
||||
"waiting", "jobId", key, "prev", ARGV[3]);
|
||||
end
|
||||
|
||||
for from, to in batches(#jobs, 7000) do
|
||||
rcall("ZREM", KEYS[3], unpack(jobs, from, to))
|
||||
rcall("LPUSH", target, unpack(jobs, from, to))
|
||||
end
|
||||
end
|
||||
|
||||
maxCount = maxCount - #jobs
|
||||
|
||||
if (maxCount <= 0) then return 1 end
|
||||
|
||||
return 0
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
--[[
|
||||
Move stalled jobs to wait.
|
||||
|
||||
Input:
|
||||
KEYS[1] 'stalled' (SET)
|
||||
KEYS[2] 'wait', (LIST)
|
||||
KEYS[3] 'active', (LIST)
|
||||
KEYS[4] 'failed', (ZSET)
|
||||
KEYS[5] 'stalled-check', (KEY)
|
||||
KEYS[6] 'meta', (KEY)
|
||||
KEYS[7] 'paused', (LIST)
|
||||
KEYS[8] 'event stream' (STREAM)
|
||||
|
||||
ARGV[1] Max stalled job count
|
||||
ARGV[2] queue.toKey('')
|
||||
ARGV[3] timestamp
|
||||
ARGV[4] max check time
|
||||
|
||||
Events:
|
||||
'stalled' with stalled job id.
|
||||
]] -- Includes
|
||||
--- @include "includes/checkStalledJobs"
|
||||
return checkStalledJobs(KEYS[1], KEYS[2], KEYS[3], KEYS[4], KEYS[5], KEYS[6],
|
||||
KEYS[7], KEYS[8], ARGV[1], ARGV[2], ARGV[3], ARGV[4])
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
--[[
|
||||
Move next job to be processed to active, lock it and fetch its data. The job
|
||||
may be delayed, in that case we need to move it to the delayed set instead.
|
||||
|
||||
This operation guarantees that the worker owns the job during the lock
|
||||
expiration time. The worker is responsible of keeping the lock fresh
|
||||
so that no other worker picks this job again.
|
||||
|
||||
Input:
|
||||
KEYS[1] wait key
|
||||
KEYS[2] active key
|
||||
KEYS[3] prioritized key
|
||||
KEYS[4] stream events key
|
||||
KEYS[5] stalled key
|
||||
|
||||
-- Rate limiting
|
||||
KEYS[6] rate limiter key
|
||||
KEYS[7] delayed key
|
||||
|
||||
-- Promote delayed jobs
|
||||
KEYS[8] paused key
|
||||
KEYS[9] meta key
|
||||
KEYS[10] pc priority counter
|
||||
|
||||
-- Arguments
|
||||
ARGV[1] key prefix
|
||||
ARGV[2] timestamp
|
||||
ARGV[3] optional job ID
|
||||
ARGV[4] opts
|
||||
|
||||
opts - token - lock token
|
||||
opts - lockDuration
|
||||
opts - limiter
|
||||
]]
|
||||
local rcall = redis.call
|
||||
local waitKey = KEYS[1]
|
||||
local activeKey = KEYS[2]
|
||||
local rateLimiterKey = KEYS[6]
|
||||
local delayedKey = KEYS[7]
|
||||
local opts = cmsgpack.unpack(ARGV[4])
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/getNextDelayedTimestamp"
|
||||
--- @include "includes/getRateLimitTTL"
|
||||
--- @include "includes/getTargetQueueList"
|
||||
--- @include "includes/moveJobFromPriorityToActive"
|
||||
--- @include "includes/prepareJobForProcessing"
|
||||
--- @include "includes/promoteDelayedJobs"
|
||||
|
||||
local target, paused = getTargetQueueList(KEYS[9], waitKey, KEYS[8])
|
||||
|
||||
-- Check if there are delayed jobs that we can move to wait.
|
||||
promoteDelayedJobs(delayedKey, waitKey, target, KEYS[3], KEYS[4], ARGV[1],
|
||||
ARGV[2], paused, KEYS[10])
|
||||
|
||||
local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
|
||||
local expireTime = getRateLimitTTL(maxJobs, rateLimiterKey)
|
||||
|
||||
local jobId = nil
|
||||
if ARGV[3] ~= "" then
|
||||
jobId = ARGV[3]
|
||||
|
||||
-- clean stalled key
|
||||
rcall("SREM", KEYS[5], jobId)
|
||||
end
|
||||
|
||||
if not jobId or (jobId and string.sub(jobId, 1, 2) == "0:") then
|
||||
-- If jobId is special ID 0:delay, then there is no job to process
|
||||
if jobId then rcall("LREM", activeKey, 1, jobId) end
|
||||
|
||||
-- Check if we are rate limited first.
|
||||
if expireTime > 0 then return {0, 0, expireTime, 0} end
|
||||
|
||||
-- paused queue
|
||||
if paused then return {0, 0, 0, 0} end
|
||||
|
||||
-- no job ID, try non-blocking move from wait to active
|
||||
jobId = rcall("RPOPLPUSH", waitKey, activeKey)
|
||||
|
||||
-- Since it is possible that between a call to BRPOPLPUSH and moveToActive
|
||||
-- another script puts a new maker in wait, we need to check again.
|
||||
if jobId and string.sub(jobId, 1, 2) == "0:" then
|
||||
rcall("LREM", activeKey, 1, jobId)
|
||||
jobId = rcall("RPOPLPUSH", waitKey, activeKey)
|
||||
end
|
||||
end
|
||||
|
||||
if jobId then
|
||||
return prepareJobForProcessing(KEYS, ARGV[1], target, jobId, ARGV[2],
|
||||
maxJobs, expireTime, opts)
|
||||
else
|
||||
jobId = moveJobFromPriorityToActive(KEYS[3], activeKey, KEYS[10])
|
||||
if jobId then
|
||||
return prepareJobForProcessing(KEYS, ARGV[1], target, jobId, ARGV[2],
|
||||
maxJobs, expireTime, opts)
|
||||
end
|
||||
end
|
||||
|
||||
-- Return the timestamp for the next delayed job if any.
|
||||
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
|
||||
if (nextTimestamp ~= nil) then return {0, 0, 0, nextTimestamp} end
|
||||
|
||||
return {0, 0, 0, 0}
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
--[[
|
||||
Moves job from active to delayed set.
|
||||
|
||||
Input:
|
||||
KEYS[1] wait key
|
||||
KEYS[2] active key
|
||||
KEYS[3] prioritized key
|
||||
KEYS[4] delayed key
|
||||
KEYS[5] job key
|
||||
KEYS[6] events stream
|
||||
KEYS[7] paused key
|
||||
KEYS[8] meta key
|
||||
|
||||
ARGV[1] key prefix
|
||||
ARGV[2] timestamp
|
||||
ARGV[3] delayedTimestamp
|
||||
ARGV[4] the id of the job
|
||||
ARGV[5] queue token
|
||||
ARGV[6] delay value
|
||||
|
||||
Output:
|
||||
0 - OK
|
||||
-1 - Missing job.
|
||||
-3 - Job not in active set.
|
||||
|
||||
Events:
|
||||
- delayed key.
|
||||
]]
|
||||
local rcall = redis.call
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/addDelayMarkerIfNeeded"
|
||||
--- @include "includes/getTargetQueueList"
|
||||
--- @include "includes/promoteDelayedJobs"
|
||||
|
||||
local jobKey = KEYS[5]
|
||||
if rcall("EXISTS", jobKey) == 1 then
|
||||
local delayedKey = KEYS[4]
|
||||
if ARGV[5] ~= "0" then
|
||||
local lockKey = jobKey .. ':lock'
|
||||
if rcall("GET", lockKey) == ARGV[5] then
|
||||
rcall("DEL", lockKey)
|
||||
else
|
||||
return -2
|
||||
end
|
||||
end
|
||||
|
||||
local jobId = ARGV[4]
|
||||
local score = tonumber(ARGV[3])
|
||||
local delayedTimestamp = (score / 0x1000)
|
||||
|
||||
local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
|
||||
if numRemovedElements < 1 then
|
||||
return -3
|
||||
end
|
||||
|
||||
rcall("HSET", jobKey, "delay", ARGV[6])
|
||||
|
||||
local maxEvents = rcall("HGET", KEYS[8], "opts.maxLenEvents") or 10000
|
||||
|
||||
rcall("ZADD", delayedKey, score, jobId)
|
||||
rcall("XADD", KEYS[6], "MAXLEN", "~", maxEvents, "*", "event", "delayed",
|
||||
"jobId", jobId, "delay", delayedTimestamp)
|
||||
|
||||
-- Check if we need to push a marker job to wake up sleeping workers.
|
||||
local target = getTargetQueueList(KEYS[8], KEYS[1], KEYS[7])
|
||||
addDelayMarkerIfNeeded(target, delayedKey)
|
||||
|
||||
return 0
|
||||
else
|
||||
return -1
|
||||
end
|
||||
+267
@@ -0,0 +1,267 @@
|
||||
--[[
|
||||
Move job from active to a finished status (completed o failed)
|
||||
A job can only be moved to completed if it was active.
|
||||
The job must be locked before it can be moved to a finished status,
|
||||
and the lock must be released in this script.
|
||||
|
||||
Input:
|
||||
KEYS[1] wait key
|
||||
KEYS[2] active key
|
||||
KEYS[3] prioritized key
|
||||
KEYS[4] event stream key
|
||||
KEYS[5] stalled key
|
||||
|
||||
-- Rate limiting
|
||||
KEYS[6] rate limiter key
|
||||
KEYS[7] delayed key
|
||||
|
||||
KEYS[8] paused key
|
||||
KEYS[9] meta key
|
||||
KEYS[10] pc priority counter
|
||||
|
||||
KEYS[11] completed/failed key
|
||||
KEYS[12] jobId key
|
||||
KEYS[13] metrics key
|
||||
|
||||
ARGV[1] jobId
|
||||
ARGV[2] timestamp
|
||||
ARGV[3] msg property returnvalue / failedReason
|
||||
ARGV[4] return value / failed reason
|
||||
ARGV[5] target (completed/failed)
|
||||
ARGV[6] event data (? maybe just send jobid).
|
||||
ARGV[7] fetch next?
|
||||
ARGV[8] keys prefix
|
||||
ARGV[9] opts
|
||||
|
||||
opts - token - lock token
|
||||
opts - keepJobs
|
||||
opts - lockDuration - lock duration in milliseconds
|
||||
opts - attempts max attempts
|
||||
opts - attemptsMade
|
||||
opts - maxMetricsSize
|
||||
opts - fpof - fail parent on fail
|
||||
opts - rdof - remove dependency on fail
|
||||
|
||||
Output:
|
||||
0 OK
|
||||
-1 Missing key.
|
||||
-2 Missing lock.
|
||||
-3 Job not in active set
|
||||
-4 Job has pending dependencies
|
||||
-6 Lock is not owned by this client
|
||||
|
||||
Events:
|
||||
'completed/failed'
|
||||
]]
|
||||
local rcall = redis.call
|
||||
|
||||
--- Includes
|
||||
--- @include "includes/collectMetrics"
|
||||
--- @include "includes/getNextDelayedTimestamp"
|
||||
--- @include "includes/moveJobFromPriorityToActive"
|
||||
--- @include "includes/prepareJobForProcessing"
|
||||
--- @include "includes/moveParentFromWaitingChildrenToFailed"
|
||||
--- @include "includes/moveParentToWaitIfNeeded"
|
||||
--- @include "includes/promoteDelayedJobs"
|
||||
--- @include "includes/removeJobsByMaxAge"
|
||||
--- @include "includes/removeJobsByMaxCount"
|
||||
--- @include "includes/removeParentDependencyKey"
|
||||
--- @include "includes/trimEvents"
|
||||
--- @include "includes/updateParentDepsIfNeeded"
|
||||
--- @include "includes/getRateLimitTTL"
|
||||
|
||||
local jobIdKey = KEYS[12]
|
||||
if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
||||
local opts = cmsgpack.unpack(ARGV[9])
|
||||
|
||||
local token = opts['token']
|
||||
local attempts = opts['attempts']
|
||||
local attemptsMade = opts['attemptsMade']
|
||||
local maxMetricsSize = opts['maxMetricsSize']
|
||||
local maxCount = opts['keepJobs']['count']
|
||||
local maxAge = opts['keepJobs']['age']
|
||||
|
||||
if token ~= "0" then
|
||||
local lockKey = jobIdKey .. ':lock'
|
||||
local lockToken = rcall("GET", lockKey)
|
||||
if lockToken == token then
|
||||
rcall("DEL", lockKey)
|
||||
rcall("SREM", KEYS[5], ARGV[1])
|
||||
else
|
||||
if lockToken then
|
||||
-- Lock exists but token does not match
|
||||
return -6
|
||||
else
|
||||
-- Lock is missing completely
|
||||
return -2
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
if rcall("SCARD", jobIdKey .. ":dependencies") ~= 0 then -- // Make sure it does not have pending dependencies
|
||||
return -4
|
||||
end
|
||||
|
||||
local parentReferences = rcall("HMGET", jobIdKey, "parentKey", "parent")
|
||||
local parentKey = parentReferences[1] or ""
|
||||
local parentId = ""
|
||||
local parentQueueKey = ""
|
||||
if parentReferences[2] ~= false then
|
||||
local jsonDecodedParent = cjson.decode(parentReferences[2])
|
||||
parentId = jsonDecodedParent['id']
|
||||
parentQueueKey = jsonDecodedParent['queueKey']
|
||||
end
|
||||
|
||||
local jobId = ARGV[1]
|
||||
local timestamp = ARGV[2]
|
||||
|
||||
-- Remove from active list (if not active we shall return error)
|
||||
local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
|
||||
|
||||
if (numRemovedElements < 1) then return -3 end
|
||||
|
||||
-- Trim events before emiting them to avoid trimming events emitted in this script
|
||||
trimEvents(KEYS[9], KEYS[4])
|
||||
|
||||
-- If job has a parent we need to
|
||||
-- 1) remove this job id from parents dependencies
|
||||
-- 2) move the job Id to parent "processed" set
|
||||
-- 3) push the results into parent "results" list
|
||||
-- 4) if parent's dependencies is empty, then move parent to "wait/paused". Note it may be a different queue!.
|
||||
if parentId == "" and parentKey ~= "" then
|
||||
parentId = getJobIdFromKey(parentKey)
|
||||
parentQueueKey = getJobKeyPrefix(parentKey, ":" .. parentId)
|
||||
end
|
||||
|
||||
if parentId ~= "" then
|
||||
if ARGV[5] == "completed" then
|
||||
local dependenciesSet = parentKey .. ":dependencies"
|
||||
if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
|
||||
updateParentDepsIfNeeded(parentKey, parentQueueKey,
|
||||
dependenciesSet, parentId, jobIdKey,
|
||||
ARGV[4], timestamp)
|
||||
end
|
||||
else
|
||||
if opts['fpof'] then
|
||||
moveParentFromWaitingChildrenToFailed(parentQueueKey, parentKey,
|
||||
parentId, jobIdKey, timestamp)
|
||||
elseif opts['rdof'] then
|
||||
local dependenciesSet = parentKey .. ":dependencies"
|
||||
if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
|
||||
moveParentToWaitIfNeeded(parentQueueKey, dependenciesSet,
|
||||
parentKey, parentId, timestamp)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
-- Remove job?
|
||||
if maxCount ~= 0 then
|
||||
local targetSet = KEYS[11]
|
||||
-- Add to complete/failed set
|
||||
rcall("ZADD", targetSet, timestamp, jobId)
|
||||
rcall("HMSET", jobIdKey, ARGV[3], ARGV[4], "finishedOn", timestamp)
|
||||
-- "returnvalue" / "failedReason" and "finishedOn"
|
||||
|
||||
-- Remove old jobs?
|
||||
local prefix = ARGV[8]
|
||||
|
||||
if maxAge ~= nil then
|
||||
removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
|
||||
end
|
||||
|
||||
if maxCount ~= nil and maxCount > 0 then
|
||||
removeJobsByMaxCount(maxCount, targetSet, prefix)
|
||||
end
|
||||
else
|
||||
rcall("DEL", jobIdKey, jobIdKey .. ':logs', jobIdKey .. ':processed')
|
||||
if parentKey ~= "" then
|
||||
removeParentDependencyKey(jobIdKey, false, parentKey)
|
||||
end
|
||||
end
|
||||
|
||||
rcall("XADD", KEYS[4], "*", "event", ARGV[5], "jobId", jobId, ARGV[3],
|
||||
ARGV[4])
|
||||
|
||||
if ARGV[5] == "failed" then
|
||||
if tonumber(attemptsMade) >= tonumber(attempts) then
|
||||
rcall("XADD", KEYS[4], "*", "event", "retries-exhausted", "jobId",
|
||||
jobId, "attemptsMade", attemptsMade)
|
||||
end
|
||||
end
|
||||
|
||||
-- Collect metrics
|
||||
if maxMetricsSize ~= "" then
|
||||
collectMetrics(KEYS[13], KEYS[13] .. ':data', maxMetricsSize, timestamp)
|
||||
end
|
||||
|
||||
-- Try to get next job to avoid an extra roundtrip if the queue is not closing,
|
||||
-- and not rate limited.
|
||||
if (ARGV[7] == "1") then
|
||||
|
||||
local target, paused = getTargetQueueList(KEYS[9], KEYS[1], KEYS[8])
|
||||
|
||||
-- Check if there are delayed jobs that can be promoted
|
||||
promoteDelayedJobs(KEYS[7], KEYS[1], target, KEYS[3],
|
||||
KEYS[4], ARGV[8], timestamp, paused, KEYS[10])
|
||||
|
||||
local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
|
||||
-- Check if we are rate limited first.
|
||||
local expireTime = getRateLimitTTL(maxJobs, KEYS[6])
|
||||
|
||||
if expireTime > 0 then return {0, 0, expireTime, 0} end
|
||||
|
||||
-- paused queue
|
||||
if paused then return {0, 0, 0, 0} end
|
||||
|
||||
jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
|
||||
|
||||
if jobId then
|
||||
if string.sub(jobId, 1, 2) == "0:" then
|
||||
rcall("LREM", KEYS[2], 1, jobId)
|
||||
|
||||
-- If jobId is special ID 0:delay (delay greater than 0), then there is no job to process
|
||||
-- but if ID is 0:0, then there is at least 1 prioritized job to process
|
||||
if jobId == "0:0" then
|
||||
jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2], KEYS[10])
|
||||
return prepareJobForProcessing(KEYS, ARGV[8], target, jobId, timestamp,
|
||||
maxJobs, expireTime, opts)
|
||||
end
|
||||
else
|
||||
return prepareJobForProcessing(KEYS, ARGV[8], target, jobId, timestamp, maxJobs,
|
||||
expireTime, opts)
|
||||
end
|
||||
else
|
||||
jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2], KEYS[10])
|
||||
if jobId then
|
||||
return prepareJobForProcessing(KEYS, ARGV[8], target, jobId, timestamp, maxJobs,
|
||||
expireTime, opts)
|
||||
end
|
||||
end
|
||||
|
||||
-- Return the timestamp for the next delayed job if any.
|
||||
local nextTimestamp = getNextDelayedTimestamp(KEYS[7])
|
||||
if nextTimestamp ~= nil then
|
||||
-- The result is guaranteed to be positive, since the
|
||||
-- ZRANGEBYSCORE command would have return a job otherwise.
|
||||
return {0, 0, 0, nextTimestamp}
|
||||
end
|
||||
end
|
||||
|
||||
local waitLen = rcall("LLEN", KEYS[1])
|
||||
if waitLen == 0 then
|
||||
local activeLen = rcall("LLEN", KEYS[2])
|
||||
|
||||
if activeLen == 0 then
|
||||
local prioritizedLen = rcall("ZCARD", KEYS[3])
|
||||
|
||||
if prioritizedLen == 0 then
|
||||
rcall("XADD", KEYS[4], "*", "event", "drained")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
return 0
|
||||
else
|
||||
return -1
|
||||
end
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
--[[
|
||||
Moves job from active to waiting children set.
|
||||
|
||||
Input:
|
||||
KEYS[1] lock key
|
||||
KEYS[2] active key
|
||||
KEYS[3] waitChildrenKey key
|
||||
KEYS[4] job key
|
||||
|
||||
ARGV[1] token
|
||||
ARGV[2] child key
|
||||
ARGV[3] timestamp
|
||||
ARGV[4] the id of the job
|
||||
|
||||
Output:
|
||||
0 - OK
|
||||
1 - There are not pending dependencies.
|
||||
-1 - Missing job.
|
||||
-2 - Missing lock
|
||||
-3 - Job not in active set
|
||||
]]
|
||||
local rcall = redis.call
|
||||
|
||||
local function moveToWaitingChildren (activeKey, waitingChildrenKey, jobId, timestamp, lockKey, token)
|
||||
if token ~= "0" then
|
||||
if rcall("GET", lockKey) == token then
|
||||
rcall("DEL", lockKey)
|
||||
else
|
||||
return -2
|
||||
end
|
||||
end
|
||||
|
||||
local score = tonumber(timestamp)
|
||||
|
||||
local numRemovedElements = rcall("LREM", activeKey, -1, jobId)
|
||||
|
||||
if(numRemovedElements < 1) then
|
||||
return -3
|
||||
end
|
||||
|
||||
rcall("ZADD", waitingChildrenKey, score, jobId)
|
||||
|
||||
return 0
|
||||
end
|
||||
|
||||
if rcall("EXISTS", KEYS[4]) == 1 then
|
||||
if ARGV[2] ~= "" then
|
||||
if rcall("SISMEMBER", KEYS[4] .. ":dependencies", ARGV[2]) ~= 0 then
|
||||
return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1], ARGV[1])
|
||||
end
|
||||
|
||||
return 1
|
||||
else
|
||||
if rcall("SCARD", KEYS[4] .. ":dependencies") ~= 0 then
|
||||
return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1], ARGV[1])
|
||||
end
|
||||
|
||||
return 1
|
||||
end
|
||||
end
|
||||
|
||||
return -1
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
--[[
|
||||
Completely obliterates a queue and all of its contents
|
||||
|
||||
Input:
|
||||
KEYS[1] meta
|
||||
KEYS[2] base
|
||||
|
||||
ARGV[1] count
|
||||
ARGV[2] force
|
||||
]]
|
||||
|
||||
-- This command completely destroys a queue including all of its jobs, current or past
|
||||
-- leaving no trace of its existence. Since this script needs to iterate to find all the job
|
||||
-- keys, consider that this call may be slow for very large queues.
|
||||
|
||||
-- The queue needs to be "paused" or it will return an error
|
||||
-- If the queue has currently active jobs then the script by default will return error,
|
||||
-- however this behaviour can be overrided using the 'force' option.
|
||||
local maxCount = tonumber(ARGV[1])
|
||||
local baseKey = KEYS[2]
|
||||
|
||||
local rcall = redis.call
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/removeJobs"
|
||||
--- @include "includes/removeListJobs"
|
||||
--- @include "includes/removeZSetJobs"
|
||||
|
||||
local function removeLockKeys(keys)
|
||||
for i, key in ipairs(keys) do
|
||||
rcall("DEL", baseKey .. key .. ':lock')
|
||||
end
|
||||
end
|
||||
|
||||
-- 1) Check if paused, if not return with error.
|
||||
if rcall("HEXISTS", KEYS[1], "paused") ~= 1 then
|
||||
return -1 -- Error, NotPaused
|
||||
end
|
||||
|
||||
-- 2) Check if there are active jobs, if there are and not "force" return error.
|
||||
local activeKey = baseKey .. 'active'
|
||||
local activeJobs = getListItems(activeKey, maxCount)
|
||||
if (#activeJobs > 0) then
|
||||
if(ARGV[2] == "") then
|
||||
return -2 -- Error, ExistActiveJobs
|
||||
end
|
||||
end
|
||||
|
||||
removeLockKeys(activeJobs)
|
||||
maxCount = removeJobs(activeJobs, true, baseKey, maxCount)
|
||||
rcall("LTRIM", activeKey, #activeJobs, -1)
|
||||
if(maxCount <= 0) then
|
||||
return 1
|
||||
end
|
||||
|
||||
local delayedKey = baseKey .. 'delayed'
|
||||
maxCount = removeZSetJobs(delayedKey, true, baseKey, maxCount)
|
||||
if(maxCount <= 0) then
|
||||
return 1
|
||||
end
|
||||
|
||||
local completedKey = baseKey .. 'completed'
|
||||
maxCount = removeZSetJobs(completedKey, true, baseKey, maxCount)
|
||||
if(maxCount <= 0) then
|
||||
return 1
|
||||
end
|
||||
|
||||
local waitKey = baseKey .. 'paused'
|
||||
maxCount = removeListJobs(waitKey, true, baseKey, maxCount)
|
||||
if(maxCount <= 0) then
|
||||
return 1
|
||||
end
|
||||
|
||||
local prioritizedKey = baseKey .. 'prioritized'
|
||||
maxCount = removeZSetJobs(prioritizedKey, true, baseKey, maxCount)
|
||||
if(maxCount <= 0) then
|
||||
return 1
|
||||
end
|
||||
|
||||
local failedKey = baseKey .. 'failed'
|
||||
maxCount = removeZSetJobs(failedKey, true, baseKey, maxCount)
|
||||
if(maxCount <= 0) then
|
||||
return 1
|
||||
end
|
||||
|
||||
if(maxCount > 0) then
|
||||
rcall("DEL",
|
||||
baseKey .. 'events',
|
||||
baseKey .. 'delay',
|
||||
baseKey .. 'stalled-check',
|
||||
baseKey .. 'stalled',
|
||||
baseKey .. 'id',
|
||||
baseKey .. 'pc',
|
||||
baseKey .. 'meta',
|
||||
baseKey .. 'repeat',
|
||||
baseKey .. 'metrics:completed',
|
||||
baseKey .. 'metrics:completed:data',
|
||||
baseKey .. 'metrics:failed',
|
||||
baseKey .. 'metrics:failed:data')
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
end
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
--[[
|
||||
Paginate a set or hash
|
||||
|
||||
Input:
|
||||
KEYS[1] key pointing to the set or hash to be paginated.
|
||||
|
||||
ARGV[1] page start offset
|
||||
ARGV[2] page end offset (-1 for all the elements)
|
||||
ARGV[3] cursor
|
||||
ARGV[4] offset
|
||||
ARGV[5] max iterations
|
||||
ARGV[6] fetch jobs?
|
||||
|
||||
Output:
|
||||
[cursor, offset, items, numItems]
|
||||
]]
|
||||
local rcall = redis.call
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/findPage"
|
||||
|
||||
local key = KEYS[1]
|
||||
local scanCommand = "SSCAN"
|
||||
local countCommand = "SCARD"
|
||||
local type = rcall("TYPE", key)["ok"]
|
||||
|
||||
if type == "none" then
|
||||
return {0, 0, {}, 0}
|
||||
elseif type == "hash" then
|
||||
scanCommand = "HSCAN"
|
||||
countCommand = "HLEN"
|
||||
elseif type ~= "set" then
|
||||
return
|
||||
redis.error_reply("Pagination is only supported for sets and hashes.")
|
||||
end
|
||||
|
||||
local numItems = rcall(countCommand, key)
|
||||
local startOffset = tonumber(ARGV[1])
|
||||
local endOffset = tonumber(ARGV[2])
|
||||
if endOffset == -1 then
|
||||
endOffset = numItems
|
||||
end
|
||||
local pageSize = (endOffset - startOffset) + 1
|
||||
|
||||
local cursor, offset, items, jobs = findPage(key, scanCommand, startOffset,
|
||||
pageSize, ARGV[3], tonumber(ARGV[4]),
|
||||
tonumber(ARGV[5]), ARGV[6])
|
||||
|
||||
return {cursor, offset, items, numItems, jobs}
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
--[[
|
||||
Pauses or resumes a queue globably.
|
||||
|
||||
Input:
|
||||
KEYS[1] 'wait' or 'paused''
|
||||
KEYS[2] 'paused' or 'wait'
|
||||
KEYS[3] 'meta'
|
||||
KEYS[4] 'prioritized'
|
||||
KEYS[5] events stream key
|
||||
|
||||
ARGV[1] 'paused' or 'resumed'
|
||||
|
||||
Event:
|
||||
publish paused or resumed event.
|
||||
]]
|
||||
local rcall = redis.call
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/addPriorityMarkerIfNeeded"
|
||||
|
||||
if rcall("EXISTS", KEYS[1]) == 1 then
|
||||
rcall("RENAME", KEYS[1], KEYS[2])
|
||||
end
|
||||
|
||||
if ARGV[1] == "paused" then
|
||||
rcall("HSET", KEYS[3], "paused", 1)
|
||||
else
|
||||
rcall("HDEL", KEYS[3], "paused")
|
||||
local priorityCount = rcall("ZCARD", KEYS[4])
|
||||
|
||||
if priorityCount > 0 then
|
||||
addPriorityMarkerIfNeeded(KEYS[2])
|
||||
end
|
||||
end
|
||||
|
||||
rcall("XADD", KEYS[5], "*", "event", ARGV[1]);
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
--[[
|
||||
Promotes a job that is currently "delayed" to the "waiting" state
|
||||
|
||||
Input:
|
||||
KEYS[1] 'delayed'
|
||||
KEYS[2] 'wait'
|
||||
KEYS[3] 'paused'
|
||||
KEYS[4] 'meta'
|
||||
KEYS[5] 'prioritized'
|
||||
KEYS[6] 'pc' priority counter
|
||||
KEYS[7] 'event stream'
|
||||
|
||||
ARGV[1] queue.toKey('')
|
||||
ARGV[2] jobId
|
||||
|
||||
Output:
|
||||
0 - OK
|
||||
-3 - Job not in delayed zset.
|
||||
|
||||
Events:
|
||||
'waiting'
|
||||
]]
|
||||
local rcall = redis.call
|
||||
local jobId = ARGV[2]
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/addJobWithPriority"
|
||||
--- @include "includes/getTargetQueueList"
|
||||
|
||||
if rcall("ZREM", KEYS[1], jobId) == 1 then
|
||||
local jobKey = ARGV[1] .. jobId
|
||||
local priority = tonumber(rcall("HGET", jobKey, "priority")) or 0
|
||||
local target, paused = getTargetQueueList(KEYS[4], KEYS[2], KEYS[3])
|
||||
|
||||
-- Remove delayed "marker" from the wait list if there is any.
|
||||
-- Since we are adding a job we do not need the marker anymore.
|
||||
local marker = rcall("LINDEX", target, 0)
|
||||
if marker and string.sub(marker, 1, 2) == "0:" then
|
||||
rcall("LPOP", target)
|
||||
end
|
||||
|
||||
if priority == 0 then
|
||||
-- LIFO or FIFO
|
||||
rcall("LPUSH", target, jobId)
|
||||
else
|
||||
addJobWithPriority(KEYS[2], KEYS[5], priority, paused, jobId, KEYS[6])
|
||||
end
|
||||
|
||||
-- Emit waiting event (wait..ing@token)
|
||||
rcall("XADD", KEYS[7], "*", "event", "waiting", "jobId", jobId, "prev", "delayed");
|
||||
|
||||
rcall("HSET", jobKey, "delay", 0)
|
||||
|
||||
return 0
|
||||
else
|
||||
return -3
|
||||
end
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
--[[
|
||||
Release lock
|
||||
|
||||
Input:
|
||||
KEYS[1] 'lock',
|
||||
|
||||
ARGV[1] token
|
||||
ARGV[2] lock duration in milliseconds
|
||||
|
||||
Output:
|
||||
"OK" if lock extented succesfully.
|
||||
]]
|
||||
local rcall = redis.call
|
||||
|
||||
if rcall("GET", KEYS[1]) == ARGV[1] then
|
||||
return rcall("DEL", KEYS[1])
|
||||
else
|
||||
return 0
|
||||
end
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
--[[
|
||||
Remove a job from all the queues it may be in as well as all its data.
|
||||
In order to be able to remove a job, it cannot be active.
|
||||
|
||||
Input:
|
||||
KEYS[1] queue prefix
|
||||
|
||||
ARGV[1] jobId
|
||||
ARGV[2] remove children
|
||||
|
||||
Events:
|
||||
'removed'
|
||||
]]
|
||||
|
||||
local rcall = redis.call
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/destructureJobKey"
|
||||
--- @include "includes/isLocked"
|
||||
--- @include "includes/removeJobFromAnyState"
|
||||
--- @include "includes/removeParentDependencyKey"
|
||||
|
||||
local function removeJob( prefix, jobId, parentKey, removeChildren)
|
||||
local jobKey = prefix .. jobId;
|
||||
|
||||
removeParentDependencyKey(jobKey, false, parentKey)
|
||||
|
||||
if removeChildren == "1" then
|
||||
-- Check if this job has children
|
||||
-- If so, we are going to try to remove the children recursively in deep first way because
|
||||
-- if some job is locked we must exit with and error.
|
||||
--local countProcessed = rcall("HLEN", jobKey .. ":processed")
|
||||
local processed = rcall("HGETALL", jobKey .. ":processed")
|
||||
|
||||
if (#processed > 0) then
|
||||
for i = 1, #processed, 2 do
|
||||
local childJobId = getJobIdFromKey(processed[i])
|
||||
local childJobPrefix = getJobKeyPrefix(processed[i], childJobId)
|
||||
removeJob( childJobPrefix, childJobId, jobKey, removeChildren )
|
||||
end
|
||||
end
|
||||
|
||||
local dependencies = rcall("SMEMBERS", jobKey .. ":dependencies")
|
||||
if (#dependencies > 0) then
|
||||
for i, childJobKey in ipairs(dependencies) do
|
||||
-- We need to get the jobId for this job.
|
||||
local childJobId = getJobIdFromKey(childJobKey)
|
||||
local childJobPrefix = getJobKeyPrefix(childJobKey, childJobId)
|
||||
removeJob( childJobPrefix, childJobId, jobKey, removeChildren )
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
local prev = removeJobFromAnyState(prefix, jobId)
|
||||
|
||||
if rcall("DEL", jobKey, jobKey .. ":logs", jobKey .. ":dependencies", jobKey .. ":processed") > 0 then
|
||||
local maxEvents = rcall("HGET", prefix .. "meta", "opts.maxLenEvents") or 10000
|
||||
rcall("XADD", prefix .. "events", "MAXLEN", "~", maxEvents, "*", "event", "removed",
|
||||
"jobId", jobId, "prev", prev)
|
||||
end
|
||||
end
|
||||
|
||||
local prefix = KEYS[1]
|
||||
|
||||
if not isLocked(prefix, ARGV[1], ARGV[2]) then
|
||||
removeJob(prefix, ARGV[1], nil, ARGV[2])
|
||||
return 1
|
||||
end
|
||||
return 0
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
|
||||
--[[
|
||||
Removes a repeatable job
|
||||
Input:
|
||||
KEYS[1] repeat jobs key
|
||||
KEYS[2] delayed jobs key
|
||||
|
||||
ARGV[1] repeat job id
|
||||
ARGV[2] repeat job key
|
||||
ARGV[3] queue key
|
||||
|
||||
Output:
|
||||
0 - OK
|
||||
1 - Missing repeat job
|
||||
|
||||
Events:
|
||||
'removed'
|
||||
]]
|
||||
local rcall = redis.call
|
||||
local millis = rcall("ZSCORE", KEYS[1], ARGV[2])
|
||||
|
||||
if(millis) then
|
||||
-- Delete next programmed job.
|
||||
local repeatJobId = ARGV[1] .. millis
|
||||
if(rcall("ZREM", KEYS[2], repeatJobId) == 1) then
|
||||
rcall("DEL", ARGV[3] .. repeatJobId)
|
||||
rcall("XADD", ARGV[3] .. "events", "*", "event", "removed", "jobId", repeatJobId, "prev", "delayed");
|
||||
end
|
||||
end
|
||||
|
||||
if(rcall("ZREM", KEYS[1], ARGV[2]) == 1) then
|
||||
return 0
|
||||
end
|
||||
|
||||
return 1
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
--[[
|
||||
Attempts to reprocess a job
|
||||
|
||||
Input:
|
||||
KEYS[1] job key
|
||||
KEYS[2] events stream
|
||||
KEYS[3] job state
|
||||
KEYS[4] wait key
|
||||
KEYS[5] meta
|
||||
KEYS[6] paused key
|
||||
|
||||
ARGV[1] job.id
|
||||
ARGV[2] (job.opts.lifo ? 'R' : 'L') + 'PUSH'
|
||||
ARGV[3] propVal - failedReason/returnvalue
|
||||
ARGV[4] prev state - failed/completed
|
||||
|
||||
Output:
|
||||
1 means the operation was a success
|
||||
-1 means the job does not exist
|
||||
-3 means the job was not found in the expected set.
|
||||
]]
|
||||
local rcall = redis.call;
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/getTargetQueueList"
|
||||
|
||||
if (rcall("EXISTS", KEYS[1]) == 1) then
|
||||
local jobId = ARGV[1]
|
||||
if (rcall("ZREM", KEYS[3], jobId) == 1) then
|
||||
rcall("HDEL", KEYS[1], "finishedOn", "processedOn", ARGV[3])
|
||||
|
||||
local target = getTargetQueueList(KEYS[5], KEYS[4], KEYS[6])
|
||||
rcall(ARGV[2], target, jobId)
|
||||
|
||||
-- Emit waiting event
|
||||
rcall("XADD", KEYS[2], "*", "event", "waiting", "jobId", jobId, "prev", ARGV[4]);
|
||||
return 1
|
||||
else
|
||||
return -3
|
||||
end
|
||||
else
|
||||
return -1
|
||||
end
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
--[[
|
||||
Retries a failed job by moving it back to the wait queue.
|
||||
|
||||
Input:
|
||||
KEYS[1] 'active',
|
||||
KEYS[2] 'wait'
|
||||
KEYS[3] 'paused'
|
||||
KEYS[4] job key
|
||||
KEYS[5] 'meta'
|
||||
KEYS[6] events stream
|
||||
KEYS[7] delayed key
|
||||
KEYS[8] prioritized key
|
||||
KEYS[9] 'pc' priority counter
|
||||
|
||||
ARGV[1] key prefix
|
||||
ARGV[2] timestamp
|
||||
ARGV[3] pushCmd
|
||||
ARGV[4] jobId
|
||||
ARGV[5] token
|
||||
|
||||
Events:
|
||||
'waiting'
|
||||
|
||||
Output:
|
||||
0 - OK
|
||||
-1 - Missing key
|
||||
-2 - Missing lock
|
||||
]]
|
||||
local rcall = redis.call
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/addJobWithPriority"
|
||||
--- @include "includes/getTargetQueueList"
|
||||
--- @include "includes/promoteDelayedJobs"
|
||||
|
||||
local target, paused = getTargetQueueList(KEYS[5], KEYS[2], KEYS[3])
|
||||
-- Check if there are delayed jobs that we can move to wait.
|
||||
-- test example: when there are delayed jobs between retries
|
||||
promoteDelayedJobs(KEYS[7], KEYS[2], target, KEYS[8], KEYS[6], ARGV[1], ARGV[2], paused, KEYS[9])
|
||||
|
||||
if rcall("EXISTS", KEYS[4]) == 1 then
|
||||
|
||||
if ARGV[5] ~= "0" then
|
||||
local lockKey = KEYS[4] .. ':lock'
|
||||
if rcall("GET", lockKey) == ARGV[5] then
|
||||
rcall("DEL", lockKey)
|
||||
else
|
||||
return -2
|
||||
end
|
||||
end
|
||||
|
||||
rcall("LREM", KEYS[1], 0, ARGV[4])
|
||||
|
||||
local priority = tonumber(rcall("HGET", KEYS[4], "priority")) or 0
|
||||
|
||||
-- Standard or priority add
|
||||
if priority == 0 then
|
||||
rcall(ARGV[3], target, ARGV[4])
|
||||
else
|
||||
addJobWithPriority(KEYS[2], KEYS[8], priority, paused, ARGV[4], KEYS[9])
|
||||
end
|
||||
|
||||
local maxEvents = rcall("HGET", KEYS[5], "opts.maxLenEvents") or 10000
|
||||
|
||||
-- Emit waiting event
|
||||
rcall("XADD", KEYS[6], "MAXLEN", "~", maxEvents, "*", "event", "waiting",
|
||||
"jobId", ARGV[4], "prev", "failed")
|
||||
|
||||
return 0
|
||||
else
|
||||
return -1
|
||||
end
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
--[[
|
||||
Save stacktrace and failedReason.
|
||||
|
||||
Input:
|
||||
KEYS[1] job key
|
||||
|
||||
ARGV[1] stacktrace
|
||||
ARGV[2] failedReason
|
||||
|
||||
Output:
|
||||
0 - OK
|
||||
-1 - Missing key
|
||||
]]
|
||||
local rcall = redis.call
|
||||
|
||||
if rcall("EXISTS", KEYS[1]) == 1 then
|
||||
rcall("HMSET", KEYS[1], "stacktrace", ARGV[1], "failedReason", ARGV[2])
|
||||
|
||||
return 0
|
||||
else
|
||||
return -1
|
||||
end
|
||||
+408
@@ -0,0 +1,408 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.ScriptLoader = exports.ScriptLoaderError = void 0;
|
||||
const crypto_1 = require("crypto");
|
||||
const glob_1 = require("glob");
|
||||
const path = require("path");
|
||||
const fs = require("fs");
|
||||
const util_1 = require("util");
|
||||
const readFile = (0, util_1.promisify)(fs.readFile);
|
||||
const readdir = (0, util_1.promisify)(fs.readdir);
|
||||
const GlobOptions = { dot: true, silent: false };
|
||||
const IncludeRegex = /^[-]{2,3}[ \t]*@include[ \t]+(["'])(.+?)\1[; \t\n]*$/m;
|
||||
const EmptyLineRegex = /^\s*[\r\n]/gm;
|
||||
class ScriptLoaderError extends Error {
|
||||
constructor(message, path, stack = [], line, position = 0) {
|
||||
super(message);
|
||||
// Ensure the name of this error is the same as the class name
|
||||
this.name = this.constructor.name;
|
||||
Error.captureStackTrace(this, this.constructor);
|
||||
this.includes = stack;
|
||||
this.line = line !== null && line !== void 0 ? line : 0;
|
||||
this.position = position;
|
||||
}
|
||||
}
|
||||
exports.ScriptLoaderError = ScriptLoaderError;
|
||||
const isPossiblyMappedPath = (path) => path && ['~', '<'].includes(path[0]);
|
||||
const hasFilenamePattern = (path) => (0, glob_1.hasMagic)(path, GlobOptions);
|
||||
/**
|
||||
* Lua script loader with include support
|
||||
*/
|
||||
class ScriptLoader {
|
||||
constructor() {
|
||||
/**
|
||||
* Map an alias to a path
|
||||
*/
|
||||
this.pathMapper = new Map();
|
||||
this.clientScripts = new WeakMap();
|
||||
/**
|
||||
* Cache commands by dir
|
||||
*/
|
||||
this.commandCache = new Map();
|
||||
this.rootPath = getPkgJsonDir();
|
||||
this.pathMapper.set('~', this.rootPath);
|
||||
this.pathMapper.set('rootDir', this.rootPath);
|
||||
this.pathMapper.set('base', __dirname);
|
||||
}
|
||||
/**
|
||||
* Add a script path mapping. Allows includes of the form "<includes>/utils.lua" where `includes` is a user
|
||||
* defined path
|
||||
* @param name - the name of the mapping. Note: do not include angle brackets
|
||||
* @param mappedPath - if a relative path is passed, it's relative to the *caller* of this function.
|
||||
* Mapped paths are also accepted, e.g. "~/server/scripts/lua" or "<base>/includes"
|
||||
*/
|
||||
addPathMapping(name, mappedPath) {
|
||||
let resolved;
|
||||
if (isPossiblyMappedPath(mappedPath)) {
|
||||
resolved = this.resolvePath(mappedPath);
|
||||
}
|
||||
else {
|
||||
const caller = getCallerFile();
|
||||
const callerPath = path.dirname(caller);
|
||||
resolved = path.normalize(path.resolve(callerPath, mappedPath));
|
||||
}
|
||||
const last = resolved.length - 1;
|
||||
if (resolved[last] === path.sep) {
|
||||
resolved = resolved.substr(0, last);
|
||||
}
|
||||
this.pathMapper.set(name, resolved);
|
||||
}
|
||||
/**
|
||||
* Resolve the script path considering path mappings
|
||||
* @param scriptName - the name of the script
|
||||
* @param stack - the include stack, for nicer errors
|
||||
*/
|
||||
resolvePath(scriptName, stack = []) {
|
||||
const first = scriptName[0];
|
||||
if (first === '~') {
|
||||
scriptName = path.join(this.rootPath, scriptName.substr(2));
|
||||
}
|
||||
else if (first === '<') {
|
||||
const p = scriptName.indexOf('>');
|
||||
if (p > 0) {
|
||||
const name = scriptName.substring(1, p);
|
||||
const mappedPath = this.pathMapper.get(name);
|
||||
if (!mappedPath) {
|
||||
throw new ScriptLoaderError(`No path mapping found for "${name}"`, scriptName, stack);
|
||||
}
|
||||
scriptName = path.join(mappedPath, scriptName.substring(p + 1));
|
||||
}
|
||||
}
|
||||
return path.normalize(scriptName);
|
||||
}
|
||||
/**
|
||||
* Recursively collect all scripts included in a file
|
||||
* @param file - the parent file
|
||||
* @param cache - a cache for file metadata to increase efficiency. Since a file can be included
|
||||
* multiple times, we make sure to load it only once.
|
||||
* @param stack - internal stack to prevent circular references
|
||||
*/
|
||||
async resolveDependencies(file, cache, isInclude = false, stack = []) {
|
||||
cache = cache !== null && cache !== void 0 ? cache : new Map();
|
||||
if (stack.includes(file.path)) {
|
||||
throw new ScriptLoaderError(`circular reference: "${file.path}"`, file.path, stack);
|
||||
}
|
||||
stack.push(file.path);
|
||||
function findPos(content, match) {
|
||||
const pos = content.indexOf(match);
|
||||
const arr = content.slice(0, pos).split('\n');
|
||||
return {
|
||||
line: arr.length,
|
||||
column: arr[arr.length - 1].length + match.indexOf('@include') + 1,
|
||||
};
|
||||
}
|
||||
function raiseError(msg, match) {
|
||||
const pos = findPos(file.content, match);
|
||||
throw new ScriptLoaderError(msg, file.path, stack, pos.line, pos.column);
|
||||
}
|
||||
let res;
|
||||
let content = file.content;
|
||||
while ((res = IncludeRegex.exec(content)) !== null) {
|
||||
const [match, , reference] = res;
|
||||
const includeFilename = isPossiblyMappedPath(reference)
|
||||
? // mapped paths imply absolute reference
|
||||
this.resolvePath(ensureExt(reference), stack)
|
||||
: // include path is relative to the file being processed
|
||||
path.resolve(path.dirname(file.path), ensureExt(reference));
|
||||
let includePaths;
|
||||
if (hasFilenamePattern(includeFilename)) {
|
||||
const filesMatched = await getFilenamesByPattern(includeFilename);
|
||||
includePaths = filesMatched.map((x) => path.resolve(x));
|
||||
}
|
||||
else {
|
||||
includePaths = [includeFilename];
|
||||
}
|
||||
includePaths = includePaths.filter((file) => path.extname(file) === '.lua');
|
||||
if (includePaths.length === 0) {
|
||||
raiseError(`include not found: "${reference}"`, match);
|
||||
}
|
||||
const tokens = [];
|
||||
for (let i = 0; i < includePaths.length; i++) {
|
||||
const includePath = includePaths[i];
|
||||
const hasInclude = file.includes.find((x) => x.path === includePath);
|
||||
if (hasInclude) {
|
||||
/**
|
||||
* We have something like
|
||||
* --- \@include "a"
|
||||
* ...
|
||||
* --- \@include "a"
|
||||
*/
|
||||
raiseError(`file "${reference}" already included in "${file.path}"`, match);
|
||||
}
|
||||
let includeMetadata = cache.get(includePath);
|
||||
let token;
|
||||
if (!includeMetadata) {
|
||||
const { name, numberOfKeys } = splitFilename(includePath);
|
||||
let childContent = '';
|
||||
try {
|
||||
const buf = await readFile(includePath, { flag: 'r' });
|
||||
childContent = buf.toString();
|
||||
}
|
||||
catch (err) {
|
||||
if (err.code === 'ENOENT') {
|
||||
raiseError(`include not found: "${reference}"`, match);
|
||||
}
|
||||
else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
// this represents a normalized version of the path to make replacement easy
|
||||
token = getPathHash(includePath);
|
||||
includeMetadata = {
|
||||
name,
|
||||
numberOfKeys,
|
||||
path: includePath,
|
||||
content: childContent,
|
||||
token,
|
||||
includes: [],
|
||||
};
|
||||
cache.set(includePath, includeMetadata);
|
||||
}
|
||||
else {
|
||||
token = includeMetadata.token;
|
||||
}
|
||||
tokens.push(token);
|
||||
file.includes.push(includeMetadata);
|
||||
await this.resolveDependencies(includeMetadata, cache, true, stack);
|
||||
}
|
||||
// Replace @includes with normalized path hashes
|
||||
const substitution = tokens.join('\n');
|
||||
content = content.replace(match, substitution);
|
||||
}
|
||||
file.content = content;
|
||||
if (isInclude) {
|
||||
cache.set(file.path, file);
|
||||
}
|
||||
else {
|
||||
cache.set(file.name, file);
|
||||
}
|
||||
stack.pop();
|
||||
}
|
||||
/**
|
||||
* Parse a (top-level) lua script
|
||||
* @param filename - the full path to the script
|
||||
* @param content - the content of the script
|
||||
* @param cache - cache
|
||||
*/
|
||||
async parseScript(filename, content, cache) {
|
||||
const { name, numberOfKeys } = splitFilename(filename);
|
||||
const meta = cache === null || cache === void 0 ? void 0 : cache.get(name);
|
||||
if ((meta === null || meta === void 0 ? void 0 : meta.content) === content) {
|
||||
return meta;
|
||||
}
|
||||
const fileInfo = {
|
||||
path: filename,
|
||||
token: getPathHash(filename),
|
||||
content,
|
||||
name,
|
||||
numberOfKeys,
|
||||
includes: [],
|
||||
};
|
||||
await this.resolveDependencies(fileInfo, cache);
|
||||
return fileInfo;
|
||||
}
|
||||
/**
|
||||
* Construct the final version of a file by interpolating its includes in dependency order.
|
||||
* @param file - the file whose content we want to construct
|
||||
* @param processed - a cache to keep track of which includes have already been processed
|
||||
*/
|
||||
interpolate(file, processed) {
|
||||
processed = processed || new Set();
|
||||
let content = file.content;
|
||||
file.includes.forEach((child) => {
|
||||
const emitted = processed.has(child.path);
|
||||
const fragment = this.interpolate(child, processed);
|
||||
const replacement = emitted ? '' : fragment;
|
||||
if (!replacement) {
|
||||
content = replaceAll(content, child.token, '');
|
||||
}
|
||||
else {
|
||||
// replace the first instance with the dependency
|
||||
content = content.replace(child.token, replacement);
|
||||
// remove the rest
|
||||
content = replaceAll(content, child.token, '');
|
||||
}
|
||||
processed.add(child.path);
|
||||
});
|
||||
return content;
|
||||
}
|
||||
async loadCommand(filename, cache) {
|
||||
filename = path.resolve(filename);
|
||||
const { name: scriptName } = splitFilename(filename);
|
||||
let script = cache === null || cache === void 0 ? void 0 : cache.get(scriptName);
|
||||
if (!script) {
|
||||
const content = (await readFile(filename)).toString();
|
||||
script = await this.parseScript(filename, content, cache);
|
||||
}
|
||||
const lua = removeEmptyLines(this.interpolate(script));
|
||||
const { name, numberOfKeys } = script;
|
||||
return {
|
||||
name,
|
||||
options: { numberOfKeys: numberOfKeys, lua },
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Load redis lua scripts.
|
||||
* The name of the script must have the following format:
|
||||
*
|
||||
* cmdName-numKeys.lua
|
||||
*
|
||||
* cmdName must be in camel case format.
|
||||
*
|
||||
* For example:
|
||||
* moveToFinish-3.lua
|
||||
*
|
||||
*/
|
||||
async loadScripts(dir, cache) {
|
||||
dir = path.normalize(dir || __dirname);
|
||||
let commands = this.commandCache.get(dir);
|
||||
if (commands) {
|
||||
return commands;
|
||||
}
|
||||
const files = await readdir(dir);
|
||||
const luaFiles = files.filter((file) => path.extname(file) === '.lua');
|
||||
if (luaFiles.length === 0) {
|
||||
/**
|
||||
* To prevent unclarified runtime error "updateDelayset is not a function
|
||||
* @see https://github.com/OptimalBits/bull/issues/920
|
||||
*/
|
||||
throw new ScriptLoaderError('No .lua files found!', dir, []);
|
||||
}
|
||||
commands = [];
|
||||
cache = cache !== null && cache !== void 0 ? cache : new Map();
|
||||
for (let i = 0; i < luaFiles.length; i++) {
|
||||
const file = path.join(dir, luaFiles[i]);
|
||||
const command = await this.loadCommand(file, cache);
|
||||
commands.push(command);
|
||||
}
|
||||
this.commandCache.set(dir, commands);
|
||||
return commands;
|
||||
}
|
||||
/**
|
||||
* Attach all lua scripts in a given directory to a client instance
|
||||
* @param client - redis client to attach script to
|
||||
* @param pathname - the path to the directory containing the scripts
|
||||
*/
|
||||
async load(client, pathname, cache) {
|
||||
let paths = this.clientScripts.get(client);
|
||||
if (!paths) {
|
||||
paths = new Set();
|
||||
this.clientScripts.set(client, paths);
|
||||
}
|
||||
if (!paths.has(pathname)) {
|
||||
paths.add(pathname);
|
||||
const scripts = await this.loadScripts(pathname, cache !== null && cache !== void 0 ? cache : new Map());
|
||||
scripts.forEach((command) => {
|
||||
// Only define the command if not already defined
|
||||
if (!client[command.name]) {
|
||||
client.defineCommand(command.name, command.options);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Clears the command cache
|
||||
*/
|
||||
clearCache() {
|
||||
this.commandCache.clear();
|
||||
}
|
||||
}
|
||||
exports.ScriptLoader = ScriptLoader;
|
||||
function ensureExt(filename, ext = 'lua') {
|
||||
const foundExt = path.extname(filename);
|
||||
if (foundExt && foundExt !== '.') {
|
||||
return filename;
|
||||
}
|
||||
if (ext && ext[0] !== '.') {
|
||||
ext = `.${ext}`;
|
||||
}
|
||||
return `${filename}${ext}`;
|
||||
}
|
||||
function splitFilename(filePath) {
|
||||
const longName = path.basename(filePath, '.lua');
|
||||
const [name, num] = longName.split('-');
|
||||
const numberOfKeys = num ? parseInt(num, 10) : undefined;
|
||||
return { name, numberOfKeys };
|
||||
}
|
||||
async function getFilenamesByPattern(pattern) {
|
||||
return new Promise((resolve, reject) => {
|
||||
(0, glob_1.glob)(pattern, GlobOptions, (err, files) => {
|
||||
return err ? reject(err) : resolve(files);
|
||||
});
|
||||
});
|
||||
}
|
||||
// Determine the project root
|
||||
// https://stackoverflow.com/a/18721515
|
||||
function getPkgJsonDir() {
|
||||
for (const modPath of module.paths || []) {
|
||||
try {
|
||||
const prospectivePkgJsonDir = path.dirname(modPath);
|
||||
fs.accessSync(modPath, fs.constants.F_OK);
|
||||
return prospectivePkgJsonDir;
|
||||
// eslint-disable-next-line no-empty
|
||||
}
|
||||
catch (e) { }
|
||||
}
|
||||
return '';
|
||||
}
|
||||
// https://stackoverflow.com/a/66842927
|
||||
// some dark magic here :-)
|
||||
// this version is preferred to the simpler version because of
|
||||
// https://github.com/facebook/jest/issues/5303 -
|
||||
// tldr: dont assume you're the only one with the doing something like this
|
||||
function getCallerFile() {
|
||||
var _a, _b, _c;
|
||||
const originalFunc = Error.prepareStackTrace;
|
||||
let callerFile = '';
|
||||
try {
|
||||
Error.prepareStackTrace = (_, stack) => stack;
|
||||
const sites = new Error().stack;
|
||||
const currentFile = (_a = sites.shift()) === null || _a === void 0 ? void 0 : _a.getFileName();
|
||||
while (sites.length) {
|
||||
callerFile = (_c = (_b = sites.shift()) === null || _b === void 0 ? void 0 : _b.getFileName()) !== null && _c !== void 0 ? _c : '';
|
||||
if (currentFile !== callerFile) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
// eslint-disable-next-line no-empty
|
||||
}
|
||||
catch (e) {
|
||||
}
|
||||
finally {
|
||||
Error.prepareStackTrace = originalFunc;
|
||||
}
|
||||
return callerFile;
|
||||
}
|
||||
function sha1(data) {
|
||||
return (0, crypto_1.createHash)('sha1').update(data).digest('hex');
|
||||
}
|
||||
function getPathHash(normalizedPath) {
|
||||
return `@@${sha1(normalizedPath)}`;
|
||||
}
|
||||
function replaceAll(str, find, replace) {
|
||||
return str.replace(new RegExp(find, 'g'), replace);
|
||||
}
|
||||
function removeEmptyLines(str) {
|
||||
return str.replace(EmptyLineRegex, '');
|
||||
}
|
||||
//# sourceMappingURL=script-loader.js.map
|
||||
+1
File diff suppressed because one or more lines are too long
+20
@@ -0,0 +1,20 @@
|
||||
--[[
|
||||
Update job data
|
||||
|
||||
Input:
|
||||
KEYS[1] Job id key
|
||||
|
||||
ARGV[1] data
|
||||
|
||||
Output:
|
||||
0 - OK
|
||||
-1 - Missing job.
|
||||
]]
|
||||
local rcall = redis.call
|
||||
|
||||
if rcall("EXISTS",KEYS[1]) == 1 then -- // Make sure job exists
|
||||
rcall("HSET", KEYS[1], "data", ARGV[1])
|
||||
return 0
|
||||
else
|
||||
return -1
|
||||
end
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
--[[
|
||||
Update job progress
|
||||
|
||||
Input:
|
||||
KEYS[1] Job id key
|
||||
KEYS[2] event stream key
|
||||
KEYS[3] meta key
|
||||
|
||||
ARGV[1] id
|
||||
ARGV[2] progress
|
||||
|
||||
Output:
|
||||
0 - OK
|
||||
-1 - Missing job.
|
||||
|
||||
Event:
|
||||
progress(jobId, progress)
|
||||
]]
|
||||
local rcall = redis.call
|
||||
|
||||
if rcall("EXISTS", KEYS[1]) == 1 then -- // Make sure job exists
|
||||
local maxEvents = rcall("HGET", KEYS[3], "opts.maxLenEvents") or 10000
|
||||
|
||||
rcall("HSET", KEYS[1], "progress", ARGV[2])
|
||||
rcall("XADD", KEYS[2], "MAXLEN", "~", maxEvents, "*", "event", "progress",
|
||||
"jobId", ARGV[1], "data", ARGV[2]);
|
||||
return 0
|
||||
else
|
||||
return -1
|
||||
end
|
||||
Reference in New Issue
Block a user