Initial commit of working RSS Aggregator build

This commit is contained in:
2026-05-12 17:04:02 -03:00
parent ea3a2ca53e
commit 7ac2f6e384
4962 changed files with 1032666 additions and 0 deletions
+124
View File
@@ -0,0 +1,124 @@
--[[
Adds a delayed job to the queue by doing the following:
- Increases the job counter if needed.
- Creates a new job key with the job data.
- computes timestamp.
- adds to delayed zset.
- Emits a global event 'delayed' if the job is delayed.
Input:
KEYS[1] 'wait',
KEYS[2] 'paused'
KEYS[3] 'meta'
KEYS[4] 'id'
KEYS[5] 'delayed'
KEYS[6] 'completed'
KEYS[7] events stream key
ARGV[1] msgpacked arguments array
[1] key prefix,
[2] custom id (use custom instead of one generated automatically)
[3] name
[4] timestamp
[5] parentKey?
x [6] waitChildrenKey key.
[7] parent dependencies key.
[8] parent? {id, queueKey}
[9] repeat job key
ARGV[2] Json stringified job data
ARGV[3] msgpacked options
Output:
jobId - OK
-5 - Missing parent key
]]
local waitKey = KEYS[1]
local pausedKey = KEYS[2]
local metaKey = KEYS[3]
local idKey = KEYS[4]
local delayedKey = KEYS[5]
local completedKey = KEYS[6]
local eventsKey = KEYS[7]
local jobId
local jobIdKey
local rcall = redis.call
local args = cmsgpack.unpack(ARGV[1])
local data = ARGV[2]
local opts = cmsgpack.unpack(ARGV[3])
local parentKey = args[5]
local repeatJobKey = args[9]
local parent = args[8]
local parentData
-- Includes
--- @include "includes/storeJob"
--- @include "includes/addDelayMarkerIfNeeded"
--- @include "includes/getTargetQueueList"
--- @include "includes/getNextDelayedTimestamp"
--- @include "includes/updateExistingJobsParent"
--- @include "includes/getOrSetMaxEvents"
if parentKey ~= nil then
if rcall("EXISTS", parentKey) ~= 1 then return -5 end
parentData = cjson.encode(parent)
end
local jobCounter = rcall("INCR", idKey)
local maxEvents = getOrSetMaxEvents(metaKey)
local parentDependenciesKey = args[7]
local timestamp = args[4]
if args[2] == "" then
jobId = jobCounter
jobIdKey = args[1] .. jobId
else
-- Refactor to: handleDuplicateJob.lua
jobId = args[2]
jobIdKey = args[1] .. jobId
if rcall("EXISTS", jobIdKey) == 1 then
updateExistingJobsParent(parentKey, parent, parentData,
parentDependenciesKey, completedKey, jobIdKey,
jobId, timestamp)
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
"duplicated", "jobId", jobId)
return jobId .. "" -- convert to string
end
end
-- Store the job.
local delay, priority = storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2],
opts, timestamp, parentKey, parentData,
repeatJobKey)
-- Compute delayed timestamp and the score.
local delayedTimestamp = (delay > 0 and (timestamp + delay)) or 0
local score = delayedTimestamp * 0x1000 + bit.band(jobCounter, 0xfff)
rcall("ZADD", delayedKey, score, jobId)
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
"jobId", jobId, "delay", delayedTimestamp)
-- If wait list is empty, and this delayed job is the next one to be processed,
-- then we need to signal the workers by adding a dummy job (jobId 0:delay) to the wait list.
local target = getTargetQueueList(metaKey, KEYS[1], KEYS[2])
addDelayMarkerIfNeeded(target, delayedKey)
-- Check if this job is a child of another job, if so add it to the parents dependencies
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
-- fail in this case.
if parentDependenciesKey ~= nil then
rcall("SADD", parentDependenciesKey, jobIdKey)
end
return jobId .. "" -- convert to string
+101
View File
@@ -0,0 +1,101 @@
--[[
Adds a parent job to the queue by doing the following:
- Increases the job counter if needed.
- Creates a new job key with the job data.
- adds the job to the waiting-children zset
Input:
KEYS[1] 'meta'
KEYS[2] 'id'
KEYS[3] 'completed'
KEYS[4] events stream key
ARGV[1] msgpacked arguments array
[1] key prefix,
[2] custom id (will not generate one automatically)
[3] name
[4] timestamp
[5] parentKey?
[6] waitChildrenKey key.
[7] parent dependencies key.
[8] parent? {id, queueKey}
[9] repeat job key
ARGV[2] Json stringified job data
ARGV[3] msgpacked options
Output:
jobId - OK
-5 - Missing parent key
]]
local metaKey = KEYS[1]
local idKey = KEYS[2]
local completedKey = KEYS[3]
local eventsKey = KEYS[4]
local jobId
local jobIdKey
local rcall = redis.call
local args = cmsgpack.unpack(ARGV[1])
local data = ARGV[2]
local opts = cmsgpack.unpack(ARGV[3])
local parentKey = args[5]
local repeatJobKey = args[9]
local parent = args[8]
local parentData
-- Includes
--- @include "includes/storeJob"
--- @include "includes/updateExistingJobsParent"
--- @include "includes/getOrSetMaxEvents"
if parentKey ~= nil then
if rcall("EXISTS", parentKey) ~= 1 then return -5 end
parentData = cjson.encode(parent)
end
local jobCounter = rcall("INCR", idKey)
local maxEvents = getOrSetMaxEvents(metaKey)
local parentDependenciesKey = args[7]
local timestamp = args[4]
if args[2] == "" then
jobId = jobCounter
jobIdKey = args[1] .. jobId
else
jobId = args[2]
jobIdKey = args[1] .. jobId
if rcall("EXISTS", jobIdKey) == 1 then
updateExistingJobsParent(parentKey, parent, parentData,
parentDependenciesKey, completedKey, jobIdKey,
jobId, timestamp)
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
"duplicated", "jobId", jobId)
return jobId .. "" -- convert to string
end
end
-- Store the job.
storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2], opts, timestamp,
parentKey, parentData, repeatJobKey)
local waitChildrenKey = args[6]
rcall("ZADD", waitChildrenKey, timestamp, jobId)
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
"waiting-children", "jobId", jobId)
-- Check if this job is a child of another job, if so add it to the parents dependencies
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
-- fail in this case.
if parentDependenciesKey ~= nil then
rcall("SADD", parentDependenciesKey, jobIdKey)
end
return jobId .. "" -- convert to string
+117
View File
@@ -0,0 +1,117 @@
--[[
Adds a priotitized job to the queue by doing the following:
- Increases the job counter if needed.
- Creates a new job key with the job data.
- Adds the job to the "added" list so that workers gets notified.
Input:
KEYS[1] 'wait',
KEYS[2] 'paused'
KEYS[3] 'meta'
KEYS[4] 'id'
KEYS[5] 'prioritized'
KEYS[6] 'completed'
KEYS[7] events stream key
KEYS[8] 'pc' priority counter
ARGV[1] msgpacked arguments array
[1] key prefix,
[2] custom id (will not generate one automatically)
[3] name
[4] timestamp
[5] parentKey?
[6] waitChildrenKey key.
[7] parent dependencies key.
[8] parent? {id, queueKey}
[9] repeat job key
ARGV[2] Json stringified job data
ARGV[3] msgpacked options
Output:
jobId - OK
-5 - Missing parent key
]]
local waitKey = KEYS[1]
local pausedKey = KEYS[2]
local metaKey = KEYS[3]
local idKey = KEYS[4]
local priorityKey = KEYS[5]
local completedKey = KEYS[6]
local eventsKey = KEYS[7]
local priorityCounterKey = KEYS[8]
local jobId
local jobIdKey
local rcall = redis.call
local args = cmsgpack.unpack(ARGV[1])
local data = ARGV[2]
local opts = cmsgpack.unpack(ARGV[3])
local parentKey = args[5]
local repeatJobKey = args[9]
local parent = args[8]
local parentData
-- Includes
--- @include "includes/storeJob"
--- @include "includes/addJobWithPriority"
--- @include "includes/getTargetQueueList"
--- @include "includes/updateExistingJobsParent"
--- @include "includes/getOrSetMaxEvents"
if parentKey ~= nil then
if rcall("EXISTS", parentKey) ~= 1 then return -5 end
parentData = cjson.encode(parent)
end
local jobCounter = rcall("INCR", idKey)
local maxEvents = getOrSetMaxEvents(metaKey)
local parentDependenciesKey = args[7]
local timestamp = args[4]
if args[2] == "" then
jobId = jobCounter
jobIdKey = args[1] .. jobId
else
jobId = args[2]
jobIdKey = args[1] .. jobId
if rcall("EXISTS", jobIdKey) == 1 then
updateExistingJobsParent(parentKey, parent, parentData,
parentDependenciesKey, completedKey, jobIdKey,
jobId, timestamp)
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
"duplicated", "jobId", jobId)
return jobId .. "" -- convert to string
end
end
-- Store the job.
local delay, priority = storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2],
opts, timestamp, parentKey, parentData,
repeatJobKey)
local target, paused = getTargetQueueList(metaKey, waitKey, pausedKey)
addJobWithPriority(waitKey, priorityKey, priority, paused, jobId,
priorityCounterKey)
-- Emit waiting event
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting",
"jobId", jobId)
-- Check if this job is a child of another job, if so add it to the parents dependencies
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
-- fail in this case.
if parentDependenciesKey ~= nil then
rcall("SADD", parentDependenciesKey, jobIdKey)
end
return jobId .. "" -- convert to string
+117
View File
@@ -0,0 +1,117 @@
--[[
Adds a job to the queue by doing the following:
- Increases the job counter if needed.
- Creates a new job key with the job data.
- if delayed:
- computes timestamp.
- adds to delayed zset.
- Emits a global event 'delayed' if the job is delayed.
- if not delayed
- Adds the jobId to the wait/paused list in one of three ways:
- LIFO
- FIFO
- prioritized.
- Adds the job to the "added" list so that workers gets notified.
Input:
KEYS[1] 'wait',
KEYS[2] 'paused'
KEYS[3] 'meta'
KEYS[4] 'id'
KEYS[5] 'completed'
KEYS[6] events stream key
ARGV[1] msgpacked arguments array
[1] key prefix,
[2] custom id (will not generate one automatically)
[3] name
[4] timestamp
[5] parentKey?
[6] waitChildrenKey key.
[7] parent dependencies key.
[8] parent? {id, queueKey}
[9] repeat job key
ARGV[2] Json stringified job data
ARGV[3] msgpacked options
Output:
jobId - OK
-5 - Missing parent key
]]
local eventsKey = KEYS[6]
local jobId
local jobIdKey
local rcall = redis.call
local args = cmsgpack.unpack(ARGV[1])
local data = ARGV[2]
local opts = cmsgpack.unpack(ARGV[3])
local parentKey = args[5]
local repeatJobKey = args[9]
local parent = args[8]
local parentData
-- Includes
--- @include "includes/storeJob"
--- @include "includes/updateExistingJobsParent"
--- @include "includes/getTargetQueueList"
--- @include "includes/getOrSetMaxEvents"
if parentKey ~= nil then
if rcall("EXISTS", parentKey) ~= 1 then return -5 end
parentData = cjson.encode(parent)
end
local jobCounter = rcall("INCR", KEYS[4])
local metaKey = KEYS[3]
local maxEvents = getOrSetMaxEvents(metaKey)
local parentDependenciesKey = args[7]
local timestamp = args[4]
if args[2] == "" then
jobId = jobCounter
jobIdKey = args[1] .. jobId
else
jobId = args[2]
jobIdKey = args[1] .. jobId
if rcall("EXISTS", jobIdKey) == 1 then
updateExistingJobsParent(parentKey, parent, parentData,
parentDependenciesKey, KEYS[5], jobIdKey,
jobId, timestamp)
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
"duplicated", "jobId", jobId)
return jobId .. "" -- convert to string
end
end
-- Store the job.
storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2], opts, timestamp,
parentKey, parentData, repeatJobKey)
local target, paused = getTargetQueueList(metaKey, KEYS[1], KEYS[2])
-- LIFO or FIFO
local pushCmd = opts['lifo'] and 'RPUSH' or 'LPUSH'
rcall(pushCmd, target, jobId)
-- Emit waiting event
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting",
"jobId", jobId)
-- Check if this job is a child of another job, if so add it to the parents dependencies
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
-- fail in this case.
if parentDependenciesKey ~= nil then
rcall("SADD", parentDependenciesKey, jobIdKey)
end
return jobId .. "" -- convert to string
+40
View File
@@ -0,0 +1,40 @@
--[[
Change job delay when it is in delayed set.
Input:
KEYS[1] delayed key
KEYS[2] job key
KEYS[3] events stream
ARGV[1] delay
ARGV[2] delayedTimestamp
ARGV[3] the id of the job
Output:
0 - OK
-1 - Missing job.
-3 - Job not in delayed set.
Events:
- delayed key.
]]
local rcall = redis.call
if rcall("EXISTS", KEYS[2]) == 1 then
local jobId = ARGV[3]
local score = tonumber(ARGV[2])
local delayedTimestamp = (score / 0x1000)
local numRemovedElements = rcall("ZREM", KEYS[1], jobId)
if numRemovedElements < 1 then
return -3
end
rcall("HSET", KEYS[2], "delay", tonumber(ARGV[1]))
rcall("ZADD", KEYS[1], score, jobId)
rcall("XADD", KEYS[3], "*", "event", "delayed", "jobId", jobId, "delay", delayedTimestamp)
return 0
else
return -1
end
+52
View File
@@ -0,0 +1,52 @@
--[[
Change job priority
Input:
KEYS[1] 'wait',
KEYS[2] 'paused'
KEYS[3] 'meta'
KEYS[4] 'prioritized'
KEYS[5] 'pc' priority counter
ARGV[1] priority value
ARGV[2] job key
ARGV[3] job id
ARGV[4] lifo
Output:
0 - OK
-1 - Missing job
]]
local jobKey = ARGV[2]
local jobId = ARGV[3]
local priority = tonumber(ARGV[1])
local rcall = redis.call
-- Includes
--- @include "includes/addJobWithPriority"
--- @include "includes/getTargetQueueList"
if rcall("EXISTS", jobKey) == 1 then
local target, paused = getTargetQueueList(KEYS[3], KEYS[1], KEYS[2])
if rcall("ZREM", KEYS[4], jobId) > 0 then
addJobWithPriority(KEYS[1], KEYS[4], priority, paused, jobId, KEYS[5])
else
local numRemovedElements = rcall("LREM", target, -1, jobId)
if numRemovedElements > 0 then
-- Standard or priority add
if priority == 0 then
-- LIFO or FIFO
local pushCmd = ARGV[4] == '1' and 'RPUSH' or 'LPUSH';
rcall(pushCmd, target, jobId)
else
addJobWithPriority(KEYS[1], KEYS[4], priority, paused, jobId, KEYS[5])
end
end
end
rcall("HSET", jobKey, "priority", priority)
return 0
else
return -1
end
+54
View File
@@ -0,0 +1,54 @@
--[[
Remove jobs from the specific set.
Input:
KEYS[1] set key,
KEYS[2] events stream key
ARGV[1] jobKey prefix
ARGV[2] timestamp
ARGV[3] limit the number of jobs to be removed. 0 is unlimited
ARGV[4] set name, can be any of 'wait', 'active', 'paused', 'delayed', 'completed', or 'failed'
]]
local rcall = redis.call
local rangeStart = 0
local rangeEnd = -1
local limit = tonumber(ARGV[3])
-- If we're only deleting _n_ items, avoid retrieving all items
-- for faster performance
--
-- Start from the tail of the list, since that's where oldest elements
-- are generally added for FIFO lists
if limit > 0 then
rangeStart = -1 - limit + 1
rangeEnd = -1
end
-- Includes
--- @include "includes/cleanList"
--- @include "includes/cleanSet"
local result
if ARGV[4] == "active" then
result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], false)
elseif ARGV[4] == "delayed" then
rangeEnd = "+inf"
result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
{"processedOn", "timestamp"}, false)
elseif ARGV[4] == "prioritized" then
rangeEnd = "+inf"
result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
{"timestamp"}, false)
elseif ARGV[4] == "wait" or ARGV[4] == "paused" then
result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], true)
else
rangeEnd = ARGV[2]
result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
{"finishedOn"}, true)
end
rcall("XADD", KEYS[2], "*", "event", "cleaned", "count", result[2])
return result[1]
+26
View File
@@ -0,0 +1,26 @@
--[[
Drains the queue, removes all jobs that are waiting
or delayed, but not active, completed or failed
Input:
KEYS[1] 'wait',
KEYS[2] 'paused'
KEYS[3] 'delayed'
KEYS[4] 'prioritized'
ARGV[1] queue key prefix
]]
local rcall = redis.call
local queueBaseKey = ARGV[1]
--- @include "includes/removeListJobs"
--- @include "includes/removeZSetJobs"
removeListJobs(KEYS[1], true, queueBaseKey, 0) --wait
removeListJobs(KEYS[2], true, queueBaseKey, 0) --paused
if KEYS[3] ~= "" then
removeZSetJobs(KEYS[3], true, queueBaseKey, 0) --delayed
end
removeZSetJobs(KEYS[4], true, queueBaseKey, 0) --prioritized
+23
View File
@@ -0,0 +1,23 @@
--[[
Extend lock and removes the job from the stalled set.
Input:
KEYS[1] 'lock',
KEYS[2] 'stalled'
ARGV[1] token
ARGV[2] lock duration in milliseconds
ARGV[3] jobid
Output:
"1" if lock extented succesfully.
]]
local rcall = redis.call
if rcall("GET", KEYS[1]) == ARGV[1] then
-- if rcall("SET", KEYS[1], ARGV[1], "PX", ARGV[2], "XX") then
if rcall("SET", KEYS[1], ARGV[1], "PX", ARGV[2]) then
rcall("SREM", KEYS[2], ARGV[3])
return 1
end
end
return 0
+35
View File
@@ -0,0 +1,35 @@
--[[
Get counts per provided states
Input:
KEYS[1] 'prefix'
ARGV[1...] types
]]
local rcall = redis.call;
local prefix = KEYS[1]
local results = {}
for i = 1, #ARGV do
local stateKey = prefix .. ARGV[i]
if ARGV[i] == "wait" or ARGV[i] == "paused" then
local marker = rcall("LINDEX", stateKey, -1)
if marker and string.sub(marker, 1, 2) == "0:" then
local count = rcall("LLEN", stateKey)
if count > 1 then
rcall("RPOP", stateKey)
results[#results+1] = count-1
else
results[#results+1] = 0
end
else
results[#results+1] = rcall("LLEN", stateKey)
end
elseif ARGV[i] == "active" then
results[#results+1] = rcall("LLEN", stateKey)
else
results[#results+1] = rcall("ZCARD", stateKey)
end
end
return results
+69
View File
@@ -0,0 +1,69 @@
--[[
Get job ids per provided states
Input:
KEYS[1] 'prefix'
ARGV[1] start
ARGV[2] end
ARGV[3] asc
ARGV[4...] types
]]
local rcall = redis.call
local prefix = KEYS[1]
local rangeStart = tonumber(ARGV[1])
local rangeEnd = tonumber(ARGV[2])
local asc = ARGV[3]
local results = {}
local function getRangeInList(listKey, asc, rangeStart, rangeEnd, results)
if asc == "1" then
local modifiedRangeStart
local modifiedRangeEnd
if rangeStart == -1 then
modifiedRangeStart = 0
else
modifiedRangeStart = -(rangeStart + 1)
end
if rangeEnd == -1 then
modifiedRangeEnd = 0
else
modifiedRangeEnd = -(rangeEnd + 1)
end
results[#results+1] = rcall("LRANGE", listKey,
modifiedRangeEnd,
modifiedRangeStart)
else
results[#results+1] = rcall("LRANGE", listKey, rangeStart, rangeEnd)
end
end
for i = 4, #ARGV do
local stateKey = prefix .. ARGV[i]
if ARGV[i] == "wait" or ARGV[i] == "paused" then
local marker = rcall("LINDEX", stateKey, -1)
if marker and string.sub(marker, 1, 2) == "0:" then
local count = rcall("LLEN", stateKey)
if count > 1 then
rcall("RPOP", stateKey)
getRangeInList(stateKey, asc, rangeStart, rangeEnd, results)
else
results[#results+1] = {}
end
else
getRangeInList(stateKey, asc, rangeStart, rangeEnd, results)
end
elseif ARGV[i] == "active" then
getRangeInList(stateKey, asc, rangeStart, rangeEnd, results)
else
if asc == "1" then
results[#results+1] = rcall("ZRANGE", stateKey, rangeStart, rangeEnd)
else
results[#results+1] = rcall("ZREVRANGE", stateKey, rangeStart, rangeEnd)
end
end
end
return results
+65
View File
@@ -0,0 +1,65 @@
--[[
Get a job state
Input:
KEYS[1] 'completed' key,
KEYS[2] 'failed' key
KEYS[3] 'delayed' key
KEYS[4] 'active' key
KEYS[5] 'wait' key
KEYS[6] 'paused' key
KEYS[7] 'waiting-children' key
KEYS[8] 'prioritized' key
ARGV[1] job id
Output:
'completed'
'failed'
'delayed'
'active'
'prioritized'
'waiting'
'waiting-children'
'unknown'
]]
local rcall = redis.call
if rcall("ZSCORE", KEYS[1], ARGV[1]) ~= false then
return "completed"
end
if rcall("ZSCORE", KEYS[2], ARGV[1]) ~= false then
return "failed"
end
if rcall("ZSCORE", KEYS[3], ARGV[1]) ~= false then
return "delayed"
end
if rcall("ZSCORE", KEYS[8], ARGV[1]) ~= false then
return "prioritized"
end
-- Includes
--- @include "includes/checkItemInList"
local active_items = rcall("LRANGE", KEYS[4] , 0, -1)
if checkItemInList(active_items, ARGV[1]) ~= nil then
return "active"
end
local wait_items = rcall("LRANGE", KEYS[5] , 0, -1)
if checkItemInList(wait_items, ARGV[1]) ~= nil then
return "waiting"
end
local paused_items = rcall("LRANGE", KEYS[6] , 0, -1)
if checkItemInList(paused_items, ARGV[1]) ~= nil then
return "waiting"
end
if rcall("ZSCORE", KEYS[7], ARGV[1]) ~= false then
return "waiting-children"
end
return "unknown"
+58
View File
@@ -0,0 +1,58 @@
--[[
Get a job state
Input:
KEYS[1] 'completed' key,
KEYS[2] 'failed' key
KEYS[3] 'delayed' key
KEYS[4] 'active' key
KEYS[5] 'wait' key
KEYS[6] 'paused' key
KEYS[7] 'waiting-children' key
KEYS[8] 'prioritized' key
ARGV[1] job id
Output:
'completed'
'failed'
'delayed'
'active'
'waiting'
'waiting-children'
'unknown'
]]
local rcall = redis.call
if rcall("ZSCORE", KEYS[1], ARGV[1]) ~= false then
return "completed"
end
if rcall("ZSCORE", KEYS[2], ARGV[1]) ~= false then
return "failed"
end
if rcall("ZSCORE", KEYS[3], ARGV[1]) ~= false then
return "delayed"
end
if rcall("ZSCORE", KEYS[8], ARGV[1]) ~= false then
return "prioritized"
end
if rcall("LPOS", KEYS[4] , ARGV[1]) ~= false then
return "active"
end
if rcall("LPOS", KEYS[5] , ARGV[1]) ~= false then
return "waiting"
end
if rcall("LPOS", KEYS[6] , ARGV[1]) ~= false then
return "waiting"
end
if rcall("ZSCORE", KEYS[7] , ARGV[1]) ~= false then
return "waiting-children"
end
return "unknown"
@@ -0,0 +1,27 @@
--[[
Add delay marker if needed.
]]
-- Includes
--- @include "getNextDelayedTimestamp"
local function addDelayMarkerIfNeeded(targetKey, delayedKey)
local waitLen = rcall("LLEN", targetKey)
if waitLen <= 1 then
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
if nextTimestamp ~= nil then
-- Check if there is already a marker with older timestamp
-- if there is, we need to replace it.
if waitLen == 1 then
local marker = rcall("LINDEX", targetKey, 0)
local oldTimestamp = tonumber(marker:sub(3))
if oldTimestamp and oldTimestamp > nextTimestamp then
rcall("LSET", targetKey, 0, "0:" .. nextTimestamp)
end
else
-- if there is no marker, then we need to add one
rcall("LPUSH", targetKey, "0:" .. nextTimestamp)
end
end
end
end
+15
View File
@@ -0,0 +1,15 @@
--[[
Function to add job considering priority.
]]
-- Includes
--- @include "addPriorityMarkerIfNeeded"
--- @include "getPriorityScore"
local function addJobWithPriority(waitKey, prioritizedKey, priority, paused, jobId, priorityCounterKey)
local score = getPriorityScore(priority, priorityCounterKey)
rcall("ZADD", prioritizedKey, score, jobId)
if not paused then
addPriorityMarkerIfNeeded(waitKey)
end
end
@@ -0,0 +1,12 @@
--[[
Function priority marker to wait if needed
in order to wake up our workers and to respect priority
order as much as possible
]]
local function addPriorityMarkerIfNeeded(waitKey)
local waitLen = rcall("LLEN", waitKey)
if waitLen == 0 then
rcall("LPUSH", waitKey, "0:0")
end
end
+18
View File
@@ -0,0 +1,18 @@
--[[
Function to loop in batches.
Just a bit of warning, some commands as ZREM
could receive a maximum of 7000 parameters per call.
]]
local function batches(n, batchSize)
local i = 0
return function()
local from = i * batchSize + 1
i = i + 1
if (from <= n) then
local to = math.min(from + batchSize - 1, n)
return from, to
end
end
end
+12
View File
@@ -0,0 +1,12 @@
--[[
Functions to check if a item belongs to a list.
]]
local function checkItemInList(list, item)
for _, v in pairs(list) do
if v == item then
return 1
end
end
return nil
end
+139
View File
@@ -0,0 +1,139 @@
--[[
Move stalled jobs to wait.
Input:
stalledKey 'stalled' (SET)
waitKey 'wait', (LIST)
activeKey 'active', (LIST)
failedKey 'failed', (ZSET)
stalledCheckKey 'stalled-check', (KEY)
metaKey 'meta', (KEY)
pausedKey 'paused', (LIST)
eventStreamKey 'event stream' (STREAM)
maxStalledJobCount Max stalled job count
queueKeyPrefix queue.toKey('')
timestamp timestamp
maxCheckTime max check time
Events:
'stalled' with stalled job id.
]]
local rcall = redis.call
-- Includes
--- @include "batches"
--- @include "getTargetQueueList"
--- @include "removeJob"
--- @include "removeJobsByMaxAge"
--- @include "removeJobsByMaxCount"
--- @include "trimEvents"
-- Check if we need to check for stalled jobs now.
local function checkStalledJobs(stalledKey, waitKey, activeKey, failedKey,
stalledCheckKey, metaKey, pausedKey,
eventStreamKey, maxStalledJobCount,
queueKeyPrefix, timestamp, maxCheckTime)
if rcall("EXISTS", stalledCheckKey) == 1 then return {{}, {}} end
rcall("SET", stalledCheckKey, timestamp, "PX", maxCheckTime)
-- Trim events before emiting them to avoid trimming events emitted in this script
trimEvents(metaKey, eventStreamKey)
-- Move all stalled jobs to wait
local stalling = rcall('SMEMBERS', stalledKey)
local stalled = {}
local failed = {}
if (#stalling > 0) then
rcall('DEL', stalledKey)
local MAX_STALLED_JOB_COUNT = tonumber(maxStalledJobCount)
-- Remove from active list
for i, jobId in ipairs(stalling) do
if string.sub(jobId, 1, 2) == "0:" then
-- If the jobId is a delay marker ID we just remove it.
rcall("LREM", activeKey, 1, jobId)
else
local jobKey = queueKeyPrefix .. jobId
-- Check that the lock is also missing, then we can handle this job as really stalled.
if (rcall("EXISTS", jobKey .. ":lock") == 0) then
-- Remove from the active queue.
local removed = rcall("LREM", activeKey, 1, jobId)
if (removed > 0) then
-- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
local stalledCount =
rcall("HINCRBY", jobKey, "stalledCounter", 1)
if (stalledCount > MAX_STALLED_JOB_COUNT) then
local rawOpts = rcall("HGET", jobKey, "opts")
local opts = cjson.decode(rawOpts)
local removeOnFailType = type(opts["removeOnFail"])
rcall("ZADD", failedKey, timestamp, jobId)
local failedReason =
"job stalled more than allowable limit"
rcall("HMSET", jobKey, "failedReason", failedReason,
"finishedOn", timestamp)
rcall("XADD", eventStreamKey, "*", "event",
"failed", "jobId", jobId, 'prev', 'active',
'failedReason', failedReason)
if removeOnFailType == "number" then
removeJobsByMaxCount(opts["removeOnFail"],
failedKey, queueKeyPrefix)
elseif removeOnFailType == "boolean" then
if opts["removeOnFail"] then
removeJob(jobId, false, queueKeyPrefix)
rcall("ZREM", failedKey, jobId)
end
elseif removeOnFailType ~= "nil" then
local maxAge = opts["removeOnFail"]["age"]
local maxCount = opts["removeOnFail"]["count"]
if maxAge ~= nil then
removeJobsByMaxAge(timestamp, maxAge,
failedKey, queueKeyPrefix)
end
if maxCount ~= nil and maxCount > 0 then
removeJobsByMaxCount(maxCount, failedKey,
queueKeyPrefix)
end
end
table.insert(failed, jobId)
else
local target =
getTargetQueueList(metaKey, waitKey, pausedKey)
-- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
rcall("RPUSH", target, jobId)
rcall("XADD", eventStreamKey, "*", "event",
"waiting", "jobId", jobId, 'prev', 'active')
-- Emit the stalled event
rcall("XADD", eventStreamKey, "*", "event",
"stalled", "jobId", jobId)
table.insert(stalled, jobId)
end
end
end
end
end
end
-- Mark potentially stalled jobs
local active = rcall('LRANGE', activeKey, 0, -1)
if (#active > 0) then
for from, to in batches(#active, 7000) do
rcall('SADD', stalledKey, unpack(active, from, to))
end
end
return {failed, stalled}
end
+47
View File
@@ -0,0 +1,47 @@
--[[
Function to clean job list.
Returns jobIds and deleted count number.
]]
-- Includes
--- @include "getTimestamp"
--- @include "removeJob"
local function cleanList(listKey, jobKeyPrefix, rangeStart, rangeEnd,
timestamp, isWaiting)
local jobs = rcall("LRANGE", listKey, rangeStart, rangeEnd)
local deleted = {}
local deletedCount = 0
local jobTS
local deletionMarker = ''
local jobIdsLen = #jobs
for i, job in ipairs(jobs) do
if limit > 0 and deletedCount >= limit then
break
end
local jobKey = jobKeyPrefix .. job
if (isWaiting or rcall("EXISTS", jobKey .. ":lock") == 0) then
-- Find the right timestamp of the job to compare to maxTimestamp:
-- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed
-- * processedOn represents when the job was last attempted, but it doesn't get populated until
-- the job is first tried
-- * timestamp is the original job submission time
-- Fetch all three of these (in that order) and use the first one that is set so that we'll leave jobs
-- that have been active within the grace period:
jobTS = getTimestamp(jobKey, {"finishedOn", "processedOn", "timestamp"})
if (not jobTS or jobTS <= timestamp) then
-- replace the entry with a deletion marker; the actual deletion will
-- occur at the end of the script
rcall("LSET", listKey, rangeEnd - jobIdsLen + i, deletionMarker)
removeJob(job, true, jobKeyPrefix)
deletedCount = deletedCount + 1
table.insert(deleted, job)
end
end
end
rcall("LREM", listKey, 0, deletionMarker)
return {deleted, deletedCount}
end
+45
View File
@@ -0,0 +1,45 @@
--[[
Function to clean job set.
Returns jobIds and deleted count number.
]]
-- Includes
--- @include "batches"
--- @include "getJobsInZset"
--- @include "getTimestamp"
--- @include "removeJob"
local function cleanSet(setKey, jobKeyPrefix, rangeEnd, timestamp, limit, attributes, isFinished)
local jobs = getJobsInZset(setKey, rangeEnd, limit)
local deleted = {}
local deletedCount = 0
local jobTS
for i, job in ipairs(jobs) do
if limit > 0 and deletedCount >= limit then
break
end
local jobKey = jobKeyPrefix .. job
if isFinished then
removeJob(job, true, jobKeyPrefix)
deletedCount = deletedCount + 1
table.insert(deleted, job)
else
-- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed
jobTS = getTimestamp(jobKey, attributes)
if (not jobTS or jobTS <= timestamp) then
removeJob(job, true, jobKeyPrefix)
deletedCount = deletedCount + 1
table.insert(deleted, job)
end
end
end
if(#deleted > 0) then
for from, to in batches(#deleted, 7000) do
rcall("ZREM", setKey, unpack(deleted, from, to))
end
end
return {deleted, deletedCount}
end
+46
View File
@@ -0,0 +1,46 @@
--[[
Functions to collect metrics based on a current and previous count of jobs.
Granualarity is fixed at 1 minute.
]]
--- @include "batches"
local function collectMetrics(metaKey, dataPointsList, maxDataPoints,
timestamp)
-- Increment current count
local count = rcall("HINCRBY", metaKey, "count", 1) - 1
-- Compute how many data points we need to add to the list, N.
local prevTS = rcall("HGET", metaKey, "prevTS")
if not prevTS then
-- If prevTS is nil, set it to the current timestamp
rcall("HSET", metaKey, "prevTS", timestamp, "prevCount", 0)
return
end
local N = math.floor((timestamp - prevTS) / 60000)
if N > 0 then
local delta = count - rcall("HGET", metaKey, "prevCount")
-- If N > 1, add N-1 zeros to the list
if N > 1 then
local points = {}
points[1] = delta
for i = 2, N do
points[i] = 0
end
for from, to in batches(#points, 7000) do
rcall("LPUSH", dataPointsList, unpack(points, from, to))
end
else
-- LPUSH delta to the list
rcall("LPUSH", dataPointsList, delta)
end
-- LTRIM to keep list to its max size
rcall("LTRIM", dataPointsList, 0, maxDataPoints - 1)
-- update prev count with current count
rcall("HSET", metaKey, "prevCount", count, "prevTS", timestamp)
end
end
+12
View File
@@ -0,0 +1,12 @@
--[[
Functions to destructure job key.
Just a bit of warning, these functions may be a bit slow and affect performance significantly.
]]
local getJobIdFromKey = function (jobKey)
return string.match(jobKey, ".*:(.*)")
end
local getJobKeyPrefix = function (jobKey, jobId)
return string.sub(jobKey, 0, #jobKey - #jobId)
end
+70
View File
@@ -0,0 +1,70 @@
--[[
Function to achieve pagination for a set or hash.
This function simulates pagination in the most efficient way possible
for a set using sscan or hscan.
The main limitation is that sets are not order preserving, so the
pagination is not stable. This means that if the set is modified
between pages, the same element may appear in different pages.
]] -- Maximum number of elements to be returned by sscan per iteration.
local maxCount = 100
-- Finds the cursor, and returns the first elements available for the requested page.
local function findPage(key, command, pageStart, pageSize, cursor, offset,
maxIterations, fetchJobs)
local items = {}
local jobs = {}
local iterations = 0
repeat
-- Iterate over the set using sscan/hscan.
local result = rcall(command, key, cursor, "COUNT", maxCount)
cursor = result[1]
local members = result[2]
local step = 1
if command == "HSCAN" then
step = 2
end
if #members == 0 then
-- If the result is empty, we can return the result.
return cursor, offset, items, jobs
end
local chunkStart = offset
local chunkEnd = offset + #members / step
local pageEnd = pageStart + pageSize
if chunkEnd < pageStart then
-- If the chunk is before the page, we can skip it.
offset = chunkEnd
elseif chunkStart > pageEnd then
-- If the chunk is after the page, we can return the result.
return cursor, offset, items, jobs
else
-- If the chunk is overlapping the page, we need to add the elements to the result.
for i = 1, #members, step do
if offset >= pageEnd then
return cursor, offset, items, jobs
end
if offset >= pageStart then
local index = #items + 1
if fetchJobs ~= nil then
jobs[#jobs+1] = rcall("HGETALL", members[i])
end
if step == 2 then
items[index] = {members[i], members[i + 1]}
else
items[index] = members[i]
end
end
offset = offset + 1
end
end
iterations = iterations + 1
until cursor == "0" or iterations >= maxIterations
return cursor, offset, items, jobs
end
+11
View File
@@ -0,0 +1,11 @@
-- We use ZRANGEBYSCORE to make the case where we're deleting a limited number
-- of items in a sorted set only run a single iteration. If we simply used
-- ZRANGE, we may take a long time traversing through jobs that are within the
-- grace period.
local function getJobsInZset(zsetKey, rangeEnd, limit)
if limit > 0 then
return rcall("ZRANGEBYSCORE", zsetKey, 0, rangeEnd, "LIMIT", 0, limit)
else
return rcall("ZRANGEBYSCORE", zsetKey, 0, rangeEnd)
end
end
@@ -0,0 +1,13 @@
--[[
Function to return the next delayed job timestamp.
]]
local function getNextDelayedTimestamp(delayedKey)
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
if #result then
local nextTimestamp = tonumber(result[2])
if (nextTimestamp ~= nil) then
nextTimestamp = nextTimestamp / 0x1000
end
return nextTimestamp
end
end
+9
View File
@@ -0,0 +1,9 @@
local function getOrSetMaxEvents(metaKey)
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
if not maxEvents then
maxEvents = 10000
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
end
return maxEvents
end
+7
View File
@@ -0,0 +1,7 @@
--[[
Function to get priority score.
]]
local function getPriorityScore(priority, priorityCounterKey)
local prioCounter = rcall("INCR", priorityCounterKey)
return priority * 0x100000000 + prioCounter % 0x100000000
end
+14
View File
@@ -0,0 +1,14 @@
local function getRateLimitTTL(maxJobs, rateLimiterKey)
if maxJobs and maxJobs <= tonumber(rcall("GET", rateLimiterKey) or 0) then
local pttl = rcall("PTTL", rateLimiterKey)
if pttl == 0 then
rcall("DEL", rateLimiterKey)
end
if pttl > 0 then
return pttl
end
end
return 0
end
+12
View File
@@ -0,0 +1,12 @@
--[[
Function to check for the meta.paused key to decide if we are paused or not
(since an empty list and !EXISTS are not really the same).
]]
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
return waitKey, false
else
return pausedKey, true
end
end
+19
View File
@@ -0,0 +1,19 @@
--[[
Function to get the latest saved timestamp.
]]
local function getTimestamp(jobKey, attributes)
if #attributes == 1 then
return rcall("HGET", jobKey, attributes[1])
end
local jobTs
for _, ts in ipairs(rcall("HMGET", jobKey, unpack(attributes))) do
if (ts) then
jobTs = ts
break
end
end
return jobTs
end
+7
View File
@@ -0,0 +1,7 @@
--[[
Function to get ZSet items.
]]
local function getZSetItems(keyName, max)
return rcall('ZRANGE', keyName, 0, max - 1)
end
+33
View File
@@ -0,0 +1,33 @@
--[[
Function to recursively check if there are no locks
on the jobs to be removed.
returns:
boolean
]]
local function isLocked( prefix, jobId, removeChildren)
local jobKey = prefix .. jobId;
-- Check if this job is locked
local lockKey = jobKey .. ':lock'
local lock = rcall("GET", lockKey)
if not lock then
if removeChildren == "1" then
local dependencies = rcall("SMEMBERS", jobKey .. ":dependencies")
if (#dependencies > 0) then
for i, childJobKey in ipairs(dependencies) do
-- We need to get the jobId for this job.
local childJobId = getJobIdFromKey(childJobKey)
local childJobPrefix = getJobKeyPrefix(childJobKey, childJobId)
local result = isLocked( childJobPrefix, childJobId, removeChildren )
if result then
return true
end
end
end
end
return false
end
return true
end
@@ -0,0 +1,13 @@
--[[
Function to move job from prioritized state to active.
]]
local function moveJobFromPriorityToActive(priorityKey, activeKey, priorityCounterKey)
local prioritizedJob = rcall("ZPOPMIN", priorityKey)
if #prioritizedJob > 0 then
rcall("LPUSH", activeKey, prioritizedJob[1])
return prioritizedJob[1]
else
rcall("DEL", priorityCounterKey)
end
end
@@ -0,0 +1,38 @@
--[[
Function to recursively move from waitingChildren to failed.
]]
-- Includes
--- @include "moveParentToWaitIfNeeded"
local function moveParentFromWaitingChildrenToFailed( parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
if rcall("ZREM", parentQueueKey .. ":waiting-children", parentId) == 1 then
rcall("ZADD", parentQueueKey .. ":failed", timestamp, parentId)
local failedReason = "child " .. jobIdKey .. " failed"
rcall("HMSET", parentKey, "failedReason", failedReason, "finishedOn", timestamp)
rcall("XADD", parentQueueKey .. ":events", "*", "event", "failed", "jobId", parentId, "failedReason",
failedReason, "prev", "waiting-children")
local rawParentData = rcall("HGET", parentKey, "parent")
if rawParentData ~= false then
local parentData = cjson.decode(rawParentData)
if parentData['fpof'] then
moveParentFromWaitingChildrenToFailed(
parentData['queueKey'],
parentData['queueKey'] .. ':' .. parentData['id'],
parentData['id'],
parentKey,
timestamp
)
elseif parentData['rdof'] then
local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
local grandParentDependenciesSet = grandParentKey .. ":dependencies"
if rcall("SREM", grandParentDependenciesSet, parentKey) == 1 then
moveParentToWaitIfNeeded(parentData['queueKey'], grandParentDependenciesSet,
grandParentKey, parentData['id'], timestamp)
end
end
end
end
end
@@ -0,0 +1,42 @@
--[[
Validate and move parent to active if needed.
]]
-- Includes
--- @include "addDelayMarkerIfNeeded"
--- @include "addJobWithPriority"
--- @include "getTargetQueueList"
local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
local isParentActive = rcall("ZSCORE", parentQueueKey .. ":waiting-children", parentId)
if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then
rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
local parentWaitKey = parentQueueKey .. ":wait"
local parentTarget, paused = getTargetQueueList(parentQueueKey .. ":meta", parentWaitKey,
parentQueueKey .. ":paused")
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
local priority = tonumber(jobAttributes[1]) or 0
local delay = tonumber(jobAttributes[2]) or 0
if delay > 0 then
local delayedTimestamp = tonumber(timestamp) + delay
local score = delayedTimestamp * 0x1000
local parentDelayedKey = parentQueueKey .. ":delayed"
rcall("ZADD", parentDelayedKey, score, parentId)
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId,
"delay", delayedTimestamp)
addDelayMarkerIfNeeded(parentTarget, parentDelayedKey)
else
if priority == 0 then
rcall("RPUSH", parentTarget, parentId)
else
addJobWithPriority(parentWaitKey, parentQueueKey .. ":prioritized", priority, paused,
parentId, parentQueueKey .. ":pc")
end
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId,
"prev", "waiting-children")
end
end
end
@@ -0,0 +1,73 @@
--[[
Function to move job from wait state to active.
Input:
keys[1] wait key
keys[2] active key
keys[3] prioritized key
keys[4] stream events key
keys[5] stalled key
-- Rate limiting
keys[6] rate limiter key
keys[7] delayed key
keys[8] paused key
keys[9] meta key
keys[10] pc priority counter
opts - token - lock token
opts - lockDuration
opts - limiter
]]
-- Includes
--- @include "pushBackJobWithPriority"
local function prepareJobForProcessing(keys, keyPrefix, targetKey, jobId, processedOn,
maxJobs, expireTime, opts)
local jobKey = keyPrefix .. jobId
-- Check if we need to perform rate limiting.
if maxJobs then
local rateLimiterKey = keys[6];
-- check if we exceeded rate limit, we need to remove the job and return expireTime
if expireTime > 0 then
-- remove from active queue and add back to the wait list
rcall("LREM", keys[2], 1, jobId)
local priority = tonumber(rcall("HGET", jobKey, "priority")) or 0
if priority == 0 then
rcall("RPUSH", targetKey, jobId)
else
pushBackJobWithPriority(keys[3], priority, jobId)
end
-- Return when we can process more jobs
return {0, 0, expireTime, 0}
end
local jobCounter = tonumber(rcall("INCR", rateLimiterKey))
if jobCounter == 1 then
local limiterDuration = opts['limiter'] and opts['limiter']['duration']
local integerDuration = math.floor(math.abs(limiterDuration))
rcall("PEXPIRE", rateLimiterKey, integerDuration)
end
end
local lockKey = jobKey .. ':lock'
-- get a lock
if opts['token'] ~= "0" then
rcall("SET", lockKey, opts['token'], "PX", opts['lockDuration'])
end
rcall("XADD", keys[4], "*", "event", "active", "jobId", jobId, "prev", "waiting")
rcall("HSET", jobKey, "processedOn", processedOn)
rcall("HINCRBY", jobKey, "attemptsMade", 1)
return {rcall("HGETALL", jobKey), jobId, 0, 0} -- get job data
end
+44
View File
@@ -0,0 +1,44 @@
--[[
Updates the delay set, by moving delayed jobs that should
be processed now to "wait".
Events:
'waiting'
]]
-- Includes
--- @include "addPriorityMarkerIfNeeded"
--- @include "getPriorityScore"
-- Try to get as much as 1000 jobs at once
local function promoteDelayedJobs(delayedKey, waitKey, targetKey, prioritizedKey,
eventStreamKey, prefix, timestamp, paused, priorityCounterKey)
local jobs = rcall("ZRANGEBYSCORE", delayedKey, 0, (timestamp + 1) * 0x1000, "LIMIT", 0, 1000)
if (#jobs > 0) then
rcall("ZREM", delayedKey, unpack(jobs))
for _, jobId in ipairs(jobs) do
local jobKey = prefix .. jobId
local priority =
tonumber(rcall("HGET", jobKey, "priority")) or 0
if priority == 0 then
-- LIFO or FIFO
rcall("LPUSH", targetKey, jobId)
else
local score = getPriorityScore(priority, priorityCounterKey)
rcall("ZADD", prioritizedKey, score, jobId)
end
-- Emit waiting event
rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId",
jobId, "prev", "delayed")
rcall("HSET", jobKey, "delay", 0)
end
if not paused then
addPriorityMarkerIfNeeded(targetKey)
end
end
end
@@ -0,0 +1,9 @@
--[[
Function to push back job considering priority in front of same prioritized jobs.
]]
local function pushBackJobWithPriority(prioritizedKey, priority, jobId)
-- in order to put it at front of same prioritized jobs
-- we consider prioritized counter as 0
local score = priority * 0x100000000
rcall("ZADD", prioritizedKey, score, jobId)
end
+13
View File
@@ -0,0 +1,13 @@
--[[
Function to remove job.
]]
-- Includes
--- @include "removeParentDependencyKey"
local function removeJob(jobId, hard, baseKey)
local jobKey = baseKey .. jobId
removeParentDependencyKey(jobKey, hard, nil, baseKey)
rcall("DEL", jobKey, jobKey .. ':logs',
jobKey .. ':dependencies', jobKey .. ':processed')
end
@@ -0,0 +1,35 @@
--[[
Function to remove from any state.
returns:
prev state
]]
local function removeJobFromAnyState( prefix, jobId)
-- We start with the ZSCORE checks, since they have O(1) complexity
if rcall("ZSCORE", prefix .. "completed", jobId) then
rcall("ZREM", prefix .. "completed", jobId)
return "completed"
elseif rcall("ZSCORE", prefix .. "waiting-children", jobId) then
rcall("ZREM", prefix .. "waiting-children", jobId)
return "waiting-children"
elseif rcall("ZSCORE", prefix .. "delayed", jobId) then
rcall("ZREM", prefix .. "delayed", jobId)
return "delayed"
elseif rcall("ZSCORE", prefix .. "failed", jobId) then
rcall("ZREM", prefix .. "failed", jobId)
return "failed"
elseif rcall("ZSCORE", prefix .. "prioritized", jobId) then
rcall("ZREM", prefix .. "prioritized", jobId)
return "prioritized"
-- We remove only 1 element from the list, since we assume they are not added multiple times
elseif rcall("LREM", prefix .. "wait", 1, jobId) == 1 then
return "wait"
elseif rcall("LREM", prefix .. "paused", 1, jobId) == 1 then
return "paused"
elseif rcall("LREM", prefix .. "active", 1, jobId) == 1 then
return "active"
end
return "unknown"
end
+13
View File
@@ -0,0 +1,13 @@
--[[
Functions to remove jobs.
]]
-- Includes
--- @include "removeJob"
local function removeJobs(keys, hard, baseKey, max)
for i, key in ipairs(keys) do
removeJob(key, hard, baseKey)
end
return max - #keys
end
+15
View File
@@ -0,0 +1,15 @@
--[[
Functions to remove jobs by max age.
]]
-- Includes
--- @include "removeJob"
local function removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
local start = timestamp - maxAge * 1000
local jobIds = rcall("ZREVRANGEBYSCORE", targetSet, start, "-inf")
for i, jobId in ipairs(jobIds) do
removeJob(jobId, false, prefix)
end
rcall("ZREMRANGEBYSCORE", targetSet, "-inf", start)
end
@@ -0,0 +1,15 @@
--[[
Functions to remove jobs by max count.
]]
-- Includes
--- @include "removeJob"
local function removeJobsByMaxCount(maxCount, targetSet, prefix)
local start = maxCount
local jobIds = rcall("ZREVRANGE", targetSet, start, -1)
for i, jobId in ipairs(jobIds) do
removeJob(jobId, false, prefix)
end
rcall("ZREMRANGEBYRANK", targetSet, 0, -(maxCount + 1))
end
+17
View File
@@ -0,0 +1,17 @@
--[[
Functions to remove jobs.
]]
-- Includes
--- @include "removeJobs"
local function getListItems(keyName, max)
return rcall('LRANGE', keyName, 0, max - 1)
end
local function removeListJobs(keyName, hard, baseKey, max)
local jobs = getListItems(keyName, max)
local count = removeJobs(jobs, hard, baseKey, max)
rcall("LTRIM", keyName, #jobs, -1)
return count
end
@@ -0,0 +1,77 @@
--[[
Check if this job has a parent. If so we will just remove it from
the parent child list, but if it is the last child we should move the parent to "wait/paused"
which requires code from "moveToFinished"
]]
--- @include "destructureJobKey"
--- @include "getTargetQueueList"
local function moveParentToWait(parentPrefix, parentId, emitEvent)
local parentTarget = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "wait", parentPrefix .. "paused")
rcall("RPUSH", parentTarget, parentId)
if emitEvent then
local parentEventStream = parentPrefix .. "events"
rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
end
end
local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey)
if parentKey then
local parentDependenciesKey = parentKey .. ":dependencies"
local result = rcall("SREM", parentDependenciesKey, jobKey)
if result > 0 then
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
if pendingDependencies == 0 then
local parentId = getJobIdFromKey(parentKey)
local parentPrefix = getJobKeyPrefix(parentKey, parentId)
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
if numRemovedElements == 1 then
if hard then
if parentPrefix == baseKey then
removeParentDependencyKey(parentKey, hard, nil, baseKey)
rcall("DEL", parentKey, parentKey .. ':logs',
parentKey .. ':dependencies', parentKey .. ':processed')
else
moveParentToWait(parentPrefix, parentId)
end
else
moveParentToWait(parentPrefix, parentId, true)
end
end
end
end
else
local missedParentKey = rcall("HGET", jobKey, "parentKey")
if( (type(missedParentKey) == "string") and missedParentKey ~= "" and (rcall("EXISTS", missedParentKey) == 1)) then
local parentDependenciesKey = missedParentKey .. ":dependencies"
local result = rcall("SREM", parentDependenciesKey, jobKey)
if result > 0 then
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
if pendingDependencies == 0 then
local parentId = getJobIdFromKey(missedParentKey)
local parentPrefix = getJobKeyPrefix(missedParentKey, parentId)
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
if numRemovedElements == 1 then
if hard then
if parentPrefix == baseKey then
removeParentDependencyKey(missedParentKey, hard, nil, baseKey)
rcall("DEL", missedParentKey, missedParentKey .. ':logs',
missedParentKey .. ':dependencies', missedParentKey .. ':processed')
else
moveParentToWait(parentPrefix, parentId)
end
else
moveParentToWait(parentPrefix, parentId, true)
end
end
end
end
end
end
end
+15
View File
@@ -0,0 +1,15 @@
-- Includes
--- @include "batches"
--- @include "getZSetItems"
--- @include "removeJobs"
local function removeZSetJobs(keyName, hard, baseKey, max)
local jobs = getZSetItems(keyName, max)
local count = removeJobs(jobs, hard, baseKey, max)
if(#jobs > 0) then
for from, to in batches(#jobs, 7000) do
rcall("ZREM", keyName, unpack(jobs, from, to))
end
end
return count
end
+30
View File
@@ -0,0 +1,30 @@
--[[
Function to store a job
]]
local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
parentKey, parentData, repeatJobKey)
local jsonOpts = cjson.encode(opts)
local delay = opts['delay'] or 0
local priority = opts['priority'] or 0
local optionalValues = {}
if parentKey ~= nil then
table.insert(optionalValues, "parentKey")
table.insert(optionalValues, parentKey)
table.insert(optionalValues, "parent")
table.insert(optionalValues, parentData)
end
if repeatJobKey ~= nil then
table.insert(optionalValues, "rjk")
table.insert(optionalValues, repeatJobKey)
end
rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
"timestamp", timestamp, "delay", delay, "priority", priority,
unpack(optionalValues))
rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
return delay, priority
end
+12
View File
@@ -0,0 +1,12 @@
--[[
Function to trim events, default 10000.
]]
local function trimEvents(metaKey, eventStreamKey)
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
if maxEvents ~= false then
rcall("XTRIM", eventStreamKey, "MAXLEN", "~", maxEvents)
else
rcall("XTRIM", eventStreamKey, "MAXLEN", "~", 10000)
end
end
@@ -0,0 +1,25 @@
--- @include "updateParentDepsIfNeeded"
--[[
This function is used to update the parent's dependencies if the job
is already completed and about to be ignored. The parent must get its
dependencies updated to avoid the parent job being stuck forever in
the waiting-children state.
]]
local function updateExistingJobsParent(parentKey, parent, parentData,
parentDependenciesKey, completedKey,
jobIdKey, jobId, timestamp)
if parentKey ~= nil then
if rcall("ZSCORE", completedKey, jobId) ~= false then
local returnvalue = rcall("HGET", jobIdKey, "returnvalue")
updateParentDepsIfNeeded(parentKey, parent['queueKey'],
parentDependenciesKey, parent['id'],
jobIdKey, returnvalue, timestamp)
else
if parentDependenciesKey ~= nil then
rcall("SADD", parentDependenciesKey, jobIdKey)
end
end
rcall("HMSET", jobIdKey, "parentKey", parentKey, "parent", parentData)
end
end
@@ -0,0 +1,13 @@
--[[
Validate and move or add dependencies to parent.
]]
-- Includes
--- @include "moveParentToWaitIfNeeded"
local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
parentId, jobIdKey, returnvalue, timestamp )
local processedSet = parentKey .. ":processed"
rcall("HSET", processedSet, jobIdKey, returnvalue)
moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
end
+10
View File
@@ -0,0 +1,10 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.scriptLoader = exports.ScriptLoader = exports.ScriptLoaderError = void 0;
const script_loader_1 = require("./script-loader");
Object.defineProperty(exports, "ScriptLoader", { enumerable: true, get: function () { return script_loader_1.ScriptLoader; } });
var script_loader_2 = require("./script-loader");
Object.defineProperty(exports, "ScriptLoaderError", { enumerable: true, get: function () { return script_loader_2.ScriptLoaderError; } });
const scriptLoader = new script_loader_1.ScriptLoader();
exports.scriptLoader = scriptLoader;
//# sourceMappingURL=index.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/commands/index.ts"],"names":[],"mappings":";;;AAAA,mDAA+C;AAKtC,6FALA,4BAAY,OAKA;AAJrB,iDAA6E;AAA3C,kHAAA,iBAAiB,OAAA;AAEnD,MAAM,YAAY,GAAG,IAAI,4BAAY,EAAE,CAAC;AAEjB,oCAAY"}
+48
View File
@@ -0,0 +1,48 @@
--[[
Checks if a job is finished (.i.e. is in the completed or failed set)
Input:
KEYS[1] completed key
KEYS[2] failed key
KEYS[3] job key
ARGV[1] job id
ARGV[2] return value?
Output:
0 - Not finished.
1 - Completed.
2 - Failed.
-1 - Missing job.
]]
local rcall = redis.call
if rcall("EXISTS", KEYS[3]) ~= 1 then
if ARGV[2] == "1" then
return {-1,"Missing key for job " .. KEYS[3] .. ". isFinished"}
end
return -1
end
if rcall("ZSCORE", KEYS[1], ARGV[1]) ~= false then
if ARGV[2] == "1" then
local returnValue = rcall("HGET", KEYS[3], "returnvalue")
return {1,returnValue}
end
return 1
end
if rcall("ZSCORE", KEYS[2], ARGV[1]) ~= false then
if ARGV[2] == "1" then
local failedReason = rcall("HGET", KEYS[3], "failedReason")
return {2,failedReason}
end
return 2
end
if ARGV[2] == "1" then
return {0}
end
return 0
+16
View File
@@ -0,0 +1,16 @@
--[[
Checks if job is in a given list.
Input:
KEYS[1]
ARGV[1]
Output:
1 if element found in the list.
]]
-- Includes
--- @include "includes/checkItemInList"
local items = redis.call("LRANGE", KEYS[1] , 0, -1)
return checkItemInList(items, ARGV[1])
+53
View File
@@ -0,0 +1,53 @@
--[[
Function to move job from active state to wait.
Input:
KEYS[1] active key
KEYS[2] wait key
KEYS[3] stalled key
KEYS[4] job lock key
KEYS[5] paused key
KEYS[6] meta key
KEYS[7] limiter key
KEYS[8] prioritized key
KEYS[9] event key
ARGV[1] job id
ARGV[2] lock token
ARGV[3] job id key
]]
local rcall = redis.call
-- Includes
--- @include "includes/pushBackJobWithPriority"
--- @include "includes/getTargetQueueList"
local jobId = ARGV[1]
local token = ARGV[2]
local lockKey = KEYS[4]
local lockToken = rcall("GET", lockKey)
local pttl = rcall("PTTL", KEYS[7])
if lockToken == token and pttl > 0 then
local removed = rcall("LREM", KEYS[1], 1, jobId)
if (removed > 0) then
local target = getTargetQueueList(KEYS[6], KEYS[2], KEYS[5])
rcall("SREM", KEYS[3], jobId)
local priority = tonumber(rcall("HGET", ARGV[3], "priority")) or 0
if priority > 0 then
pushBackJobWithPriority(KEYS[8], priority, jobId)
else
rcall("RPUSH", target, jobId)
end
rcall("DEL", lockKey)
-- Emit waiting event
rcall("XADD", KEYS[9], "*", "event", "waiting", "jobId", jobId)
end
end
return pttl
+67
View File
@@ -0,0 +1,67 @@
--[[
Move completed, failed or delayed jobs to wait.
Note: Does not support jobs with priorities.
Input:
KEYS[1] base key
KEYS[2] events stream
KEYS[3] state key (failed, completed, delayed)
KEYS[4] 'wait'
KEYS[5] 'paused'
KEYS[6] 'meta'
ARGV[1] count
ARGV[2] timestamp
ARGV[3] prev state
Output:
1 means the operation is not completed
0 means the operation is completed
]]
local maxCount = tonumber(ARGV[1])
local timestamp = tonumber(ARGV[2])
local rcall = redis.call;
-- Includes
--- @include "includes/batches"
--- @include "includes/getTargetQueueList"
local metaKey = KEYS[6]
local target = getTargetQueueList(metaKey, KEYS[4], KEYS[5])
local jobs = rcall('ZRANGEBYSCORE', KEYS[3], 0, timestamp, 'LIMIT', 0, maxCount)
if (#jobs > 0) then
if ARGV[3] == "failed" then
for i, key in ipairs(jobs) do
local jobKey = KEYS[1] .. key
rcall("HDEL", jobKey, "finishedOn", "processedOn", "failedReason")
end
elseif ARGV[3] == "completed" then
for i, key in ipairs(jobs) do
local jobKey = KEYS[1] .. key
rcall("HDEL", jobKey, "finishedOn", "processedOn", "returnvalue")
end
end
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents") or 10000
for i, key in ipairs(jobs) do
-- Emit waiting event
rcall("XADD", KEYS[2], "MAXLEN", "~", maxEvents, "*", "event",
"waiting", "jobId", key, "prev", ARGV[3]);
end
for from, to in batches(#jobs, 7000) do
rcall("ZREM", KEYS[3], unpack(jobs, from, to))
rcall("LPUSH", target, unpack(jobs, from, to))
end
end
maxCount = maxCount - #jobs
if (maxCount <= 0) then return 1 end
return 0
+24
View File
@@ -0,0 +1,24 @@
--[[
Move stalled jobs to wait.
Input:
KEYS[1] 'stalled' (SET)
KEYS[2] 'wait', (LIST)
KEYS[3] 'active', (LIST)
KEYS[4] 'failed', (ZSET)
KEYS[5] 'stalled-check', (KEY)
KEYS[6] 'meta', (KEY)
KEYS[7] 'paused', (LIST)
KEYS[8] 'event stream' (STREAM)
ARGV[1] Max stalled job count
ARGV[2] queue.toKey('')
ARGV[3] timestamp
ARGV[4] max check time
Events:
'stalled' with stalled job id.
]] -- Includes
--- @include "includes/checkStalledJobs"
return checkStalledJobs(KEYS[1], KEYS[2], KEYS[3], KEYS[4], KEYS[5], KEYS[6],
KEYS[7], KEYS[8], ARGV[1], ARGV[2], ARGV[3], ARGV[4])
+103
View File
@@ -0,0 +1,103 @@
--[[
Move next job to be processed to active, lock it and fetch its data. The job
may be delayed, in that case we need to move it to the delayed set instead.
This operation guarantees that the worker owns the job during the lock
expiration time. The worker is responsible of keeping the lock fresh
so that no other worker picks this job again.
Input:
KEYS[1] wait key
KEYS[2] active key
KEYS[3] prioritized key
KEYS[4] stream events key
KEYS[5] stalled key
-- Rate limiting
KEYS[6] rate limiter key
KEYS[7] delayed key
-- Promote delayed jobs
KEYS[8] paused key
KEYS[9] meta key
KEYS[10] pc priority counter
-- Arguments
ARGV[1] key prefix
ARGV[2] timestamp
ARGV[3] optional job ID
ARGV[4] opts
opts - token - lock token
opts - lockDuration
opts - limiter
]]
local rcall = redis.call
local waitKey = KEYS[1]
local activeKey = KEYS[2]
local rateLimiterKey = KEYS[6]
local delayedKey = KEYS[7]
local opts = cmsgpack.unpack(ARGV[4])
-- Includes
--- @include "includes/getNextDelayedTimestamp"
--- @include "includes/getRateLimitTTL"
--- @include "includes/getTargetQueueList"
--- @include "includes/moveJobFromPriorityToActive"
--- @include "includes/prepareJobForProcessing"
--- @include "includes/promoteDelayedJobs"
local target, paused = getTargetQueueList(KEYS[9], waitKey, KEYS[8])
-- Check if there are delayed jobs that we can move to wait.
promoteDelayedJobs(delayedKey, waitKey, target, KEYS[3], KEYS[4], ARGV[1],
ARGV[2], paused, KEYS[10])
local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
local expireTime = getRateLimitTTL(maxJobs, rateLimiterKey)
local jobId = nil
if ARGV[3] ~= "" then
jobId = ARGV[3]
-- clean stalled key
rcall("SREM", KEYS[5], jobId)
end
if not jobId or (jobId and string.sub(jobId, 1, 2) == "0:") then
-- If jobId is special ID 0:delay, then there is no job to process
if jobId then rcall("LREM", activeKey, 1, jobId) end
-- Check if we are rate limited first.
if expireTime > 0 then return {0, 0, expireTime, 0} end
-- paused queue
if paused then return {0, 0, 0, 0} end
-- no job ID, try non-blocking move from wait to active
jobId = rcall("RPOPLPUSH", waitKey, activeKey)
-- Since it is possible that between a call to BRPOPLPUSH and moveToActive
-- another script puts a new maker in wait, we need to check again.
if jobId and string.sub(jobId, 1, 2) == "0:" then
rcall("LREM", activeKey, 1, jobId)
jobId = rcall("RPOPLPUSH", waitKey, activeKey)
end
end
if jobId then
return prepareJobForProcessing(KEYS, ARGV[1], target, jobId, ARGV[2],
maxJobs, expireTime, opts)
else
jobId = moveJobFromPriorityToActive(KEYS[3], activeKey, KEYS[10])
if jobId then
return prepareJobForProcessing(KEYS, ARGV[1], target, jobId, ARGV[2],
maxJobs, expireTime, opts)
end
end
-- Return the timestamp for the next delayed job if any.
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
if (nextTimestamp ~= nil) then return {0, 0, 0, nextTimestamp} end
return {0, 0, 0, 0}
+72
View File
@@ -0,0 +1,72 @@
--[[
Moves job from active to delayed set.
Input:
KEYS[1] wait key
KEYS[2] active key
KEYS[3] prioritized key
KEYS[4] delayed key
KEYS[5] job key
KEYS[6] events stream
KEYS[7] paused key
KEYS[8] meta key
ARGV[1] key prefix
ARGV[2] timestamp
ARGV[3] delayedTimestamp
ARGV[4] the id of the job
ARGV[5] queue token
ARGV[6] delay value
Output:
0 - OK
-1 - Missing job.
-3 - Job not in active set.
Events:
- delayed key.
]]
local rcall = redis.call
-- Includes
--- @include "includes/addDelayMarkerIfNeeded"
--- @include "includes/getTargetQueueList"
--- @include "includes/promoteDelayedJobs"
local jobKey = KEYS[5]
if rcall("EXISTS", jobKey) == 1 then
local delayedKey = KEYS[4]
if ARGV[5] ~= "0" then
local lockKey = jobKey .. ':lock'
if rcall("GET", lockKey) == ARGV[5] then
rcall("DEL", lockKey)
else
return -2
end
end
local jobId = ARGV[4]
local score = tonumber(ARGV[3])
local delayedTimestamp = (score / 0x1000)
local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
if numRemovedElements < 1 then
return -3
end
rcall("HSET", jobKey, "delay", ARGV[6])
local maxEvents = rcall("HGET", KEYS[8], "opts.maxLenEvents") or 10000
rcall("ZADD", delayedKey, score, jobId)
rcall("XADD", KEYS[6], "MAXLEN", "~", maxEvents, "*", "event", "delayed",
"jobId", jobId, "delay", delayedTimestamp)
-- Check if we need to push a marker job to wake up sleeping workers.
local target = getTargetQueueList(KEYS[8], KEYS[1], KEYS[7])
addDelayMarkerIfNeeded(target, delayedKey)
return 0
else
return -1
end
+267
View File
@@ -0,0 +1,267 @@
--[[
Move job from active to a finished status (completed o failed)
A job can only be moved to completed if it was active.
The job must be locked before it can be moved to a finished status,
and the lock must be released in this script.
Input:
KEYS[1] wait key
KEYS[2] active key
KEYS[3] prioritized key
KEYS[4] event stream key
KEYS[5] stalled key
-- Rate limiting
KEYS[6] rate limiter key
KEYS[7] delayed key
KEYS[8] paused key
KEYS[9] meta key
KEYS[10] pc priority counter
KEYS[11] completed/failed key
KEYS[12] jobId key
KEYS[13] metrics key
ARGV[1] jobId
ARGV[2] timestamp
ARGV[3] msg property returnvalue / failedReason
ARGV[4] return value / failed reason
ARGV[5] target (completed/failed)
ARGV[6] event data (? maybe just send jobid).
ARGV[7] fetch next?
ARGV[8] keys prefix
ARGV[9] opts
opts - token - lock token
opts - keepJobs
opts - lockDuration - lock duration in milliseconds
opts - attempts max attempts
opts - attemptsMade
opts - maxMetricsSize
opts - fpof - fail parent on fail
opts - rdof - remove dependency on fail
Output:
0 OK
-1 Missing key.
-2 Missing lock.
-3 Job not in active set
-4 Job has pending dependencies
-6 Lock is not owned by this client
Events:
'completed/failed'
]]
local rcall = redis.call
--- Includes
--- @include "includes/collectMetrics"
--- @include "includes/getNextDelayedTimestamp"
--- @include "includes/moveJobFromPriorityToActive"
--- @include "includes/prepareJobForProcessing"
--- @include "includes/moveParentFromWaitingChildrenToFailed"
--- @include "includes/moveParentToWaitIfNeeded"
--- @include "includes/promoteDelayedJobs"
--- @include "includes/removeJobsByMaxAge"
--- @include "includes/removeJobsByMaxCount"
--- @include "includes/removeParentDependencyKey"
--- @include "includes/trimEvents"
--- @include "includes/updateParentDepsIfNeeded"
--- @include "includes/getRateLimitTTL"
local jobIdKey = KEYS[12]
if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
local opts = cmsgpack.unpack(ARGV[9])
local token = opts['token']
local attempts = opts['attempts']
local attemptsMade = opts['attemptsMade']
local maxMetricsSize = opts['maxMetricsSize']
local maxCount = opts['keepJobs']['count']
local maxAge = opts['keepJobs']['age']
if token ~= "0" then
local lockKey = jobIdKey .. ':lock'
local lockToken = rcall("GET", lockKey)
if lockToken == token then
rcall("DEL", lockKey)
rcall("SREM", KEYS[5], ARGV[1])
else
if lockToken then
-- Lock exists but token does not match
return -6
else
-- Lock is missing completely
return -2
end
end
end
if rcall("SCARD", jobIdKey .. ":dependencies") ~= 0 then -- // Make sure it does not have pending dependencies
return -4
end
local parentReferences = rcall("HMGET", jobIdKey, "parentKey", "parent")
local parentKey = parentReferences[1] or ""
local parentId = ""
local parentQueueKey = ""
if parentReferences[2] ~= false then
local jsonDecodedParent = cjson.decode(parentReferences[2])
parentId = jsonDecodedParent['id']
parentQueueKey = jsonDecodedParent['queueKey']
end
local jobId = ARGV[1]
local timestamp = ARGV[2]
-- Remove from active list (if not active we shall return error)
local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
if (numRemovedElements < 1) then return -3 end
-- Trim events before emiting them to avoid trimming events emitted in this script
trimEvents(KEYS[9], KEYS[4])
-- If job has a parent we need to
-- 1) remove this job id from parents dependencies
-- 2) move the job Id to parent "processed" set
-- 3) push the results into parent "results" list
-- 4) if parent's dependencies is empty, then move parent to "wait/paused". Note it may be a different queue!.
if parentId == "" and parentKey ~= "" then
parentId = getJobIdFromKey(parentKey)
parentQueueKey = getJobKeyPrefix(parentKey, ":" .. parentId)
end
if parentId ~= "" then
if ARGV[5] == "completed" then
local dependenciesSet = parentKey .. ":dependencies"
if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
updateParentDepsIfNeeded(parentKey, parentQueueKey,
dependenciesSet, parentId, jobIdKey,
ARGV[4], timestamp)
end
else
if opts['fpof'] then
moveParentFromWaitingChildrenToFailed(parentQueueKey, parentKey,
parentId, jobIdKey, timestamp)
elseif opts['rdof'] then
local dependenciesSet = parentKey .. ":dependencies"
if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
moveParentToWaitIfNeeded(parentQueueKey, dependenciesSet,
parentKey, parentId, timestamp)
end
end
end
end
-- Remove job?
if maxCount ~= 0 then
local targetSet = KEYS[11]
-- Add to complete/failed set
rcall("ZADD", targetSet, timestamp, jobId)
rcall("HMSET", jobIdKey, ARGV[3], ARGV[4], "finishedOn", timestamp)
-- "returnvalue" / "failedReason" and "finishedOn"
-- Remove old jobs?
local prefix = ARGV[8]
if maxAge ~= nil then
removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
end
if maxCount ~= nil and maxCount > 0 then
removeJobsByMaxCount(maxCount, targetSet, prefix)
end
else
rcall("DEL", jobIdKey, jobIdKey .. ':logs', jobIdKey .. ':processed')
if parentKey ~= "" then
removeParentDependencyKey(jobIdKey, false, parentKey)
end
end
rcall("XADD", KEYS[4], "*", "event", ARGV[5], "jobId", jobId, ARGV[3],
ARGV[4])
if ARGV[5] == "failed" then
if tonumber(attemptsMade) >= tonumber(attempts) then
rcall("XADD", KEYS[4], "*", "event", "retries-exhausted", "jobId",
jobId, "attemptsMade", attemptsMade)
end
end
-- Collect metrics
if maxMetricsSize ~= "" then
collectMetrics(KEYS[13], KEYS[13] .. ':data', maxMetricsSize, timestamp)
end
-- Try to get next job to avoid an extra roundtrip if the queue is not closing,
-- and not rate limited.
if (ARGV[7] == "1") then
local target, paused = getTargetQueueList(KEYS[9], KEYS[1], KEYS[8])
-- Check if there are delayed jobs that can be promoted
promoteDelayedJobs(KEYS[7], KEYS[1], target, KEYS[3],
KEYS[4], ARGV[8], timestamp, paused, KEYS[10])
local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
-- Check if we are rate limited first.
local expireTime = getRateLimitTTL(maxJobs, KEYS[6])
if expireTime > 0 then return {0, 0, expireTime, 0} end
-- paused queue
if paused then return {0, 0, 0, 0} end
jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
if jobId then
if string.sub(jobId, 1, 2) == "0:" then
rcall("LREM", KEYS[2], 1, jobId)
-- If jobId is special ID 0:delay (delay greater than 0), then there is no job to process
-- but if ID is 0:0, then there is at least 1 prioritized job to process
if jobId == "0:0" then
jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2], KEYS[10])
return prepareJobForProcessing(KEYS, ARGV[8], target, jobId, timestamp,
maxJobs, expireTime, opts)
end
else
return prepareJobForProcessing(KEYS, ARGV[8], target, jobId, timestamp, maxJobs,
expireTime, opts)
end
else
jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2], KEYS[10])
if jobId then
return prepareJobForProcessing(KEYS, ARGV[8], target, jobId, timestamp, maxJobs,
expireTime, opts)
end
end
-- Return the timestamp for the next delayed job if any.
local nextTimestamp = getNextDelayedTimestamp(KEYS[7])
if nextTimestamp ~= nil then
-- The result is guaranteed to be positive, since the
-- ZRANGEBYSCORE command would have return a job otherwise.
return {0, 0, 0, nextTimestamp}
end
end
local waitLen = rcall("LLEN", KEYS[1])
if waitLen == 0 then
local activeLen = rcall("LLEN", KEYS[2])
if activeLen == 0 then
local prioritizedLen = rcall("ZCARD", KEYS[3])
if prioritizedLen == 0 then
rcall("XADD", KEYS[4], "*", "event", "drained")
end
end
end
return 0
else
return -1
end
+62
View File
@@ -0,0 +1,62 @@
--[[
Moves job from active to waiting children set.
Input:
KEYS[1] lock key
KEYS[2] active key
KEYS[3] waitChildrenKey key
KEYS[4] job key
ARGV[1] token
ARGV[2] child key
ARGV[3] timestamp
ARGV[4] the id of the job
Output:
0 - OK
1 - There are not pending dependencies.
-1 - Missing job.
-2 - Missing lock
-3 - Job not in active set
]]
local rcall = redis.call
local function moveToWaitingChildren (activeKey, waitingChildrenKey, jobId, timestamp, lockKey, token)
if token ~= "0" then
if rcall("GET", lockKey) == token then
rcall("DEL", lockKey)
else
return -2
end
end
local score = tonumber(timestamp)
local numRemovedElements = rcall("LREM", activeKey, -1, jobId)
if(numRemovedElements < 1) then
return -3
end
rcall("ZADD", waitingChildrenKey, score, jobId)
return 0
end
if rcall("EXISTS", KEYS[4]) == 1 then
if ARGV[2] ~= "" then
if rcall("SISMEMBER", KEYS[4] .. ":dependencies", ARGV[2]) ~= 0 then
return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1], ARGV[1])
end
return 1
else
if rcall("SCARD", KEYS[4] .. ":dependencies") ~= 0 then
return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1], ARGV[1])
end
return 1
end
end
return -1
+103
View File
@@ -0,0 +1,103 @@
--[[
Completely obliterates a queue and all of its contents
Input:
KEYS[1] meta
KEYS[2] base
ARGV[1] count
ARGV[2] force
]]
-- This command completely destroys a queue including all of its jobs, current or past
-- leaving no trace of its existence. Since this script needs to iterate to find all the job
-- keys, consider that this call may be slow for very large queues.
-- The queue needs to be "paused" or it will return an error
-- If the queue has currently active jobs then the script by default will return error,
-- however this behaviour can be overrided using the 'force' option.
local maxCount = tonumber(ARGV[1])
local baseKey = KEYS[2]
local rcall = redis.call
-- Includes
--- @include "includes/removeJobs"
--- @include "includes/removeListJobs"
--- @include "includes/removeZSetJobs"
local function removeLockKeys(keys)
for i, key in ipairs(keys) do
rcall("DEL", baseKey .. key .. ':lock')
end
end
-- 1) Check if paused, if not return with error.
if rcall("HEXISTS", KEYS[1], "paused") ~= 1 then
return -1 -- Error, NotPaused
end
-- 2) Check if there are active jobs, if there are and not "force" return error.
local activeKey = baseKey .. 'active'
local activeJobs = getListItems(activeKey, maxCount)
if (#activeJobs > 0) then
if(ARGV[2] == "") then
return -2 -- Error, ExistActiveJobs
end
end
removeLockKeys(activeJobs)
maxCount = removeJobs(activeJobs, true, baseKey, maxCount)
rcall("LTRIM", activeKey, #activeJobs, -1)
if(maxCount <= 0) then
return 1
end
local delayedKey = baseKey .. 'delayed'
maxCount = removeZSetJobs(delayedKey, true, baseKey, maxCount)
if(maxCount <= 0) then
return 1
end
local completedKey = baseKey .. 'completed'
maxCount = removeZSetJobs(completedKey, true, baseKey, maxCount)
if(maxCount <= 0) then
return 1
end
local waitKey = baseKey .. 'paused'
maxCount = removeListJobs(waitKey, true, baseKey, maxCount)
if(maxCount <= 0) then
return 1
end
local prioritizedKey = baseKey .. 'prioritized'
maxCount = removeZSetJobs(prioritizedKey, true, baseKey, maxCount)
if(maxCount <= 0) then
return 1
end
local failedKey = baseKey .. 'failed'
maxCount = removeZSetJobs(failedKey, true, baseKey, maxCount)
if(maxCount <= 0) then
return 1
end
if(maxCount > 0) then
rcall("DEL",
baseKey .. 'events',
baseKey .. 'delay',
baseKey .. 'stalled-check',
baseKey .. 'stalled',
baseKey .. 'id',
baseKey .. 'pc',
baseKey .. 'meta',
baseKey .. 'repeat',
baseKey .. 'metrics:completed',
baseKey .. 'metrics:completed:data',
baseKey .. 'metrics:failed',
baseKey .. 'metrics:failed:data')
return 0
else
return 1
end
+49
View File
@@ -0,0 +1,49 @@
--[[
Paginate a set or hash
Input:
KEYS[1] key pointing to the set or hash to be paginated.
ARGV[1] page start offset
ARGV[2] page end offset (-1 for all the elements)
ARGV[3] cursor
ARGV[4] offset
ARGV[5] max iterations
ARGV[6] fetch jobs?
Output:
[cursor, offset, items, numItems]
]]
local rcall = redis.call
-- Includes
--- @include "includes/findPage"
local key = KEYS[1]
local scanCommand = "SSCAN"
local countCommand = "SCARD"
local type = rcall("TYPE", key)["ok"]
if type == "none" then
return {0, 0, {}, 0}
elseif type == "hash" then
scanCommand = "HSCAN"
countCommand = "HLEN"
elseif type ~= "set" then
return
redis.error_reply("Pagination is only supported for sets and hashes.")
end
local numItems = rcall(countCommand, key)
local startOffset = tonumber(ARGV[1])
local endOffset = tonumber(ARGV[2])
if endOffset == -1 then
endOffset = numItems
end
local pageSize = (endOffset - startOffset) + 1
local cursor, offset, items, jobs = findPage(key, scanCommand, startOffset,
pageSize, ARGV[3], tonumber(ARGV[4]),
tonumber(ARGV[5]), ARGV[6])
return {cursor, offset, items, numItems, jobs}
+36
View File
@@ -0,0 +1,36 @@
--[[
Pauses or resumes a queue globably.
Input:
KEYS[1] 'wait' or 'paused''
KEYS[2] 'paused' or 'wait'
KEYS[3] 'meta'
KEYS[4] 'prioritized'
KEYS[5] events stream key
ARGV[1] 'paused' or 'resumed'
Event:
publish paused or resumed event.
]]
local rcall = redis.call
-- Includes
--- @include "includes/addPriorityMarkerIfNeeded"
if rcall("EXISTS", KEYS[1]) == 1 then
rcall("RENAME", KEYS[1], KEYS[2])
end
if ARGV[1] == "paused" then
rcall("HSET", KEYS[3], "paused", 1)
else
rcall("HDEL", KEYS[3], "paused")
local priorityCount = rcall("ZCARD", KEYS[4])
if priorityCount > 0 then
addPriorityMarkerIfNeeded(KEYS[2])
end
end
rcall("XADD", KEYS[5], "*", "event", ARGV[1]);
+57
View File
@@ -0,0 +1,57 @@
--[[
Promotes a job that is currently "delayed" to the "waiting" state
Input:
KEYS[1] 'delayed'
KEYS[2] 'wait'
KEYS[3] 'paused'
KEYS[4] 'meta'
KEYS[5] 'prioritized'
KEYS[6] 'pc' priority counter
KEYS[7] 'event stream'
ARGV[1] queue.toKey('')
ARGV[2] jobId
Output:
0 - OK
-3 - Job not in delayed zset.
Events:
'waiting'
]]
local rcall = redis.call
local jobId = ARGV[2]
-- Includes
--- @include "includes/addJobWithPriority"
--- @include "includes/getTargetQueueList"
if rcall("ZREM", KEYS[1], jobId) == 1 then
local jobKey = ARGV[1] .. jobId
local priority = tonumber(rcall("HGET", jobKey, "priority")) or 0
local target, paused = getTargetQueueList(KEYS[4], KEYS[2], KEYS[3])
-- Remove delayed "marker" from the wait list if there is any.
-- Since we are adding a job we do not need the marker anymore.
local marker = rcall("LINDEX", target, 0)
if marker and string.sub(marker, 1, 2) == "0:" then
rcall("LPOP", target)
end
if priority == 0 then
-- LIFO or FIFO
rcall("LPUSH", target, jobId)
else
addJobWithPriority(KEYS[2], KEYS[5], priority, paused, jobId, KEYS[6])
end
-- Emit waiting event (wait..ing@token)
rcall("XADD", KEYS[7], "*", "event", "waiting", "jobId", jobId, "prev", "delayed");
rcall("HSET", jobKey, "delay", 0)
return 0
else
return -3
end
+19
View File
@@ -0,0 +1,19 @@
--[[
Release lock
Input:
KEYS[1] 'lock',
ARGV[1] token
ARGV[2] lock duration in milliseconds
Output:
"OK" if lock extented succesfully.
]]
local rcall = redis.call
if rcall("GET", KEYS[1]) == ARGV[1] then
return rcall("DEL", KEYS[1])
else
return 0
end
+69
View File
@@ -0,0 +1,69 @@
--[[
Remove a job from all the queues it may be in as well as all its data.
In order to be able to remove a job, it cannot be active.
Input:
KEYS[1] queue prefix
ARGV[1] jobId
ARGV[2] remove children
Events:
'removed'
]]
local rcall = redis.call
-- Includes
--- @include "includes/destructureJobKey"
--- @include "includes/isLocked"
--- @include "includes/removeJobFromAnyState"
--- @include "includes/removeParentDependencyKey"
local function removeJob( prefix, jobId, parentKey, removeChildren)
local jobKey = prefix .. jobId;
removeParentDependencyKey(jobKey, false, parentKey)
if removeChildren == "1" then
-- Check if this job has children
-- If so, we are going to try to remove the children recursively in deep first way because
-- if some job is locked we must exit with and error.
--local countProcessed = rcall("HLEN", jobKey .. ":processed")
local processed = rcall("HGETALL", jobKey .. ":processed")
if (#processed > 0) then
for i = 1, #processed, 2 do
local childJobId = getJobIdFromKey(processed[i])
local childJobPrefix = getJobKeyPrefix(processed[i], childJobId)
removeJob( childJobPrefix, childJobId, jobKey, removeChildren )
end
end
local dependencies = rcall("SMEMBERS", jobKey .. ":dependencies")
if (#dependencies > 0) then
for i, childJobKey in ipairs(dependencies) do
-- We need to get the jobId for this job.
local childJobId = getJobIdFromKey(childJobKey)
local childJobPrefix = getJobKeyPrefix(childJobKey, childJobId)
removeJob( childJobPrefix, childJobId, jobKey, removeChildren )
end
end
end
local prev = removeJobFromAnyState(prefix, jobId)
if rcall("DEL", jobKey, jobKey .. ":logs", jobKey .. ":dependencies", jobKey .. ":processed") > 0 then
local maxEvents = rcall("HGET", prefix .. "meta", "opts.maxLenEvents") or 10000
rcall("XADD", prefix .. "events", "MAXLEN", "~", maxEvents, "*", "event", "removed",
"jobId", jobId, "prev", prev)
end
end
local prefix = KEYS[1]
if not isLocked(prefix, ARGV[1], ARGV[2]) then
removeJob(prefix, ARGV[1], nil, ARGV[2])
return 1
end
return 0
+35
View File
@@ -0,0 +1,35 @@
--[[
Removes a repeatable job
Input:
KEYS[1] repeat jobs key
KEYS[2] delayed jobs key
ARGV[1] repeat job id
ARGV[2] repeat job key
ARGV[3] queue key
Output:
0 - OK
1 - Missing repeat job
Events:
'removed'
]]
local rcall = redis.call
local millis = rcall("ZSCORE", KEYS[1], ARGV[2])
if(millis) then
-- Delete next programmed job.
local repeatJobId = ARGV[1] .. millis
if(rcall("ZREM", KEYS[2], repeatJobId) == 1) then
rcall("DEL", ARGV[3] .. repeatJobId)
rcall("XADD", ARGV[3] .. "events", "*", "event", "removed", "jobId", repeatJobId, "prev", "delayed");
end
end
if(rcall("ZREM", KEYS[1], ARGV[2]) == 1) then
return 0
end
return 1
+43
View File
@@ -0,0 +1,43 @@
--[[
Attempts to reprocess a job
Input:
KEYS[1] job key
KEYS[2] events stream
KEYS[3] job state
KEYS[4] wait key
KEYS[5] meta
KEYS[6] paused key
ARGV[1] job.id
ARGV[2] (job.opts.lifo ? 'R' : 'L') + 'PUSH'
ARGV[3] propVal - failedReason/returnvalue
ARGV[4] prev state - failed/completed
Output:
1 means the operation was a success
-1 means the job does not exist
-3 means the job was not found in the expected set.
]]
local rcall = redis.call;
-- Includes
--- @include "includes/getTargetQueueList"
if (rcall("EXISTS", KEYS[1]) == 1) then
local jobId = ARGV[1]
if (rcall("ZREM", KEYS[3], jobId) == 1) then
rcall("HDEL", KEYS[1], "finishedOn", "processedOn", ARGV[3])
local target = getTargetQueueList(KEYS[5], KEYS[4], KEYS[6])
rcall(ARGV[2], target, jobId)
-- Emit waiting event
rcall("XADD", KEYS[2], "*", "event", "waiting", "jobId", jobId, "prev", ARGV[4]);
return 1
else
return -3
end
else
return -1
end
+72
View File
@@ -0,0 +1,72 @@
--[[
Retries a failed job by moving it back to the wait queue.
Input:
KEYS[1] 'active',
KEYS[2] 'wait'
KEYS[3] 'paused'
KEYS[4] job key
KEYS[5] 'meta'
KEYS[6] events stream
KEYS[7] delayed key
KEYS[8] prioritized key
KEYS[9] 'pc' priority counter
ARGV[1] key prefix
ARGV[2] timestamp
ARGV[3] pushCmd
ARGV[4] jobId
ARGV[5] token
Events:
'waiting'
Output:
0 - OK
-1 - Missing key
-2 - Missing lock
]]
local rcall = redis.call
-- Includes
--- @include "includes/addJobWithPriority"
--- @include "includes/getTargetQueueList"
--- @include "includes/promoteDelayedJobs"
local target, paused = getTargetQueueList(KEYS[5], KEYS[2], KEYS[3])
-- Check if there are delayed jobs that we can move to wait.
-- test example: when there are delayed jobs between retries
promoteDelayedJobs(KEYS[7], KEYS[2], target, KEYS[8], KEYS[6], ARGV[1], ARGV[2], paused, KEYS[9])
if rcall("EXISTS", KEYS[4]) == 1 then
if ARGV[5] ~= "0" then
local lockKey = KEYS[4] .. ':lock'
if rcall("GET", lockKey) == ARGV[5] then
rcall("DEL", lockKey)
else
return -2
end
end
rcall("LREM", KEYS[1], 0, ARGV[4])
local priority = tonumber(rcall("HGET", KEYS[4], "priority")) or 0
-- Standard or priority add
if priority == 0 then
rcall(ARGV[3], target, ARGV[4])
else
addJobWithPriority(KEYS[2], KEYS[8], priority, paused, ARGV[4], KEYS[9])
end
local maxEvents = rcall("HGET", KEYS[5], "opts.maxLenEvents") or 10000
-- Emit waiting event
rcall("XADD", KEYS[6], "MAXLEN", "~", maxEvents, "*", "event", "waiting",
"jobId", ARGV[4], "prev", "failed")
return 0
else
return -1
end
+22
View File
@@ -0,0 +1,22 @@
--[[
Save stacktrace and failedReason.
Input:
KEYS[1] job key
ARGV[1] stacktrace
ARGV[2] failedReason
Output:
0 - OK
-1 - Missing key
]]
local rcall = redis.call
if rcall("EXISTS", KEYS[1]) == 1 then
rcall("HMSET", KEYS[1], "stacktrace", ARGV[1], "failedReason", ARGV[2])
return 0
else
return -1
end
+408
View File
@@ -0,0 +1,408 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.ScriptLoader = exports.ScriptLoaderError = void 0;
const crypto_1 = require("crypto");
const glob_1 = require("glob");
const path = require("path");
const fs = require("fs");
const util_1 = require("util");
const readFile = (0, util_1.promisify)(fs.readFile);
const readdir = (0, util_1.promisify)(fs.readdir);
const GlobOptions = { dot: true, silent: false };
const IncludeRegex = /^[-]{2,3}[ \t]*@include[ \t]+(["'])(.+?)\1[; \t\n]*$/m;
const EmptyLineRegex = /^\s*[\r\n]/gm;
class ScriptLoaderError extends Error {
constructor(message, path, stack = [], line, position = 0) {
super(message);
// Ensure the name of this error is the same as the class name
this.name = this.constructor.name;
Error.captureStackTrace(this, this.constructor);
this.includes = stack;
this.line = line !== null && line !== void 0 ? line : 0;
this.position = position;
}
}
exports.ScriptLoaderError = ScriptLoaderError;
const isPossiblyMappedPath = (path) => path && ['~', '<'].includes(path[0]);
const hasFilenamePattern = (path) => (0, glob_1.hasMagic)(path, GlobOptions);
/**
* Lua script loader with include support
*/
class ScriptLoader {
constructor() {
/**
* Map an alias to a path
*/
this.pathMapper = new Map();
this.clientScripts = new WeakMap();
/**
* Cache commands by dir
*/
this.commandCache = new Map();
this.rootPath = getPkgJsonDir();
this.pathMapper.set('~', this.rootPath);
this.pathMapper.set('rootDir', this.rootPath);
this.pathMapper.set('base', __dirname);
}
/**
* Add a script path mapping. Allows includes of the form "<includes>/utils.lua" where `includes` is a user
* defined path
* @param name - the name of the mapping. Note: do not include angle brackets
* @param mappedPath - if a relative path is passed, it's relative to the *caller* of this function.
* Mapped paths are also accepted, e.g. "~/server/scripts/lua" or "<base>/includes"
*/
addPathMapping(name, mappedPath) {
let resolved;
if (isPossiblyMappedPath(mappedPath)) {
resolved = this.resolvePath(mappedPath);
}
else {
const caller = getCallerFile();
const callerPath = path.dirname(caller);
resolved = path.normalize(path.resolve(callerPath, mappedPath));
}
const last = resolved.length - 1;
if (resolved[last] === path.sep) {
resolved = resolved.substr(0, last);
}
this.pathMapper.set(name, resolved);
}
/**
* Resolve the script path considering path mappings
* @param scriptName - the name of the script
* @param stack - the include stack, for nicer errors
*/
resolvePath(scriptName, stack = []) {
const first = scriptName[0];
if (first === '~') {
scriptName = path.join(this.rootPath, scriptName.substr(2));
}
else if (first === '<') {
const p = scriptName.indexOf('>');
if (p > 0) {
const name = scriptName.substring(1, p);
const mappedPath = this.pathMapper.get(name);
if (!mappedPath) {
throw new ScriptLoaderError(`No path mapping found for "${name}"`, scriptName, stack);
}
scriptName = path.join(mappedPath, scriptName.substring(p + 1));
}
}
return path.normalize(scriptName);
}
/**
* Recursively collect all scripts included in a file
* @param file - the parent file
* @param cache - a cache for file metadata to increase efficiency. Since a file can be included
* multiple times, we make sure to load it only once.
* @param stack - internal stack to prevent circular references
*/
async resolveDependencies(file, cache, isInclude = false, stack = []) {
cache = cache !== null && cache !== void 0 ? cache : new Map();
if (stack.includes(file.path)) {
throw new ScriptLoaderError(`circular reference: "${file.path}"`, file.path, stack);
}
stack.push(file.path);
function findPos(content, match) {
const pos = content.indexOf(match);
const arr = content.slice(0, pos).split('\n');
return {
line: arr.length,
column: arr[arr.length - 1].length + match.indexOf('@include') + 1,
};
}
function raiseError(msg, match) {
const pos = findPos(file.content, match);
throw new ScriptLoaderError(msg, file.path, stack, pos.line, pos.column);
}
let res;
let content = file.content;
while ((res = IncludeRegex.exec(content)) !== null) {
const [match, , reference] = res;
const includeFilename = isPossiblyMappedPath(reference)
? // mapped paths imply absolute reference
this.resolvePath(ensureExt(reference), stack)
: // include path is relative to the file being processed
path.resolve(path.dirname(file.path), ensureExt(reference));
let includePaths;
if (hasFilenamePattern(includeFilename)) {
const filesMatched = await getFilenamesByPattern(includeFilename);
includePaths = filesMatched.map((x) => path.resolve(x));
}
else {
includePaths = [includeFilename];
}
includePaths = includePaths.filter((file) => path.extname(file) === '.lua');
if (includePaths.length === 0) {
raiseError(`include not found: "${reference}"`, match);
}
const tokens = [];
for (let i = 0; i < includePaths.length; i++) {
const includePath = includePaths[i];
const hasInclude = file.includes.find((x) => x.path === includePath);
if (hasInclude) {
/**
* We have something like
* --- \@include "a"
* ...
* --- \@include "a"
*/
raiseError(`file "${reference}" already included in "${file.path}"`, match);
}
let includeMetadata = cache.get(includePath);
let token;
if (!includeMetadata) {
const { name, numberOfKeys } = splitFilename(includePath);
let childContent = '';
try {
const buf = await readFile(includePath, { flag: 'r' });
childContent = buf.toString();
}
catch (err) {
if (err.code === 'ENOENT') {
raiseError(`include not found: "${reference}"`, match);
}
else {
throw err;
}
}
// this represents a normalized version of the path to make replacement easy
token = getPathHash(includePath);
includeMetadata = {
name,
numberOfKeys,
path: includePath,
content: childContent,
token,
includes: [],
};
cache.set(includePath, includeMetadata);
}
else {
token = includeMetadata.token;
}
tokens.push(token);
file.includes.push(includeMetadata);
await this.resolveDependencies(includeMetadata, cache, true, stack);
}
// Replace @includes with normalized path hashes
const substitution = tokens.join('\n');
content = content.replace(match, substitution);
}
file.content = content;
if (isInclude) {
cache.set(file.path, file);
}
else {
cache.set(file.name, file);
}
stack.pop();
}
/**
* Parse a (top-level) lua script
* @param filename - the full path to the script
* @param content - the content of the script
* @param cache - cache
*/
async parseScript(filename, content, cache) {
const { name, numberOfKeys } = splitFilename(filename);
const meta = cache === null || cache === void 0 ? void 0 : cache.get(name);
if ((meta === null || meta === void 0 ? void 0 : meta.content) === content) {
return meta;
}
const fileInfo = {
path: filename,
token: getPathHash(filename),
content,
name,
numberOfKeys,
includes: [],
};
await this.resolveDependencies(fileInfo, cache);
return fileInfo;
}
/**
* Construct the final version of a file by interpolating its includes in dependency order.
* @param file - the file whose content we want to construct
* @param processed - a cache to keep track of which includes have already been processed
*/
interpolate(file, processed) {
processed = processed || new Set();
let content = file.content;
file.includes.forEach((child) => {
const emitted = processed.has(child.path);
const fragment = this.interpolate(child, processed);
const replacement = emitted ? '' : fragment;
if (!replacement) {
content = replaceAll(content, child.token, '');
}
else {
// replace the first instance with the dependency
content = content.replace(child.token, replacement);
// remove the rest
content = replaceAll(content, child.token, '');
}
processed.add(child.path);
});
return content;
}
async loadCommand(filename, cache) {
filename = path.resolve(filename);
const { name: scriptName } = splitFilename(filename);
let script = cache === null || cache === void 0 ? void 0 : cache.get(scriptName);
if (!script) {
const content = (await readFile(filename)).toString();
script = await this.parseScript(filename, content, cache);
}
const lua = removeEmptyLines(this.interpolate(script));
const { name, numberOfKeys } = script;
return {
name,
options: { numberOfKeys: numberOfKeys, lua },
};
}
/**
* Load redis lua scripts.
* The name of the script must have the following format:
*
* cmdName-numKeys.lua
*
* cmdName must be in camel case format.
*
* For example:
* moveToFinish-3.lua
*
*/
async loadScripts(dir, cache) {
dir = path.normalize(dir || __dirname);
let commands = this.commandCache.get(dir);
if (commands) {
return commands;
}
const files = await readdir(dir);
const luaFiles = files.filter((file) => path.extname(file) === '.lua');
if (luaFiles.length === 0) {
/**
* To prevent unclarified runtime error "updateDelayset is not a function
* @see https://github.com/OptimalBits/bull/issues/920
*/
throw new ScriptLoaderError('No .lua files found!', dir, []);
}
commands = [];
cache = cache !== null && cache !== void 0 ? cache : new Map();
for (let i = 0; i < luaFiles.length; i++) {
const file = path.join(dir, luaFiles[i]);
const command = await this.loadCommand(file, cache);
commands.push(command);
}
this.commandCache.set(dir, commands);
return commands;
}
/**
* Attach all lua scripts in a given directory to a client instance
* @param client - redis client to attach script to
* @param pathname - the path to the directory containing the scripts
*/
async load(client, pathname, cache) {
let paths = this.clientScripts.get(client);
if (!paths) {
paths = new Set();
this.clientScripts.set(client, paths);
}
if (!paths.has(pathname)) {
paths.add(pathname);
const scripts = await this.loadScripts(pathname, cache !== null && cache !== void 0 ? cache : new Map());
scripts.forEach((command) => {
// Only define the command if not already defined
if (!client[command.name]) {
client.defineCommand(command.name, command.options);
}
});
}
}
/**
* Clears the command cache
*/
clearCache() {
this.commandCache.clear();
}
}
exports.ScriptLoader = ScriptLoader;
function ensureExt(filename, ext = 'lua') {
const foundExt = path.extname(filename);
if (foundExt && foundExt !== '.') {
return filename;
}
if (ext && ext[0] !== '.') {
ext = `.${ext}`;
}
return `${filename}${ext}`;
}
function splitFilename(filePath) {
const longName = path.basename(filePath, '.lua');
const [name, num] = longName.split('-');
const numberOfKeys = num ? parseInt(num, 10) : undefined;
return { name, numberOfKeys };
}
async function getFilenamesByPattern(pattern) {
return new Promise((resolve, reject) => {
(0, glob_1.glob)(pattern, GlobOptions, (err, files) => {
return err ? reject(err) : resolve(files);
});
});
}
// Determine the project root
// https://stackoverflow.com/a/18721515
function getPkgJsonDir() {
for (const modPath of module.paths || []) {
try {
const prospectivePkgJsonDir = path.dirname(modPath);
fs.accessSync(modPath, fs.constants.F_OK);
return prospectivePkgJsonDir;
// eslint-disable-next-line no-empty
}
catch (e) { }
}
return '';
}
// https://stackoverflow.com/a/66842927
// some dark magic here :-)
// this version is preferred to the simpler version because of
// https://github.com/facebook/jest/issues/5303 -
// tldr: dont assume you're the only one with the doing something like this
function getCallerFile() {
var _a, _b, _c;
const originalFunc = Error.prepareStackTrace;
let callerFile = '';
try {
Error.prepareStackTrace = (_, stack) => stack;
const sites = new Error().stack;
const currentFile = (_a = sites.shift()) === null || _a === void 0 ? void 0 : _a.getFileName();
while (sites.length) {
callerFile = (_c = (_b = sites.shift()) === null || _b === void 0 ? void 0 : _b.getFileName()) !== null && _c !== void 0 ? _c : '';
if (currentFile !== callerFile) {
break;
}
}
// eslint-disable-next-line no-empty
}
catch (e) {
}
finally {
Error.prepareStackTrace = originalFunc;
}
return callerFile;
}
function sha1(data) {
return (0, crypto_1.createHash)('sha1').update(data).digest('hex');
}
function getPathHash(normalizedPath) {
return `@@${sha1(normalizedPath)}`;
}
function replaceAll(str, find, replace) {
return str.replace(new RegExp(find, 'g'), replace);
}
function removeEmptyLines(str) {
return str.replace(EmptyLineRegex, '');
}
//# sourceMappingURL=script-loader.js.map
File diff suppressed because one or more lines are too long
+20
View File
@@ -0,0 +1,20 @@
--[[
Update job data
Input:
KEYS[1] Job id key
ARGV[1] data
Output:
0 - OK
-1 - Missing job.
]]
local rcall = redis.call
if rcall("EXISTS",KEYS[1]) == 1 then -- // Make sure job exists
rcall("HSET", KEYS[1], "data", ARGV[1])
return 0
else
return -1
end
+30
View File
@@ -0,0 +1,30 @@
--[[
Update job progress
Input:
KEYS[1] Job id key
KEYS[2] event stream key
KEYS[3] meta key
ARGV[1] id
ARGV[2] progress
Output:
0 - OK
-1 - Missing job.
Event:
progress(jobId, progress)
]]
local rcall = redis.call
if rcall("EXISTS", KEYS[1]) == 1 then -- // Make sure job exists
local maxEvents = rcall("HGET", KEYS[3], "opts.maxLenEvents") or 10000
rcall("HSET", KEYS[1], "progress", ARGV[2])
rcall("XADD", KEYS[2], "MAXLEN", "~", maxEvents, "*", "event", "progress",
"jobId", ARGV[1], "data", ARGV[2]);
return 0
else
return -1
end