Initial commit of working RSS Aggregator build

2026-05-12 17:04:02 -03:00
parent ea3a2ca53e
commit 7ac2f6e384
4962 changed files with 1032666 additions and 0 deletions
@@ -0,0 +1,117 @@
+--[[
+  Adds a job to the queue by doing the following:
+    - Increases the job counter if needed.
+    - Creates a new job key with the job data.
+
+    - if delayed:
+      - computes timestamp.
+      - adds to delayed zset.
+      - Emits a global event 'delayed' if the job is delayed.
+    - if not delayed
+      - Adds the jobId to the wait/paused list in one of three ways:
+         - LIFO
+         - FIFO
+         - prioritized.
+      - Adds the job to the "added" list so that workers gets notified.
+
+    Input:
+      KEYS[1] 'wait',
+      KEYS[2] 'paused'
+      KEYS[3] 'meta'
+      KEYS[4] 'id'
+      KEYS[5] 'completed'
+      KEYS[6] events stream key
+
+      ARGV[1] msgpacked arguments array
+            [1]  key prefix,
+            [2]  custom id (will not generate one automatically)
+            [3]  name
+            [4]  timestamp
+            [5]  parentKey?
+            [6]  waitChildrenKey key.
+            [7]  parent dependencies key.
+            [8]  parent? {id, queueKey}
+            [9]  repeat job key
+            
+      ARGV[2] Json stringified job data
+      ARGV[3] msgpacked options
+
+      Output:
+        jobId  - OK
+        -5     - Missing parent key
+]]
+local eventsKey = KEYS[6]
+
+local jobId
+local jobIdKey
+local rcall = redis.call
+
+local args = cmsgpack.unpack(ARGV[1])
+
+local data = ARGV[2]
+local opts = cmsgpack.unpack(ARGV[3])
+
+local parentKey = args[5]
+local repeatJobKey = args[9]
+local parent = args[8]
+local parentData
+
+-- Includes
+--- @include "includes/storeJob"
+--- @include "includes/updateExistingJobsParent"
+--- @include "includes/getTargetQueueList"
+--- @include "includes/getOrSetMaxEvents"
+
+if parentKey ~= nil then
+    if rcall("EXISTS", parentKey) ~= 1 then return -5 end
+
+    parentData = cjson.encode(parent)
+end
+
+local jobCounter = rcall("INCR", KEYS[4])
+
+local metaKey = KEYS[3]
+local maxEvents = getOrSetMaxEvents(metaKey)
+
+local parentDependenciesKey = args[7]
+local timestamp = args[4]
+if args[2] == "" then
+    jobId = jobCounter
+    jobIdKey = args[1] .. jobId
+else
+    jobId = args[2]
+    jobIdKey = args[1] .. jobId
+    if rcall("EXISTS", jobIdKey) == 1 then
+        updateExistingJobsParent(parentKey, parent, parentData,
+                                 parentDependenciesKey, KEYS[5], jobIdKey,
+                                 jobId, timestamp)
+
+        rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
+              "duplicated", "jobId", jobId)
+
+        return jobId .. "" -- convert to string
+    end
+end
+
+-- Store the job.
+storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2], opts, timestamp,
+         parentKey, parentData, repeatJobKey)
+
+local target, paused = getTargetQueueList(metaKey, KEYS[1], KEYS[2])
+
+-- LIFO or FIFO
+local pushCmd = opts['lifo'] and 'RPUSH' or 'LPUSH'
+rcall(pushCmd, target, jobId)
+
+-- Emit waiting event
+rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting",
+      "jobId", jobId)
+
+-- Check if this job is a child of another job, if so add it to the parents dependencies
+-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
+-- fail in this case.
+if parentDependenciesKey ~= nil then
+    rcall("SADD", parentDependenciesKey, jobIdKey)
+end
+
+return jobId .. "" -- convert to string