Initial commit of working RSS Aggregator build
This commit is contained in:
Vendored
+1
@@ -0,0 +1 @@
|
||||
export {};
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
export {};
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
export {};
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
export {};
|
||||
Vendored
+4
@@ -0,0 +1,4 @@
|
||||
export const config = {
|
||||
redisUrl: process.env.REDIS_URL,
|
||||
port: process.env.PORT || 8080
|
||||
};
|
||||
Vendored
+2
@@ -0,0 +1,2 @@
|
||||
import { PrismaClient } from "@prisma/client";
|
||||
export const prisma = new PrismaClient();
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
export {};
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
export {};
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
export {};
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
export {};
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
export {};
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
export {};
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
export {};
|
||||
Vendored
+30
@@ -0,0 +1,30 @@
|
||||
import express from "express";
|
||||
import session from "express-session";
|
||||
import RedisStore from "connect-redis";
|
||||
import { createClient } from "redis";
|
||||
import { feedQueue } from "./queue.js";
|
||||
import { config } from "./config.js";
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
// Redis client (NO top-level await)
|
||||
const redis = createClient({ url: config.redisUrl });
|
||||
redis.connect().catch(err => {
|
||||
console.error("Redis connection failed:", err);
|
||||
});
|
||||
app.use(session({
|
||||
store: new RedisStore({ client: redis }),
|
||||
secret: "dev-secret",
|
||||
resave: false,
|
||||
saveUninitialized: false
|
||||
}));
|
||||
// Typed route handlers
|
||||
app.get("/health", (_req, res) => {
|
||||
res.json({ ok: true });
|
||||
});
|
||||
app.post("/feeds/:id/poll", async (req, res) => {
|
||||
await feedQueue.add("poll-feed", { feedId: req.params.id });
|
||||
res.json({ ok: true });
|
||||
});
|
||||
app.listen(config.port, () => {
|
||||
console.log(`API running on port ${config.port}`);
|
||||
});
|
||||
Vendored
+8
@@ -0,0 +1,8 @@
|
||||
import { Queue } from "bullmq";
|
||||
// BullMQ v4 requires host + port, NOT url:
|
||||
export const feedQueue = new Queue("feeds", {
|
||||
connection: {
|
||||
host: "redis",
|
||||
port: 6379
|
||||
}
|
||||
});
|
||||
Vendored
+5
@@ -0,0 +1,5 @@
|
||||
import Parser from "rss-parser";
|
||||
const parser = new Parser();
|
||||
export default async function fetchFeed(url) {
|
||||
return parser.parseURL(url);
|
||||
}
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
export {};
|
||||
Vendored
+22
@@ -0,0 +1,22 @@
|
||||
import { prisma } from "../db.js";
|
||||
export default async function processEntries(feedId, parsed) {
|
||||
for (const item of parsed.items) {
|
||||
await prisma.entry.upsert({
|
||||
where: {
|
||||
guid_feedId: {
|
||||
guid: item.guid || item.link,
|
||||
feedId
|
||||
}
|
||||
},
|
||||
update: {},
|
||||
create: {
|
||||
feedId,
|
||||
guid: item.guid || item.link,
|
||||
title: item.title || "Untitled",
|
||||
link: item.link,
|
||||
content: item.contentSnippet || item.content || "",
|
||||
published: item.isoDate ? new Date(item.isoDate) : null
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
export {};
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
export {};
|
||||
Vendored
+18
@@ -0,0 +1,18 @@
|
||||
import { Worker } from "bullmq";
|
||||
import { prisma } from "./db.js";
|
||||
import fetchFeed from "./rss/fetchFeed.js";
|
||||
import processEntries from "./rss/processEntries.js";
|
||||
// BullMQ v4 requires host + port, NOT url
|
||||
const connection = {
|
||||
host: "redis",
|
||||
port: 6379
|
||||
};
|
||||
new Worker("feeds", async (job) => {
|
||||
const feedId = job.data.feedId;
|
||||
const feed = await prisma.feed.findUnique({ where: { id: feedId } });
|
||||
if (!feed)
|
||||
return;
|
||||
const parsed = await fetchFeed(feed.url);
|
||||
await processEntries(feedId, parsed);
|
||||
}, { connection });
|
||||
console.log("Worker running");
|
||||
Reference in New Issue
Block a user