Move feed fetching to crawl_feed job, DomainRequestLimiter

`DomainRequestLimiter` is a distributed version of `DomainLocks` based
on redis.
This commit is contained in:
2024-08-26 01:12:18 -04:00
parent 9c75a88c69
commit 65eac1975c
9 changed files with 390 additions and 22 deletions

View File

@@ -31,6 +31,7 @@ chrono = { version = "0.4", features = ["serde"] }
clap = { version = "4.4", features = ["derive", "env"] }
dotenvy = "0.15"
feed-rs = "1.3"
fred = "9"
futures = "0.3"
headers = "0.4"
http = "1.0.0"
@@ -44,6 +45,7 @@ notify = "6"
once_cell = "1.18"
opml = "1.1"
password-auth = "1.0"
rand = { version = "0.8", features = ["small_rng"] }
readability = "0.3"
reqwest = { version = "0.12", features = ["json"] }
serde = { version = "1", features = ["derive"] }