thallada/crawlnicle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Upgrade apalis, add fred pool to state, start publishing in jobs

Browse Source
This commit is contained in:
Tyler Hallada
2024-09-22 13:44:55 -04:00
parent 6912ef9017
commit e41085425a
8 changed files with 75 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/bin/worker.rs
View File

@@ -30,9 +30,8 @@ async fn main() -> Result<()> {
let redis_config = RedisConfig::from_url(&config.redis_url)?;
let redis_pool = RedisPool::new(redis_config, None, None, None, 5)?;
redis_pool.connect();
redis_pool.wait_for_connect().await?;
let domain_request_limiter = DomainRequestLimiter::new(redis_pool, 10, 5, 100, 0.5);
redis_pool.init().await?;
let domain_request_limiter = DomainRequestLimiter::new(redis_pool.clone(), 10, 5, 100, 0.5);
let http_client = Client::builder().user_agent(USER_AGENT).build()?;
let db = PgPoolOptions::new()
@@ -51,6 +50,7 @@ async fn main() -> Result<()> {
.data(domain_request_limiter)
.data(config)
.data(apalis_storage.clone())
.data(redis_pool)
.backend(apalis_storage)
.build_fn(handle_async_job)
})