thallada/crawlnicle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Working apalis cron and worker with 0.6.0-rc.5

Browse Source 
Also renamed `pool` variables throughout codebase to `db` for clarity.
This commit is contained in:
Tyler Hallada
2024-08-21 01:10:26 -04:00
parent 764d3f23b8
commit a3450e202a
27 changed files with 148 additions and 232 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/bin/cli.rs
View File

@@ -97,7 +97,7 @@ pub async fn main() -> Result<()> {
tracing_subscriber::fmt::init();
let pool = PgPoolOptions::new()
let db = PgPoolOptions::new()
.max_connections(env::var("DATABASE_MAX_CONNECTIONS")?.parse()?)
.connect(&env::var("DATABASE_URL")?)
.await?;
@@ -108,7 +108,7 @@ pub async fn main() -> Result<()> {
match cli.commands {
Commands::AddFeed(args) => {
let feed = Feed::create(
&pool,
&db,
CreateFeed {
title: args.title,
url: args.url,
@@ -119,12 +119,12 @@ pub async fn main() -> Result<()> {
info!("Created feed with id {}", Base62Uuid::from(feed.feed_id));
}
Commands::DeleteFeed(args) => {
Feed::delete(&pool, args.id).await?;
Feed::delete(&db, args.id).await?;
info!("Deleted feed with id {}", Base62Uuid::from(args.id));
}
Commands::AddEntry(args) => {
let entry = Entry::create(
&pool,
&db,
CreateEntry {
title: args.title,
url: args.url,
@@ -137,7 +137,7 @@ pub async fn main() -> Result<()> {
info!("Created entry with id {}", Base62Uuid::from(entry.entry_id));
}
Commands::DeleteEntry(args) => {
Entry::delete(&pool, args.id).await?;
Entry::delete(&db, args.id).await?;
info!("Deleted entry with id {}", Base62Uuid::from(args.id));
}
Commands::Crawl(CrawlFeed { id }) => {
@@ -147,7 +147,7 @@ pub async fn main() -> Result<()> {
// server is running, it will *not* serialize same-domain requests with it.
let domain_locks = DomainLocks::new();
let feed_crawler = FeedCrawlerHandle::new(
pool.clone(),
db.clone(),
client.clone(),
domain_locks.clone(),
env::var("CONTENT_DIR")?,

53
src/bin/crawler.rs
View File

@@ -1,19 +1,17 @@
use anyhow::{anyhow, Result};
use apalis::cron::{CronStream, Schedule};
use apalis::layers::retry::{RetryLayer, RetryPolicy};
use apalis::layers::tracing::TraceLayer;
use apalis::prelude::*;
use apalis::redis::RedisStorage;
use apalis_cron::{CronStream, Schedule};
use apalis_redis::RedisStorage;
use chrono::{DateTime, Utc};
use clap::Parser;
use lib::actors::crawl_scheduler::CrawlSchedulerError;
use lib::jobs::AsyncJob;
use lib::models::feed::{Feed, GetFeedsOptions};
use sqlx::postgres::PgPoolOptions;
use sqlx::PgPool;
use std::str::FromStr;
use std::sync::Arc;
use tower::ServiceBuilder;
use thiserror::Error;
use tracing::{info, instrument};
use dotenvy::dotenv;
@@ -29,26 +27,32 @@ impl From<DateTime<Utc>> for Crawl {
}
}
impl Job for Crawl {
const NAME: &'static str = "apalis::Crawl";
#[derive(Debug, Error)]
enum CrawlError {
#[error("error fetching feeds")]
FetchFeedsError(#[from] sqlx::Error),
#[error("error queueing crawl feed job")]
QueueJobError(String),
}
#[derive(Clone)]
struct State {
pool: PgPool,
apalis: RedisStorage<AsyncJob>,
}
#[instrument(skip_all)]
pub async fn crawl_fn(job: Crawl, state: Data<Arc<State>>) -> Result<()> {
pub async fn crawl_fn(job: Crawl, state: Data<Arc<State>>) -> Result<(), CrawlError> {
tracing::info!(job = ?job, "crawl");
let mut apalis = (state.apalis).clone();
let mut options = GetFeedsOptions::default();
loop {
info!("fetching feeds before: {:?}", options.before);
// TODO: filter to feeds where:
// now >= feed.last_crawled_at + feed.crawl_interval_minutes
// may need more indices...
let feeds = match Feed::get_all(&state.pool, &options).await {
Err(err) => {
return Err(anyhow!(err));
}
Err(err) => return Err(CrawlError::FetchFeedsError(err)),
Ok(feeds) if feeds.is_empty() => {
info!("no more feeds found");
break;
@@ -62,14 +66,15 @@ pub async fn crawl_fn(job: Crawl, state: Data<Arc<State>>) -> Result<()> {
// self.spawn_crawler_loop(feed, respond_to.clone());
apalis
.push(AsyncJob::HelloWorld(feed.feed_id.to_string()))
.await?;
.await
.map_err(|err| CrawlError::QueueJobError(err.to_string()))?;
}
}
Ok(())
}
#[tokio::main]
async fn main() -> Result<()> {
async fn main() -> anyhow::Result<()> {
dotenv().ok();
let config = Config::parse();
let _guard = init_worker_tracing()?;
@@ -80,24 +85,24 @@ async fn main() -> Result<()> {
.connect(&config.database_url)
.await?;
// TODO: use redis_pool from above instead of making a new connection
// TODO: create connection from redis_pool for each job instead using a single connection
// See: https://github.com/geofmureithi/apalis/issues/290
let redis_conn = apalis::redis::connect(config.redis_url.clone()).await?;
let apalis_config = apalis::redis::Config::default();
let mut apalis: RedisStorage<AsyncJob> =
RedisStorage::new_with_config(redis_conn, apalis_config);
let redis_conn = apalis_redis::connect(config.redis_url.clone()).await?;
let apalis_config = apalis_redis::Config::default();
let apalis_storage = RedisStorage::new_with_config(redis_conn, apalis_config);
let state = Arc::new(State {
pool,
apalis: apalis_storage.clone(),
});
let schedule = Schedule::from_str("0 * * * * *").unwrap();
// let service = ServiceBuilder::new()
// .layer(RetryLayer::new(RetryPolicy::default()))
// .layer(TraceLayer::new())
// .service(service_fn(crawl_fn));
let worker = WorkerBuilder::new("crawler")
.stream(CronStream::new(schedule).into_stream())
.layer(RetryLayer::new(RetryPolicy::default()))
.layer(TraceLayer::new())
.data(Arc::new(State { pool, apalis }))
.data(state)
.backend(CronStream::new(schedule))
.build_fn(crawl_fn);
Monitor::<TokioExecutor>::new()

18
src/bin/web.rs
View File

@@ -2,7 +2,7 @@ use std::{collections::HashMap, net::SocketAddr, path::Path, sync::Arc};
use anyhow::Result;
use apalis::prelude::*;
use apalis::redis::RedisStorage;
use apalis_redis::RedisStorage;
use axum::{
routing::{get, post},
Router,
@@ -66,7 +66,7 @@ async fn main() -> Result<()> {
let domain_locks = DomainLocks::new();
let client = Client::builder().user_agent(USER_AGENT).build()?;
let pool = PgPoolOptions::new()
let db = PgPoolOptions::new()
.max_connections(config.database_max_connections)
.acquire_timeout(std::time::Duration::from_secs(3))
.connect(&config.database_url)
@@ -84,7 +84,7 @@ async fn main() -> Result<()> {
)))
.with_signed(Key::from(&BASE64_STANDARD.decode(&config.session_secret)?));
let backend = Backend::new(pool.clone());
let backend = Backend::new(db.clone());
let auth_layer = AuthManagerLayerBuilder::new(backend, session_layer).build();
let smtp_creds = Credentials::new(config.smtp_user.clone(), config.smtp_password.clone());
@@ -94,12 +94,12 @@ async fn main() -> Result<()> {
.credentials(smtp_creds)
.build();
sqlx::migrate!().run(&pool).await?;
sqlx::migrate!().run(&db).await?;
// TODO: use redis_pool from above instead of making a new connection
// See: https://github.com/geofmureithi/apalis/issues/290
let redis_conn = apalis::redis::connect(config.redis_url.clone()).await?;
let apalis_config = apalis::redis::Config::default();
let redis_conn = apalis_redis::connect(config.redis_url.clone()).await?;
let apalis_config = apalis_redis::Config::default();
let mut apalis: RedisStorage<AsyncJob> =
RedisStorage::new_with_config(redis_conn, apalis_config);
@@ -108,14 +108,14 @@ async fn main() -> Result<()> {
.await?;
let crawl_scheduler = CrawlSchedulerHandle::new(
pool.clone(),
db.clone(),
client.clone(),
domain_locks.clone(),
config.content_dir.clone(),
crawls.clone(),
);
let _ = crawl_scheduler.bootstrap().await;
let importer = ImporterHandle::new(pool.clone(), crawl_scheduler.clone(), imports.clone());
let importer = ImporterHandle::new(db.clone(), crawl_scheduler.clone(), imports.clone());
let ip_source_extension = config.ip_source.0.clone().into_extension();
@@ -154,7 +154,7 @@ async fn main() -> Result<()> {
.route("/reset-password", post(handlers::reset_password::post))
.nest_service("/static", ServeDir::new("static"))
.with_state(AppState {
pool,
db,
config,
log_receiver,
crawls,

13
src/bin/worker.rs
View File

@@ -1,7 +1,7 @@
use anyhow::Result;
use apalis::layers::tracing::TraceLayer;
use apalis::prelude::*;
use apalis::redis::RedisStorage;
use apalis_redis::RedisStorage;
use clap::Parser;
use dotenvy::dotenv;
@@ -18,15 +18,18 @@ async fn main() -> Result<()> {
dotenv().ok();
let config = Config::parse();
let _guard = init_worker_tracing()?;
let redis_conn = apalis::redis::connect(config.redis_url.clone()).await?;
let apalis_config = apalis::redis::Config::default();
let apalis: RedisStorage<AsyncJob> = RedisStorage::new_with_config(redis_conn, apalis_config);
// TODO: create connection from redis_pool for each job instead using a single connection
// See: https://github.com/geofmureithi/apalis/issues/290
let redis_conn = apalis_redis::connect(config.redis_url.clone()).await?;
let apalis_config = apalis_redis::Config::default();
let apalis_storage: RedisStorage<AsyncJob> =
RedisStorage::new_with_config(redis_conn, apalis_config);
Monitor::<TokioExecutor>::new()
.register_with_count(2, {
WorkerBuilder::new("worker")
.layer(TraceLayer::new())
.with_storage(apalis.clone())
.backend(apalis_storage)
.build_fn(worker_fn)
})
.run()