Add very basic crawl job

Loops through feeds and adds items from each feed.
This commit is contained in:
2023-05-09 23:46:42 -04:00
parent 89fdf8f95a
commit ae8f15f19b
18 changed files with 327 additions and 56 deletions

View File

@@ -1,12 +1,13 @@
use anyhow::Result;
use argh::FromArgs;
use dotenvy::dotenv;
use tracing::info;
use sqlx::postgres::PgPoolOptions;
use std::env;
use tracing::info;
use lib::models::feed::{CreateFeed, FeedType};
use lib::commands::add_feed::add_feed;
use lib::jobs::crawl::crawl;
use lib::models::feed::{create_feed, delete_feed, CreateFeed, FeedType};
use lib::models::item::{create_item, delete_item, CreateItem};
#[derive(FromArgs)]
/// CLI for crawlect
@@ -19,6 +20,10 @@ struct Args {
#[argh(subcommand)]
enum Commands {
AddFeed(AddFeed),
DeleteFeed(DeleteFeed),
AddItem(AddItem),
DeleteItem(DeleteItem),
Crawl(Crawl),
}
#[derive(FromArgs)]
@@ -39,6 +44,46 @@ struct AddFeed {
description: Option<String>,
}
#[derive(FromArgs)]
/// Delete a feed from the database
#[argh(subcommand, name = "delete-feed")]
struct DeleteFeed {
#[argh(positional)]
/// id of the feed to delete
id: i32,
}
#[derive(FromArgs)]
/// Add an item to the database
#[argh(subcommand, name = "add-item")]
struct AddItem {
#[argh(option)]
/// title of the item (max 255 characters)
title: String,
#[argh(option)]
/// URL of the item (max 2048 characters)
url: String,
#[argh(option)]
/// description of the item
description: Option<String>,
#[argh(option)]
/// source feed for the item
feed_id: i32,
}
#[derive(FromArgs)]
/// Delete an item from the database
#[argh(subcommand, name = "delete-item")]
struct DeleteItem {
#[argh(positional)]
/// id of the item to delete
id: i32,
}
#[derive(FromArgs)]
/// Delete an item from the database
#[argh(subcommand, name = "crawl")]
struct Crawl {}
#[tokio::main]
pub async fn main() -> Result<()> {
@@ -53,13 +98,47 @@ pub async fn main() -> Result<()> {
let args: Args = argh::from_env();
if let Commands::AddFeed(add_feed_args) = args.commands {
add_feed(pool, CreateFeed {
title: add_feed_args.title,
url: add_feed_args.url,
feed_type: add_feed_args.feed_type,
description: add_feed_args.description,
}).await?;
info!("hello?");
match args.commands {
Commands::AddFeed(args) => {
let feed = create_feed(
&pool,
CreateFeed {
title: args.title,
url: args.url,
feed_type: args.feed_type,
description: args.description,
},
)
.await?;
info!("Created feed with id {}", feed.id);
}
Commands::DeleteFeed(args) => {
delete_feed(&pool, args.id).await?;
info!("Deleted feed with id {}", args.id);
}
Commands::AddItem(args) => {
let item = create_item(
&pool,
CreateItem {
title: args.title,
url: args.url,
description: args.description,
feed_id: args.feed_id,
},
)
.await?;
info!("Created item with id {}", item.id);
}
Commands::DeleteItem(args) => {
delete_item(&pool, args.id).await?;
info!("Deleted item with id {}", args.id);
}
Commands::Crawl(_) => {
info!("Crawling...");
crawl(&pool).await?;
}
}
Ok(())