Add very basic crawl job
Loops through feeds and adds items from each feed.
This commit is contained in:
@@ -1,12 +1,13 @@
|
||||
use anyhow::Result;
|
||||
use argh::FromArgs;
|
||||
use dotenvy::dotenv;
|
||||
use tracing::info;
|
||||
use sqlx::postgres::PgPoolOptions;
|
||||
use std::env;
|
||||
use tracing::info;
|
||||
|
||||
use lib::models::feed::{CreateFeed, FeedType};
|
||||
use lib::commands::add_feed::add_feed;
|
||||
use lib::jobs::crawl::crawl;
|
||||
use lib::models::feed::{create_feed, delete_feed, CreateFeed, FeedType};
|
||||
use lib::models::item::{create_item, delete_item, CreateItem};
|
||||
|
||||
#[derive(FromArgs)]
|
||||
/// CLI for crawlect
|
||||
@@ -19,6 +20,10 @@ struct Args {
|
||||
#[argh(subcommand)]
|
||||
enum Commands {
|
||||
AddFeed(AddFeed),
|
||||
DeleteFeed(DeleteFeed),
|
||||
AddItem(AddItem),
|
||||
DeleteItem(DeleteItem),
|
||||
Crawl(Crawl),
|
||||
}
|
||||
|
||||
#[derive(FromArgs)]
|
||||
@@ -39,6 +44,46 @@ struct AddFeed {
|
||||
description: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(FromArgs)]
|
||||
/// Delete a feed from the database
|
||||
#[argh(subcommand, name = "delete-feed")]
|
||||
struct DeleteFeed {
|
||||
#[argh(positional)]
|
||||
/// id of the feed to delete
|
||||
id: i32,
|
||||
}
|
||||
|
||||
#[derive(FromArgs)]
|
||||
/// Add an item to the database
|
||||
#[argh(subcommand, name = "add-item")]
|
||||
struct AddItem {
|
||||
#[argh(option)]
|
||||
/// title of the item (max 255 characters)
|
||||
title: String,
|
||||
#[argh(option)]
|
||||
/// URL of the item (max 2048 characters)
|
||||
url: String,
|
||||
#[argh(option)]
|
||||
/// description of the item
|
||||
description: Option<String>,
|
||||
#[argh(option)]
|
||||
/// source feed for the item
|
||||
feed_id: i32,
|
||||
}
|
||||
|
||||
#[derive(FromArgs)]
|
||||
/// Delete an item from the database
|
||||
#[argh(subcommand, name = "delete-item")]
|
||||
struct DeleteItem {
|
||||
#[argh(positional)]
|
||||
/// id of the item to delete
|
||||
id: i32,
|
||||
}
|
||||
|
||||
#[derive(FromArgs)]
|
||||
/// Delete an item from the database
|
||||
#[argh(subcommand, name = "crawl")]
|
||||
struct Crawl {}
|
||||
|
||||
#[tokio::main]
|
||||
pub async fn main() -> Result<()> {
|
||||
@@ -53,13 +98,47 @@ pub async fn main() -> Result<()> {
|
||||
|
||||
let args: Args = argh::from_env();
|
||||
|
||||
if let Commands::AddFeed(add_feed_args) = args.commands {
|
||||
add_feed(pool, CreateFeed {
|
||||
title: add_feed_args.title,
|
||||
url: add_feed_args.url,
|
||||
feed_type: add_feed_args.feed_type,
|
||||
description: add_feed_args.description,
|
||||
}).await?;
|
||||
info!("hello?");
|
||||
|
||||
match args.commands {
|
||||
Commands::AddFeed(args) => {
|
||||
let feed = create_feed(
|
||||
&pool,
|
||||
CreateFeed {
|
||||
title: args.title,
|
||||
url: args.url,
|
||||
feed_type: args.feed_type,
|
||||
description: args.description,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
info!("Created feed with id {}", feed.id);
|
||||
}
|
||||
Commands::DeleteFeed(args) => {
|
||||
delete_feed(&pool, args.id).await?;
|
||||
info!("Deleted feed with id {}", args.id);
|
||||
}
|
||||
Commands::AddItem(args) => {
|
||||
let item = create_item(
|
||||
&pool,
|
||||
CreateItem {
|
||||
title: args.title,
|
||||
url: args.url,
|
||||
description: args.description,
|
||||
feed_id: args.feed_id,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
info!("Created item with id {}", item.id);
|
||||
}
|
||||
Commands::DeleteItem(args) => {
|
||||
delete_item(&pool, args.id).await?;
|
||||
info!("Deleted item with id {}", args.id);
|
||||
}
|
||||
Commands::Crawl(_) => {
|
||||
info!("Crawling...");
|
||||
crawl(&pool).await?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
||||
Reference in New Issue
Block a user