diff --git a/.gitignore b/.gitignore index b5ed0f2..3ad68c1 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ .env plugins.zip plugins -cells \ No newline at end of file +cells +mods \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 5e64de4..82fa4fa 100644 --- a/src/main.rs +++ b/src/main.rs @@ -51,6 +51,10 @@ struct Args { /// folder to output all cell data as json files #[argh(option, short = 'c')] cell_data: Option, + + /// folder to output all mod data as json files + #[argh(option, short = 'm')] + mod_data: Option, } async fn extract_with_compress_tools( @@ -243,6 +247,26 @@ pub async fn main() -> Result<()> { return Ok(()); } + if let Some(mod_data_dir) = args.mod_data { + let page_size = 20; + let mut page = 0; + loop { + let mods = game_mod::batched_get_with_cells(&pool, page_size, page).await?; + if mods.is_empty() { + break; + } + for mod_with_cells in mods { + let path = std::path::Path::new(&mod_data_dir); + std::fs::create_dir_all(&path)?; + let path = path.join(format!("{}.json", mod_with_cells.nexus_mod_id)); + let mut file = std::fs::File::create(path)?; + write!(file, "{}", serde_json::to_string(&mod_with_cells)?)?; + } + page += 1; + } + return Ok(()); + } + let mut page = args.page; let mut has_next_page = true; diff --git a/src/models/game_mod.rs b/src/models/game_mod.rs index d67871e..2757d60 100644 --- a/src/models/game_mod.rs +++ b/src/models/game_mod.rs @@ -42,6 +42,26 @@ pub struct UnsavedMod<'a> { pub first_upload_at: NaiveDateTime, } +#[derive(Debug, Serialize, Deserialize, FromRow)] +pub struct ModWithCells { + pub id: i32, + pub name: String, + pub nexus_mod_id: i32, + pub author_name: String, + pub author_id: i32, + pub category_name: Option, + pub category_id: Option, + pub description: Option, + pub thumbnail_link: Option, + pub game_id: i32, + pub updated_at: NaiveDateTime, + pub created_at: NaiveDateTime, + pub last_update_at: NaiveDateTime, + pub first_upload_at: NaiveDateTime, + pub last_updated_files_at: Option, + pub cells: Option, +} + #[instrument(level = "debug", skip(pool))] pub async fn get_by_nexus_mod_id( pool: &sqlx::Pool, @@ -297,3 +317,31 @@ pub async fn update_from_api_response<'a>( Ok(ret) } + +#[instrument(level = "debug", skip(pool))] +pub async fn batched_get_with_cells( + pool: &sqlx::Pool, + page_size: i64, + page: i64, +) -> Result> { + let offset = page_size * page; + sqlx::query_as!( + ModWithCells, + "SELECT + mods.*, + json_agg(cells.*) AS cells + FROM mods + JOIN files ON files.mod_id = mods.id + JOIN plugins ON plugins.file_id = files.id + JOIN plugin_cells ON plugin_cells.plugin_id = plugins.id + JOIN cells ON cells.id = plugin_cells.cell_id + GROUP BY mods.id + LIMIT $1 + OFFSET $2", + page_size, + offset, + ) + .fetch_all(pool) + .await + .context("Failed to batch get with cells") +}