From 7a87f0fd2291d32ddcf2be6c9c2d7b6435073692 Mon Sep 17 00:00:00 2001 From: Tyler Hallada Date: Thu, 17 Mar 2022 23:51:32 -0400 Subject: [PATCH] Add command to generate mod_cell_counts.json So that I can display cell edit counts in mod lists efficiently. --- src/commands/dump_mod_cell_counts.rs | 26 +++++++++++++++++++ src/commands/mod.rs | 2 ++ src/main.rs | 9 ++++++- src/models/cell.rs | 17 +++++++++++-- src/models/game_mod.rs | 37 ++++++++++++++++++++++++++++ 5 files changed, 88 insertions(+), 3 deletions(-) create mode 100644 src/commands/dump_mod_cell_counts.rs diff --git a/src/commands/dump_mod_cell_counts.rs b/src/commands/dump_mod_cell_counts.rs new file mode 100644 index 0000000..5437645 --- /dev/null +++ b/src/commands/dump_mod_cell_counts.rs @@ -0,0 +1,26 @@ +use anyhow::Result; +use std::collections::HashMap; +use std::fs::File; +use std::io::Write; + +use crate::models::game_mod; + +pub async fn dump_mod_cell_counts(pool: &sqlx::Pool, path: &str) -> Result<()> { + let page_size = 100; + let mut last_id = None; + let mut counts = HashMap::new(); + loop { + let mod_cell_counts = + game_mod::batched_get_cell_counts(&pool, page_size, last_id, "Skyrim.esm", 1).await?; + if mod_cell_counts.is_empty() { + break; + } + for mod_cell_count in mod_cell_counts { + counts.insert(mod_cell_count.nexus_mod_id, mod_cell_count.cells); + last_id = Some(mod_cell_count.nexus_mod_id); + } + } + let mut file = File::create(path)?; + write!(file, "{}", serde_json::to_string(&counts)?)?; + return Ok(()); +} diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 0b5b3fb..cdc8e83 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -2,6 +2,7 @@ pub mod backfills; pub mod download_tiles; pub mod dump_cell_data; pub mod dump_cell_edit_counts; +pub mod dump_mod_cell_counts; pub mod dump_mod_data; pub mod dump_mod_search_index; pub mod dump_plugin_data; @@ -10,6 +11,7 @@ pub mod update; pub use download_tiles::download_tiles; pub use dump_cell_data::dump_cell_data; pub use dump_cell_edit_counts::dump_cell_edit_counts; +pub use dump_mod_cell_counts::dump_mod_cell_counts; pub use dump_mod_data::dump_mod_data; pub use dump_mod_search_index::dump_mod_search_index; pub use dump_plugin_data::dump_plugin_data; diff --git a/src/main.rs b/src/main.rs index 7933638..23037d9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -13,7 +13,7 @@ mod plugin_processor; use commands::{ backfills::backfill_is_translation, download_tiles, dump_cell_data, dump_cell_edit_counts, - dump_mod_data, dump_mod_search_index, dump_plugin_data, update, + dump_mod_cell_counts, dump_mod_data, dump_mod_search_index, dump_plugin_data, update, }; #[derive(FromArgs)] @@ -43,6 +43,10 @@ struct Args { #[argh(option, short = 's')] mod_search_index: Option, + /// file to output all mod cell edit counts and ids as a json index + #[argh(option, short = 'M')] + mod_cell_counts: Option, + /// folder to output all plugin data as json files #[argh(option, short = 'P')] plugin_data: Option, @@ -81,6 +85,9 @@ pub async fn main() -> Result<()> { if let Some(path) = args.mod_search_index { return dump_mod_search_index(&pool, &path).await; } + if let Some(path) = args.mod_cell_counts { + return dump_mod_cell_counts(&pool, &path).await; + } if let Some(path) = args.plugin_data { return dump_plugin_data(&pool, &path).await; } diff --git a/src/models/cell.rs b/src/models/cell.rs index 9616be7..7f0e736 100644 --- a/src/models/cell.rs +++ b/src/models/cell.rs @@ -39,6 +39,7 @@ pub struct CellData { pub files_count: Option, pub mods_count: Option, pub mods: Option, + pub mod_cell_count: Option, } #[instrument(level = "debug", skip(pool))] @@ -162,13 +163,25 @@ pub async fn get_cell_data( COUNT(DISTINCT plugins.id) as plugins_count, COUNT(DISTINCT files.id) as files_count, COUNT(DISTINCT mods.id) as mods_count, - json_agg(DISTINCT mods.*) as mods + json_agg(DISTINCT mods.*) as mods, + json_agg(DISTINCT mod_cell_counts.*) as mod_cell_count FROM cells JOIN plugin_cells on cells.id = cell_id JOIN plugins ON plugins.id = plugin_id JOIN files ON files.id = plugins.file_id JOIN mods ON mods.id = files.mod_id - WHERE master = $1 AND world_id = $2 AND x = $3 and y = $4 + CROSS JOIN LATERAL ( + SELECT + m.id, + COUNT(cells.*) FILTER (WHERE mod_cells.x IS NOT NULL AND mod_cells.y IS NOT NULL AND mod_cells.master = $1 AND mod_cells.world_id = $2) AS cell_count + FROM mods m + LEFT OUTER JOIN plugin_cells mod_plugin_cells ON mod_plugin_cells.mod_id = m.id + LEFT OUTER JOIN cells mod_cells ON mod_cells.id = mod_plugin_cells.cell_id + WHERE m.id = mods.id + GROUP BY m.id + ORDER BY m.id ASC + ) mod_cell_counts + WHERE cells.master = $1 AND cells.world_id = $2 AND cells.x = $3 and cells.y = $4 GROUP BY cells.x, cells.y, cells.is_persistent, cells.form_id"#, master, world_id, diff --git a/src/models/game_mod.rs b/src/models/game_mod.rs index 95d019a..20bc5d3 100644 --- a/src/models/game_mod.rs +++ b/src/models/game_mod.rs @@ -72,6 +72,12 @@ pub struct ModWithCells { pub cells: Option, } +#[derive(Debug, Serialize, Deserialize, FromRow)] +pub struct ModCellCount { + pub nexus_mod_id: i32, + pub cells: Option, +} + #[instrument(level = "debug", skip(pool))] pub async fn get_by_nexus_mod_id( pool: &sqlx::Pool, @@ -376,3 +382,34 @@ pub async fn batched_get_with_cells( .await .context("Failed to batch get with cells") } + +#[instrument(level = "debug", skip(pool))] +pub async fn batched_get_cell_counts( + pool: &sqlx::Pool, + page_size: i64, + last_id: Option, + master: &str, + world_id: i32, +) -> Result> { + let last_id = last_id.unwrap_or(0); + sqlx::query_as!( + ModCellCount, + "SELECT + mods.nexus_mod_id, + COUNT(DISTINCT cells.*) FILTER (WHERE cells.x IS NOT NULL AND cells.y IS NOT NULL AND cells.master = $3 AND cells.world_id = $4) AS cells + FROM mods + INNER JOIN plugin_cells ON plugin_cells.mod_id = mods.id + INNER JOIN cells ON cells.id = plugin_cells.cell_id + WHERE mods.nexus_mod_id > $2 + GROUP BY mods.nexus_mod_id + ORDER BY mods.nexus_mod_id ASC + LIMIT $1", + page_size, + last_id, + master, + world_id + ) + .fetch_all(pool) + .await + .context("Failed to batch get mod cell counts") +}