Add command to generate mod_cell_counts.json

So that I can display cell edit counts in mod lists efficiently.

src/commands/dump_mod_cell_counts.rs

@@ -0,0 +1,26 @@
+use anyhow::Result;
+use std::collections::HashMap;
+use std::fs::File;
+use std::io::Write;
+
+use crate::models::game_mod;
+
+pub async fn dump_mod_cell_counts(pool: &sqlx::Pool<sqlx::Postgres>, path: &str) -> Result<()> {
+    let page_size = 100;
+    let mut last_id = None;
+    let mut counts = HashMap::new();
+    loop {
+        let mod_cell_counts =
+            game_mod::batched_get_cell_counts(&pool, page_size, last_id, "Skyrim.esm", 1).await?;
+        if mod_cell_counts.is_empty() {
+            break;
+        }
+        for mod_cell_count in mod_cell_counts {
+            counts.insert(mod_cell_count.nexus_mod_id, mod_cell_count.cells);
+            last_id = Some(mod_cell_count.nexus_mod_id);
+        }
+    }
+    let mut file = File::create(path)?;
+    write!(file, "{}", serde_json::to_string(&counts)?)?;
+    return Ok(());
+}

src/commands/mod.rs

@@ -2,6 +2,7 @@ pub mod backfills;
 pub mod download_tiles;
 pub mod dump_cell_data;
 pub mod dump_cell_edit_counts;
+pub mod dump_mod_cell_counts;
 pub mod dump_mod_data;
 pub mod dump_mod_search_index;
 pub mod dump_plugin_data;
@@ -10,6 +11,7 @@ pub mod update;
 pub use download_tiles::download_tiles;
 pub use dump_cell_data::dump_cell_data;
 pub use dump_cell_edit_counts::dump_cell_edit_counts;
+pub use dump_mod_cell_counts::dump_mod_cell_counts;
 pub use dump_mod_data::dump_mod_data;
 pub use dump_mod_search_index::dump_mod_search_index;
 pub use dump_plugin_data::dump_plugin_data;

src/main.rs

@@ -13,7 +13,7 @@ mod plugin_processor;
 
 use commands::{
     backfills::backfill_is_translation, download_tiles, dump_cell_data, dump_cell_edit_counts,
-    dump_mod_data, dump_mod_search_index, dump_plugin_data, update,
+    dump_mod_cell_counts, dump_mod_data, dump_mod_search_index, dump_plugin_data, update,
 };
 
 #[derive(FromArgs)]
@@ -43,6 +43,10 @@ struct Args {
     #[argh(option, short = 's')]
     mod_search_index: Option<String>,
 
+    /// file to output all mod cell edit counts and ids as a json index
+    #[argh(option, short = 'M')]
+    mod_cell_counts: Option<String>,
+
     /// folder to output all plugin data as json files
     #[argh(option, short = 'P')]
     plugin_data: Option<String>,
@@ -81,6 +85,9 @@ pub async fn main() -> Result<()> {
     if let Some(path) = args.mod_search_index {
         return dump_mod_search_index(&pool, &path).await;
     }
+    if let Some(path) = args.mod_cell_counts {
+        return dump_mod_cell_counts(&pool, &path).await;
+    }
     if let Some(path) = args.plugin_data {
         return dump_plugin_data(&pool, &path).await;
     }

src/models/cell.rs

@@ -39,6 +39,7 @@ pub struct CellData {
     pub files_count: Option<i64>,
     pub mods_count: Option<i64>,
     pub mods: Option<serde_json::Value>,
+    pub mod_cell_count: Option<serde_json::Value>,
 }
 
 #[instrument(level = "debug", skip(pool))]
@@ -162,13 +163,25 @@ pub async fn get_cell_data(
                 COUNT(DISTINCT plugins.id) as plugins_count,
                 COUNT(DISTINCT files.id) as files_count,
                 COUNT(DISTINCT mods.id) as mods_count,
-                json_agg(DISTINCT mods.*) as mods
+                json_agg(DISTINCT mods.*) as mods,
+                json_agg(DISTINCT mod_cell_counts.*) as mod_cell_count
             FROM cells
             JOIN plugin_cells on cells.id = cell_id
             JOIN plugins ON plugins.id = plugin_id
             JOIN files ON files.id = plugins.file_id
             JOIN mods ON mods.id = files.mod_id
-            WHERE master = $1 AND world_id = $2 AND x = $3 and y = $4
+            CROSS JOIN LATERAL (
+                SELECT
+                    m.id,
+                    COUNT(cells.*) FILTER (WHERE mod_cells.x IS NOT NULL AND mod_cells.y IS NOT NULL AND mod_cells.master = $1 AND mod_cells.world_id = $2) AS cell_count
+                FROM mods m
+                LEFT OUTER JOIN plugin_cells mod_plugin_cells ON mod_plugin_cells.mod_id = m.id
+                LEFT OUTER JOIN cells mod_cells ON mod_cells.id = mod_plugin_cells.cell_id
+                WHERE m.id = mods.id
+                GROUP BY m.id
+                ORDER BY m.id ASC
+            ) mod_cell_counts
+            WHERE cells.master = $1 AND cells.world_id = $2 AND cells.x = $3 and cells.y = $4
             GROUP BY cells.x, cells.y, cells.is_persistent, cells.form_id"#,
         master,
         world_id,

src/models/game_mod.rs

@@ -72,6 +72,12 @@ pub struct ModWithCells {
     pub cells: Option<serde_json::Value>,
 }
 
+#[derive(Debug, Serialize, Deserialize, FromRow)]
+pub struct ModCellCount {
+    pub nexus_mod_id: i32,
+    pub cells: Option<i64>,
+}
+
 #[instrument(level = "debug", skip(pool))]
 pub async fn get_by_nexus_mod_id(
     pool: &sqlx::Pool<sqlx::Postgres>,
@@ -376,3 +382,34 @@ pub async fn batched_get_with_cells(
     .await
     .context("Failed to batch get with cells")
 }
+
+#[instrument(level = "debug", skip(pool))]
+pub async fn batched_get_cell_counts(
+    pool: &sqlx::Pool<sqlx::Postgres>,
+    page_size: i64,
+    last_id: Option<i32>,
+    master: &str,
+    world_id: i32,
+) -> Result<Vec<ModCellCount>> {
+    let last_id = last_id.unwrap_or(0);
+    sqlx::query_as!(
+        ModCellCount,
+        "SELECT
+            mods.nexus_mod_id,
+            COUNT(DISTINCT cells.*) FILTER (WHERE cells.x IS NOT NULL AND cells.y IS NOT NULL AND cells.master = $3 AND cells.world_id = $4) AS cells
+        FROM mods
+        INNER JOIN plugin_cells ON plugin_cells.mod_id = mods.id
+        INNER JOIN cells ON cells.id = plugin_cells.cell_id
+        WHERE mods.nexus_mod_id > $2
+        GROUP BY mods.nexus_mod_id
+        ORDER BY mods.nexus_mod_id ASC
+        LIMIT $1",
+        page_size,
+        last_id,
+        master,
+        world_id
+    )
+    .fetch_all(pool)
+    .await
+    .context("Failed to batch get mod cell counts")
+}