Add a new command to dump files to json files

scripts/sync.sh

@@ -2,4 +2,5 @@
 export $(grep -v '^#' .env | xargs -d '\n')
 rsync -raz -e "ssh -p ${STATIC_SERVER_PORT}" cells ${STATIC_SERVER_USER}@${STATIC_SERVER_HOST}:/srv/
 rsync -raz -e "ssh -p ${STATIC_SERVER_PORT}" mods ${STATIC_SERVER_USER}@${STATIC_SERVER_HOST}:/srv/
-rsync -raz -e "ssh -p ${STATIC_SERVER_PORT}" plugins_data ${STATIC_SERVER_USER}@${STATIC_SERVER_HOST}:/srv/
+rsync -raz -e "ssh -p ${STATIC_SERVER_PORT}" plugins_data ${STATIC_SERVER_USER}@${STATIC_SERVER_HOST}:/srv/
+rsync -raz -e "ssh -p ${STATIC_SERVER_PORT}" files ${STATIC_SERVER_USER}@${STATIC_SERVER_HOST}:/srv/

scripts/update.sh

@@ -3,10 +3,12 @@ mkdir -p logs
 ./target/release/mod-mapper &>> logs/modmapper.log
 mkdir -p cells
 mkdir -p mods
+mkdir -p files
 mkdir -p plugins_data
 ./target/release/mod-mapper -e cells/edits.json
 ./target/release/mod-mapper -c cells
 ./target/release/mod-mapper -s mods/mod_search_index.json
 ./target/release/mod-mapper -M mods/mod_cell_counts.json
 ./target/release/mod-mapper -m mods
-./target/release/mod-mapper -P plugins_data
+./target/release/mod-mapper -P plugins_data
+./target/release/mod-mapper -F files

src/commands/dump_file_data.rs

@@ -0,0 +1,27 @@
+use anyhow::Result;
+use std::fs::File;
+use std::io::Write;
+use std::path::Path;
+
+use crate::models::file;
+
+pub async fn dump_file_data(pool: &sqlx::Pool<sqlx::Postgres>, dir: &str) -> Result<()> {
+    let page_size = 20;
+    let mut last_id = None;
+    loop {
+        let files =
+            file::batched_get_with_cells(&pool, page_size, last_id, "Skyrim.esm", 1).await?;
+        if files.is_empty() {
+            break;
+        }
+        for file_with_cells in files {
+            let path = Path::new(&dir);
+            std::fs::create_dir_all(&path)?;
+            let path = path.join(format!("{}.json", file_with_cells.nexus_file_id));
+            let mut file = File::create(path)?;
+            write!(file, "{}", serde_json::to_string(&file_with_cells)?)?;
+            last_id = Some(file_with_cells.id);
+        }
+    }
+    return Ok(());
+}

src/commands/mod.rs

@@ -6,6 +6,7 @@ pub mod dump_mod_cell_counts;
 pub mod dump_mod_data;
 pub mod dump_mod_search_index;
 pub mod dump_plugin_data;
+pub mod dump_file_data;
 pub mod update;
 
 pub use download_tiles::download_tiles;
@@ -15,4 +16,5 @@ pub use dump_mod_cell_counts::dump_mod_cell_counts;
 pub use dump_mod_data::dump_mod_data;
 pub use dump_mod_search_index::dump_mod_search_index;
 pub use dump_plugin_data::dump_plugin_data;
+pub use dump_file_data::dump_file_data;
 pub use update::update;

src/main.rs

@@ -13,7 +13,7 @@ mod plugin_processor;
 
 use commands::{
     backfills::backfill_is_translation, download_tiles, dump_cell_data, dump_cell_edit_counts,
-    dump_mod_cell_counts, dump_mod_data, dump_mod_search_index, dump_plugin_data, update,
+    dump_mod_cell_counts, dump_mod_data, dump_mod_search_index, dump_plugin_data, dump_file_data, update,
 };
 
 #[derive(FromArgs)]
@@ -51,6 +51,10 @@ struct Args {
     #[argh(option, short = 'P')]
     plugin_data: Option<String>,
 
+    /// folder to output all files data as json files
+    #[argh(option, short = 'F')]
+    file_data: Option<String>,
+
     /// folder to output all map tile images downloaded from the UESP wiki
     #[argh(option, short = 't')]
     download_tiles: Option<String>,
@@ -91,6 +95,9 @@ pub async fn main() -> Result<()> {
     if let Some(path) = args.plugin_data {
         return dump_plugin_data(&pool, &path).await;
     }
+    if let Some(path) = args.file_data {
+        return dump_file_data(&pool, &path).await;
+    }
     if let Some(dir) = args.download_tiles {
         return download_tiles(&dir).await;
     }

src/models/file.rs

@@ -23,6 +23,29 @@ pub struct File {
     pub unable_to_extract_plugins: bool,
 }
 
+#[derive(Debug, Serialize, Deserialize)]
+pub struct FileWithCells {
+    pub id: i32,
+    pub name: String,
+    pub file_name: String,
+    pub nexus_file_id: i32,
+    pub mod_id: i32,
+    pub category: Option<String>,
+    pub version: Option<String>,
+    pub mod_version: Option<String>,
+    pub size: i64,
+    pub uploaded_at: NaiveDateTime,
+    pub has_download_link: bool,
+    pub updated_at: NaiveDateTime,
+    pub created_at: NaiveDateTime,
+    pub downloaded_at: Option<NaiveDateTime>,
+    pub has_plugin: bool,
+    pub unable_to_extract_plugins: bool,
+    pub cells: Option<serde_json::Value>,
+    pub plugins: Option<serde_json::Value>,
+    pub plugin_count: Option<i64>,
+}
+
 #[derive(Debug)]
 pub struct UnsavedFile<'a> {
     pub name: &'a str,
@@ -174,3 +197,37 @@ pub async fn update_unable_to_extract_plugins(
     .await
     .context("Failed to update file")
 }
+
+#[instrument(level = "debug", skip(pool))]
+pub async fn batched_get_with_cells(
+    pool: &sqlx::Pool<sqlx::Postgres>,
+    page_size: i64,
+    last_id: Option<i32>,
+    master: &str,
+    world_id: i32,
+) -> Result<Vec<FileWithCells>> {
+    let last_id = last_id.unwrap_or(0);
+    sqlx::query_as!(
+        FileWithCells,
+        "SELECT
+            files.*,
+            COALESCE(json_agg(DISTINCT jsonb_build_object('x', cells.x, 'y', cells.y)) FILTER (WHERE cells.x IS NOT NULL AND cells.y IS NOT NULL AND cells.master = $3 AND cells.world_id = $4), '[]') AS cells,
+            COALESCE(json_agg(DISTINCT jsonb_build_object('hash', plugins.hash, 'file_name', plugins.file_name)) FILTER (WHERE plugins.hash IS NOT NULL), '[]') AS plugins,
+            COUNT(plugins.*) AS plugin_count
+        FROM files
+        LEFT OUTER JOIN plugin_cells ON plugin_cells.file_id = files.id
+        LEFT OUTER JOIN cells ON cells.id = plugin_cells.cell_id
+        LEFT OUTER JOIN plugins ON plugins.file_id = files.id
+        WHERE files.id > $2
+        GROUP BY files.id
+        ORDER BY files.id ASC
+        LIMIT $1",
+        page_size,
+        last_id,
+        master,
+        world_id
+    )
+    .fetch_all(pool)
+    .await
+    .context("Failed to batch get with cells")
+}