Dump plugins data with file and mod data

scripts/sync.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
 export $(grep -v '^#' .env | xargs -d '\n')
 rsync -raz -e "ssh -p ${STATIC_SERVER_PORT}" cells ${STATIC_SERVER_USER}@${STATIC_SERVER_HOST}:/srv/
-rsync -raz -e "ssh -p ${STATIC_SERVER_PORT}" mods ${STATIC_SERVER_USER}@${STATIC_SERVER_HOST}:/srv/
+rsync -raz -e "ssh -p ${STATIC_SERVER_PORT}" mods ${STATIC_SERVER_USER}@${STATIC_SERVER_HOST}:/srv/
+rsync -raz -e "ssh -p ${STATIC_SERVER_PORT}" plugins_data ${STATIC_SERVER_USER}@${STATIC_SERVER_HOST}:/srv/

scripts/update.sh

@@ -3,7 +3,9 @@ mkdir -p logs
 ./target/release/mod-mapper &>> logs/modmapper.log
 mkdir -p cells
 mkdir -p mods
+mkdir -p plugins_data
 ./target/release/mod-mapper -e cells/edits.json
 ./target/release/mod-mapper -c cells
 ./target/release/mod-mapper -s mods/mod_search_index.json
-./target/release/mod-mapper -m mods
+./target/release/mod-mapper -m mods
+./target/release/mod-mapper -P plugins_data

src/commands/dump_plugin_data.rs

@@ -0,0 +1,42 @@
+use anyhow::Result;
+use std::fs::{create_dir_all, File};
+use std::io::Write;
+use std::path::Path;
+
+use crate::models::plugin;
+
+// From: https://stackoverflow.com/a/50278316/6620612
+fn format_radix(mut x: u64, radix: u32) -> String {
+    let mut result = vec![];
+    loop {
+        let m = x % radix as u64;
+        x = x / radix as u64;
+
+        // will panic if you use a bad radix (< 2 or > 36).
+        result.push(std::char::from_digit(m as u32, radix).unwrap());
+        if x == 0 {
+            break;
+        }
+    }
+    result.into_iter().rev().collect()
+}
+
+pub async fn dump_plugin_data(pool: &sqlx::Pool<sqlx::Postgres>, dir: &str) -> Result<()> {
+    let page_size = 20;
+    let mut last_id = None;
+    loop {
+        let plugins = plugin::batched_get_with_file_and_mod(&pool, page_size, last_id).await?;
+        if plugins.is_empty() {
+            break;
+        }
+        for plugin in plugins {
+            let path = Path::new(&dir);
+            create_dir_all(&path)?;
+            let path = path.join(format!("{}.json", format_radix(plugin.hash as u64, 36)));
+            let mut file = File::create(path)?;
+            write!(file, "{}", serde_json::to_string(&plugin)?)?;
+            last_id = Some(plugin.id);
+        }
+    }
+    return Ok(());
+}

src/commands/mod.rs

@@ -3,6 +3,7 @@ pub mod dump_cell_data;
 pub mod dump_cell_edit_counts;
 pub mod dump_mod_data;
 pub mod dump_mod_search_index;
+pub mod dump_plugin_data;
 pub mod update;
 
 pub use download_tiles::download_tiles;
@@ -10,4 +11,5 @@ pub use dump_cell_data::dump_cell_data;
 pub use dump_cell_edit_counts::dump_cell_edit_counts;
 pub use dump_mod_data::dump_mod_data;
 pub use dump_mod_search_index::dump_mod_search_index;
+pub use dump_plugin_data::dump_plugin_data;
 pub use update::update;

src/main.rs

@@ -13,7 +13,7 @@ mod plugin_processor;
 
 use commands::{
     download_tiles, dump_cell_data, dump_cell_edit_counts, dump_mod_data, dump_mod_search_index,
-    update,
+    dump_plugin_data, update,
 };
 
 #[derive(FromArgs)]
@@ -43,6 +43,10 @@ struct Args {
     #[argh(option, short = 's')]
     mod_search_index: Option<String>,
 
+    /// folder to output all plugin data as json files
+    #[argh(option, short = 'P')]
+    plugin_data: Option<String>,
+
     /// folder to output all map tile images downloaded from the UESP wiki
     #[argh(option, short = 't')]
     download_tiles: Option<String>,
@@ -73,6 +77,9 @@ pub async fn main() -> Result<()> {
     if let Some(path) = args.mod_search_index {
         return dump_mod_search_index(&pool, &path).await;
     }
+    if let Some(path) = args.plugin_data {
+        return dump_plugin_data(&pool, &path).await;
+    }
     if let Some(dir) = args.download_tiles {
         return download_tiles(&dir).await;
     }

src/models/plugin.rs

@@ -37,6 +37,26 @@ pub struct UnsavedPlugin<'a> {
     pub file_path: &'a str,
 }
 
+#[derive(Debug, Serialize, Deserialize, FromRow)]
+pub struct PluginWithFileAndMod {
+    pub id: i32,
+    pub name: String,
+    pub hash: i64,
+    pub file_id: i32,
+    pub mod_id: i32,
+    pub version: f64,
+    pub size: i64,
+    pub author: Option<String>,
+    pub description: Option<String>,
+    pub masters: Vec<String>,
+    pub file_name: String,
+    pub file_path: String,
+    pub updated_at: NaiveDateTime,
+    pub created_at: NaiveDateTime,
+    pub file: Option<serde_json::Value>,
+    pub r#mod: Option<serde_json::Value>,
+}
+
 #[instrument(level = "debug", skip(pool))]
 pub async fn insert<'a>(
     pool: &sqlx::Pool<sqlx::Postgres>,
@@ -67,3 +87,31 @@ pub async fn insert<'a>(
     .await
     .context("Failed to insert plugin")
 }
+
+#[instrument(level = "debug", skip(pool))]
+pub async fn batched_get_with_file_and_mod(
+    pool: &sqlx::Pool<sqlx::Postgres>,
+    page_size: i64,
+    last_id: Option<i32>,
+) -> Result<Vec<PluginWithFileAndMod>> {
+    let last_id = last_id.unwrap_or(0);
+    sqlx::query_as!(
+        PluginWithFileAndMod,
+        "SELECT
+            plugins.*,
+            json_agg(files.*) as file,
+            json_agg(mods.*) as mod
+        FROM plugins
+        LEFT OUTER JOIN files ON files.id = plugins.file_id
+        LEFT OUTER JOIN mods ON mods.id = files.mod_id
+        WHERE plugins.id > $2
+        GROUP BY plugins.id
+        ORDER BY plugins.id ASC
+        LIMIT $1",
+        page_size,
+        last_id
+    )
+    .fetch_all(pool)
+    .await
+    .context("Failed to batch get with cells")
+}