diff --git a/scripts/sync.sh b/scripts/sync.sh index 6fe3900..5d9682c 100644 --- a/scripts/sync.sh +++ b/scripts/sync.sh @@ -1,4 +1,5 @@ #!/bin/bash export $(grep -v '^#' .env | xargs -d '\n') rsync -raz -e "ssh -p ${STATIC_SERVER_PORT}" cells ${STATIC_SERVER_USER}@${STATIC_SERVER_HOST}:/srv/ -rsync -raz -e "ssh -p ${STATIC_SERVER_PORT}" mods ${STATIC_SERVER_USER}@${STATIC_SERVER_HOST}:/srv/ \ No newline at end of file +rsync -raz -e "ssh -p ${STATIC_SERVER_PORT}" mods ${STATIC_SERVER_USER}@${STATIC_SERVER_HOST}:/srv/ +rsync -raz -e "ssh -p ${STATIC_SERVER_PORT}" plugins_data ${STATIC_SERVER_USER}@${STATIC_SERVER_HOST}:/srv/ \ No newline at end of file diff --git a/scripts/update.sh b/scripts/update.sh index 9f9061a..42e9804 100644 --- a/scripts/update.sh +++ b/scripts/update.sh @@ -3,7 +3,9 @@ mkdir -p logs ./target/release/mod-mapper &>> logs/modmapper.log mkdir -p cells mkdir -p mods +mkdir -p plugins_data ./target/release/mod-mapper -e cells/edits.json ./target/release/mod-mapper -c cells ./target/release/mod-mapper -s mods/mod_search_index.json -./target/release/mod-mapper -m mods \ No newline at end of file +./target/release/mod-mapper -m mods +./target/release/mod-mapper -P plugins_data \ No newline at end of file diff --git a/src/commands/dump_plugin_data.rs b/src/commands/dump_plugin_data.rs new file mode 100644 index 0000000..62cc654 --- /dev/null +++ b/src/commands/dump_plugin_data.rs @@ -0,0 +1,42 @@ +use anyhow::Result; +use std::fs::{create_dir_all, File}; +use std::io::Write; +use std::path::Path; + +use crate::models::plugin; + +// From: https://stackoverflow.com/a/50278316/6620612 +fn format_radix(mut x: u64, radix: u32) -> String { + let mut result = vec![]; + loop { + let m = x % radix as u64; + x = x / radix as u64; + + // will panic if you use a bad radix (< 2 or > 36). + result.push(std::char::from_digit(m as u32, radix).unwrap()); + if x == 0 { + break; + } + } + result.into_iter().rev().collect() +} + +pub async fn dump_plugin_data(pool: &sqlx::Pool, dir: &str) -> Result<()> { + let page_size = 20; + let mut last_id = None; + loop { + let plugins = plugin::batched_get_with_file_and_mod(&pool, page_size, last_id).await?; + if plugins.is_empty() { + break; + } + for plugin in plugins { + let path = Path::new(&dir); + create_dir_all(&path)?; + let path = path.join(format!("{}.json", format_radix(plugin.hash as u64, 36))); + let mut file = File::create(path)?; + write!(file, "{}", serde_json::to_string(&plugin)?)?; + last_id = Some(plugin.id); + } + } + return Ok(()); +} diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 463869b..f70e359 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -3,6 +3,7 @@ pub mod dump_cell_data; pub mod dump_cell_edit_counts; pub mod dump_mod_data; pub mod dump_mod_search_index; +pub mod dump_plugin_data; pub mod update; pub use download_tiles::download_tiles; @@ -10,4 +11,5 @@ pub use dump_cell_data::dump_cell_data; pub use dump_cell_edit_counts::dump_cell_edit_counts; pub use dump_mod_data::dump_mod_data; pub use dump_mod_search_index::dump_mod_search_index; +pub use dump_plugin_data::dump_plugin_data; pub use update::update; diff --git a/src/main.rs b/src/main.rs index 9930118..f63915d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -13,7 +13,7 @@ mod plugin_processor; use commands::{ download_tiles, dump_cell_data, dump_cell_edit_counts, dump_mod_data, dump_mod_search_index, - update, + dump_plugin_data, update, }; #[derive(FromArgs)] @@ -43,6 +43,10 @@ struct Args { #[argh(option, short = 's')] mod_search_index: Option, + /// folder to output all plugin data as json files + #[argh(option, short = 'P')] + plugin_data: Option, + /// folder to output all map tile images downloaded from the UESP wiki #[argh(option, short = 't')] download_tiles: Option, @@ -73,6 +77,9 @@ pub async fn main() -> Result<()> { if let Some(path) = args.mod_search_index { return dump_mod_search_index(&pool, &path).await; } + if let Some(path) = args.plugin_data { + return dump_plugin_data(&pool, &path).await; + } if let Some(dir) = args.download_tiles { return download_tiles(&dir).await; } diff --git a/src/models/plugin.rs b/src/models/plugin.rs index b13bf76..3f48da8 100644 --- a/src/models/plugin.rs +++ b/src/models/plugin.rs @@ -37,6 +37,26 @@ pub struct UnsavedPlugin<'a> { pub file_path: &'a str, } +#[derive(Debug, Serialize, Deserialize, FromRow)] +pub struct PluginWithFileAndMod { + pub id: i32, + pub name: String, + pub hash: i64, + pub file_id: i32, + pub mod_id: i32, + pub version: f64, + pub size: i64, + pub author: Option, + pub description: Option, + pub masters: Vec, + pub file_name: String, + pub file_path: String, + pub updated_at: NaiveDateTime, + pub created_at: NaiveDateTime, + pub file: Option, + pub r#mod: Option, +} + #[instrument(level = "debug", skip(pool))] pub async fn insert<'a>( pool: &sqlx::Pool, @@ -67,3 +87,31 @@ pub async fn insert<'a>( .await .context("Failed to insert plugin") } + +#[instrument(level = "debug", skip(pool))] +pub async fn batched_get_with_file_and_mod( + pool: &sqlx::Pool, + page_size: i64, + last_id: Option, +) -> Result> { + let last_id = last_id.unwrap_or(0); + sqlx::query_as!( + PluginWithFileAndMod, + "SELECT + plugins.*, + json_agg(files.*) as file, + json_agg(mods.*) as mod + FROM plugins + LEFT OUTER JOIN files ON files.id = plugins.file_id + LEFT OUTER JOIN mods ON mods.id = files.mod_id + WHERE plugins.id > $2 + GROUP BY plugins.id + ORDER BY plugins.id ASC + LIMIT $1", + page_size, + last_id + ) + .fetch_all(pool) + .await + .context("Failed to batch get with cells") +}