Add a new command to dump files to json files

This commit is contained in:
Tyler Hallada 2022-06-01 00:53:53 -04:00
parent f59a0155d8
commit bed49d0953
6 changed files with 99 additions and 3 deletions

View File

@ -3,3 +3,4 @@ export $(grep -v '^#' .env | xargs -d '\n')
rsync -raz -e "ssh -p ${STATIC_SERVER_PORT}" cells ${STATIC_SERVER_USER}@${STATIC_SERVER_HOST}:/srv/
rsync -raz -e "ssh -p ${STATIC_SERVER_PORT}" mods ${STATIC_SERVER_USER}@${STATIC_SERVER_HOST}:/srv/
rsync -raz -e "ssh -p ${STATIC_SERVER_PORT}" plugins_data ${STATIC_SERVER_USER}@${STATIC_SERVER_HOST}:/srv/
rsync -raz -e "ssh -p ${STATIC_SERVER_PORT}" files ${STATIC_SERVER_USER}@${STATIC_SERVER_HOST}:/srv/

View File

@ -3,6 +3,7 @@ mkdir -p logs
./target/release/mod-mapper &>> logs/modmapper.log
mkdir -p cells
mkdir -p mods
mkdir -p files
mkdir -p plugins_data
./target/release/mod-mapper -e cells/edits.json
./target/release/mod-mapper -c cells
@ -10,3 +11,4 @@ mkdir -p plugins_data
./target/release/mod-mapper -M mods/mod_cell_counts.json
./target/release/mod-mapper -m mods
./target/release/mod-mapper -P plugins_data
./target/release/mod-mapper -F files

View File

@ -0,0 +1,27 @@
use anyhow::Result;
use std::fs::File;
use std::io::Write;
use std::path::Path;
use crate::models::file;
pub async fn dump_file_data(pool: &sqlx::Pool<sqlx::Postgres>, dir: &str) -> Result<()> {
let page_size = 20;
let mut last_id = None;
loop {
let files =
file::batched_get_with_cells(&pool, page_size, last_id, "Skyrim.esm", 1).await?;
if files.is_empty() {
break;
}
for file_with_cells in files {
let path = Path::new(&dir);
std::fs::create_dir_all(&path)?;
let path = path.join(format!("{}.json", file_with_cells.nexus_file_id));
let mut file = File::create(path)?;
write!(file, "{}", serde_json::to_string(&file_with_cells)?)?;
last_id = Some(file_with_cells.id);
}
}
return Ok(());
}

View File

@ -6,6 +6,7 @@ pub mod dump_mod_cell_counts;
pub mod dump_mod_data;
pub mod dump_mod_search_index;
pub mod dump_plugin_data;
pub mod dump_file_data;
pub mod update;
pub use download_tiles::download_tiles;
@ -15,4 +16,5 @@ pub use dump_mod_cell_counts::dump_mod_cell_counts;
pub use dump_mod_data::dump_mod_data;
pub use dump_mod_search_index::dump_mod_search_index;
pub use dump_plugin_data::dump_plugin_data;
pub use dump_file_data::dump_file_data;
pub use update::update;

View File

@ -13,7 +13,7 @@ mod plugin_processor;
use commands::{
backfills::backfill_is_translation, download_tiles, dump_cell_data, dump_cell_edit_counts,
dump_mod_cell_counts, dump_mod_data, dump_mod_search_index, dump_plugin_data, update,
dump_mod_cell_counts, dump_mod_data, dump_mod_search_index, dump_plugin_data, dump_file_data, update,
};
#[derive(FromArgs)]
@ -51,6 +51,10 @@ struct Args {
#[argh(option, short = 'P')]
plugin_data: Option<String>,
/// folder to output all files data as json files
#[argh(option, short = 'F')]
file_data: Option<String>,
/// folder to output all map tile images downloaded from the UESP wiki
#[argh(option, short = 't')]
download_tiles: Option<String>,
@ -91,6 +95,9 @@ pub async fn main() -> Result<()> {
if let Some(path) = args.plugin_data {
return dump_plugin_data(&pool, &path).await;
}
if let Some(path) = args.file_data {
return dump_file_data(&pool, &path).await;
}
if let Some(dir) = args.download_tiles {
return download_tiles(&dir).await;
}

View File

@ -23,6 +23,29 @@ pub struct File {
pub unable_to_extract_plugins: bool,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct FileWithCells {
pub id: i32,
pub name: String,
pub file_name: String,
pub nexus_file_id: i32,
pub mod_id: i32,
pub category: Option<String>,
pub version: Option<String>,
pub mod_version: Option<String>,
pub size: i64,
pub uploaded_at: NaiveDateTime,
pub has_download_link: bool,
pub updated_at: NaiveDateTime,
pub created_at: NaiveDateTime,
pub downloaded_at: Option<NaiveDateTime>,
pub has_plugin: bool,
pub unable_to_extract_plugins: bool,
pub cells: Option<serde_json::Value>,
pub plugins: Option<serde_json::Value>,
pub plugin_count: Option<i64>,
}
#[derive(Debug)]
pub struct UnsavedFile<'a> {
pub name: &'a str,
@ -174,3 +197,37 @@ pub async fn update_unable_to_extract_plugins(
.await
.context("Failed to update file")
}
#[instrument(level = "debug", skip(pool))]
pub async fn batched_get_with_cells(
pool: &sqlx::Pool<sqlx::Postgres>,
page_size: i64,
last_id: Option<i32>,
master: &str,
world_id: i32,
) -> Result<Vec<FileWithCells>> {
let last_id = last_id.unwrap_or(0);
sqlx::query_as!(
FileWithCells,
"SELECT
files.*,
COALESCE(json_agg(DISTINCT jsonb_build_object('x', cells.x, 'y', cells.y)) FILTER (WHERE cells.x IS NOT NULL AND cells.y IS NOT NULL AND cells.master = $3 AND cells.world_id = $4), '[]') AS cells,
COALESCE(json_agg(DISTINCT jsonb_build_object('hash', plugins.hash, 'file_name', plugins.file_name)) FILTER (WHERE plugins.hash IS NOT NULL), '[]') AS plugins,
COUNT(plugins.*) AS plugin_count
FROM files
LEFT OUTER JOIN plugin_cells ON plugin_cells.file_id = files.id
LEFT OUTER JOIN cells ON cells.id = plugin_cells.cell_id
LEFT OUTER JOIN plugins ON plugins.file_id = files.id
WHERE files.id > $2
GROUP BY files.id
ORDER BY files.id ASC
LIMIT $1",
page_size,
last_id,
master,
world_id
)
.fetch_all(pool)
.await
.context("Failed to batch get with cells")
}