thallada/modmapper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update deps, rustfmt, clippy fixes

Browse Source
This commit is contained in:
Tyler Hallada
2023-04-20 00:47:14 -04:00
parent 03cf20029e
commit 01444cd928
22 changed files with 583 additions and 259 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/commands/backfills/is_translation.rs
View File

@@ -3,8 +3,8 @@ use std::time::Duration;
use tokio::time::sleep;
use tracing::{debug, info, info_span};
use crate::nexus_scraper;
use crate::nexus_api::SSE_GAME_ID;
use crate::nexus_scraper;
const REQUEST_TIMEOUT: Duration = Duration::from_secs(7200); // 2 hours
const CONNECT_TIMEOUT: Duration = Duration::from_secs(30);
@@ -25,7 +25,8 @@ pub async fn backfill_is_translation(pool: &sqlx::Pool<sqlx::Postgres>) -> Resul
while has_next_page {
let page_span = info_span!("page", page);
let _page_span = page_span.enter();
let mod_list_resp = nexus_scraper::get_mod_list_page(&client, page, SSE_GAME_ID, true).await?;
let mod_list_resp =
nexus_scraper::get_mod_list_page(&client, page, SSE_GAME_ID, true).await?;
let scraped = mod_list_resp.scrape_mods()?;
let scraped_ids: Vec<i32> = scraped.mods.iter().map(|m| m.nexus_mod_id).collect();

14
src/commands/dump_cell_data.rs
View File

@@ -9,16 +9,22 @@ use crate::models::cell;
pub async fn dump_cell_data(pool: &sqlx::Pool<sqlx::Postgres>, dir: &str) -> Result<()> {
for x in -77..75 {
for y in -50..44 {
if let Ok(data) = cell::get_cell_data(&pool, "Skyrim.esm", 1, x, y).await {
if let Ok(data) = cell::get_cell_data(pool, "Skyrim.esm", 1, x, y).await {
let path = format!("{}/{}", &dir, x);
let path = Path::new(&path);
create_dir_all(&path)?;
create_dir_all(path)?;
let path = path.join(format!("{}.json", y));
info!(x = x, y = y, form_id = data.form_id, "dumping cell data to {}", path.display());
info!(
x = x,
y = y,
form_id = data.form_id,
"dumping cell data to {}",
path.display()
);
let mut file = File::create(path)?;
write!(file, "{}", serde_json::to_string(&data)?)?;
}
}
}
return Ok(());
Ok(())
}

10
src/commands/dump_cell_edit_counts.rs
View File

@@ -10,14 +10,18 @@ pub async fn dump_cell_edit_counts(pool: &sqlx::Pool<sqlx::Postgres>, path: &str
let mut cell_mod_edit_counts = HashMap::new();
for x in -77..75 {
for y in -50..44 {
if let Some(count) = cell::count_mod_edits(&pool, "Skyrim.esm", 1, x, y).await? {
if let Some(count) = cell::count_mod_edits(pool, "Skyrim.esm", 1, x, y).await? {
info!(x = x, y = y, count = count, "read cell edit count");
cell_mod_edit_counts.insert(format!("{},{}", x, y), count);
}
}
}
info!("writing {} cell edit counts to {}", cell_mod_edit_counts.len(), path);
info!(
"writing {} cell edit counts to {}",
cell_mod_edit_counts.len(),
path
);
let mut file = File::create(path)?;
write!(file, "{}", serde_json::to_string(&cell_mod_edit_counts)?)?;
return Ok(());
Ok(())
}

20
src/commands/dump_file_data.rs
View File

@@ -7,26 +7,36 @@ use tracing::info;
use crate::models::file;
pub async fn dump_file_data(pool: &sqlx::Pool<sqlx::Postgres>, dir: &str, updated_after: Option<NaiveDateTime>) -> Result<()> {
pub async fn dump_file_data(
pool: &sqlx::Pool<sqlx::Postgres>,
dir: &str,
updated_after: Option<NaiveDateTime>,
) -> Result<()> {
let mut page = 1;
let page_size = 20;
let mut last_id = None;
loop {
let files =
file::batched_get_with_cells(&pool, page_size, last_id, "Skyrim.esm", 1, updated_after).await?;
file::batched_get_with_cells(pool, page_size, last_id, "Skyrim.esm", 1, updated_after)
.await?;
if files.is_empty() {
break;
}
for file_with_cells in files {
let path = Path::new(&dir);
std::fs::create_dir_all(&path)?;
std::fs::create_dir_all(path)?;
let path = path.join(format!("{}.json", file_with_cells.nexus_file_id));
info!(page = page, nexus_file_id = file_with_cells.nexus_file_id, "dumping file data to {}", path.display());
info!(
page = page,
nexus_file_id = file_with_cells.nexus_file_id,
"dumping file data to {}",
path.display()
);
let mut file = File::create(path)?;
write!(file, "{}", serde_json::to_string(&file_with_cells)?)?;
last_id = Some(file_with_cells.id);
}
page += 1;
}
return Ok(());
Ok(())
}

4
src/commands/dump_games.rs
View File

@@ -6,9 +6,9 @@ use tracing::info;
use crate::models::game;
pub async fn dump_games(pool: &sqlx::Pool<sqlx::Postgres>, path: &str) -> Result<()> {
let games = game::get_all(&pool).await?;
let games = game::get_all(pool).await?;
info!("writing {} games to {}", games.len(), path);
let mut file = File::create(path)?;
write!(file, "{}", serde_json::to_string(&games)?)?;
return Ok(());
Ok(())
}

11
src/commands/dump_mod_cell_counts.rs
View File

@@ -13,12 +13,17 @@ pub async fn dump_mod_cell_counts(pool: &sqlx::Pool<sqlx::Postgres>, path: &str)
let mut counts = HashMap::new();
loop {
let mod_cell_counts =
game_mod::batched_get_cell_counts(&pool, page_size, last_id, "Skyrim.esm", 1).await?;
game_mod::batched_get_cell_counts(pool, page_size, last_id, "Skyrim.esm", 1).await?;
if mod_cell_counts.is_empty() {
break;
}
for mod_cell_count in mod_cell_counts {
info!(page = page, nexus_mod_id = mod_cell_count.nexus_mod_id, count = mod_cell_count.cells.unwrap_or(0), "read mod cell count");
info!(
page = page,
nexus_mod_id = mod_cell_count.nexus_mod_id,
count = mod_cell_count.cells.unwrap_or(0),
"read mod cell count"
);
counts.insert(mod_cell_count.nexus_mod_id, mod_cell_count.cells);
last_id = Some(mod_cell_count.nexus_mod_id);
}
@@ -27,5 +32,5 @@ pub async fn dump_mod_cell_counts(pool: &sqlx::Pool<sqlx::Postgres>, path: &str)
info!("writing {} mod cell counts to {}", counts.len(), path);
let mut file = File::create(path)?;
write!(file, "{}", serde_json::to_string(&counts)?)?;
return Ok(());
Ok(())
}

38
src/commands/dump_mod_data.rs
View File

@@ -9,27 +9,51 @@ use tracing::info;
use crate::models::game;
use crate::models::game_mod;
pub async fn dump_mod_data(pool: &sqlx::Pool<sqlx::Postgres>, dir: &str, updated_after: Option<NaiveDateTime>) -> Result<()> {
pub async fn dump_mod_data(
pool: &sqlx::Pool<sqlx::Postgres>,
dir: &str,
updated_after: Option<NaiveDateTime>,
) -> Result<()> {
let mut page = 1;
let page_size = 20;
let mut last_id = None;
let game_id_to_name: HashMap<_, _> = game::get_all(&pool).await?.into_iter().map(|game| (game.id, game.name)).collect();
let game_id_to_name: HashMap<_, _> = game::get_all(pool)
.await?
.into_iter()
.map(|game| (game.id, game.name))
.collect();
loop {
let mods =
game_mod::batched_get_with_cells_and_files(&pool, page_size, last_id, "Skyrim.esm", 1, updated_after).await?;
let mods = game_mod::batched_get_with_cells_and_files(
pool,
page_size,
last_id,
"Skyrim.esm",
1,
updated_after,
)
.await?;
if mods.is_empty() {
break;
}
for mod_with_cells in mods {
let path = Path::new(&dir).join(game_id_to_name.get(&mod_with_cells.game_id).expect("valid mod.game_id"));
let path = Path::new(&dir).join(
game_id_to_name
.get(&mod_with_cells.game_id)
.expect("valid mod.game_id"),
);
std::fs::create_dir_all(&path)?;
let path = path.join(format!("{}.json", mod_with_cells.nexus_mod_id));
info!(page = page, nexus_mod_id = mod_with_cells.nexus_mod_id, "dumping mod data to {}", path.display());
info!(
page = page,
nexus_mod_id = mod_with_cells.nexus_mod_id,
"dumping mod data to {}",
path.display()
);
let mut file = File::create(path)?;
write!(file, "{}", serde_json::to_string(&mod_with_cells)?)?;
last_id = Some(mod_with_cells.id);
}
page += 1;
}
return Ok(());
Ok(())
}

24
src/commands/dump_mod_search_index.rs
View File

@@ -13,19 +13,27 @@ struct ModForSearchIdTranslated {
id: i32,
}
pub async fn dump_mod_search_index(pool: &sqlx::Pool<sqlx::Postgres>, game: &str, path: &str) -> Result<()> {
pub async fn dump_mod_search_index(
pool: &sqlx::Pool<sqlx::Postgres>,
game: &str,
path: &str,
) -> Result<()> {
let mut page = 1;
let mut search_index = vec![];
let page_size = 20;
let mut last_id = None;
let game_id = game::get_id_by_name(&pool, game).await?;
let game_id = game::get_id_by_name(pool, game).await?;
loop {
let mods = game_mod::batched_get_for_search(&pool, game_id, page_size, last_id).await?;
let mods = game_mod::batched_get_for_search(pool, game_id, page_size, last_id).await?;
if mods.is_empty() {
break;
}
for mod_for_search in mods {
info!(page = page, nexus_mod_id = mod_for_search.nexus_mod_id, "read mod name for search index");
info!(
page = page,
nexus_mod_id = mod_for_search.nexus_mod_id,
"read mod name for search index"
);
search_index.push(ModForSearchIdTranslated {
name: mod_for_search.name,
id: mod_for_search.nexus_mod_id,
@@ -34,8 +42,12 @@ pub async fn dump_mod_search_index(pool: &sqlx::Pool<sqlx::Postgres>, game: &str
}
page += 1;
}
info!("writing {} mod names for search index to {}", search_index.len(), path);
info!(
"writing {} mod names for search index to {}",
search_index.len(),
path
);
let mut file = File::create(path)?;
write!(file, "{}", serde_json::to_string(&search_index)?)?;
return Ok(());
Ok(())
}

28
src/commands/dump_plugin_data.rs
View File

@@ -5,23 +5,39 @@ use std::io::Write;
use std::path::Path;
use tracing::info;
use crate::models::{plugin, format_radix};
use crate::models::{format_radix, plugin};
pub async fn dump_plugin_data(pool: &sqlx::Pool<sqlx::Postgres>, dir: &str, updated_after: Option<NaiveDateTime>) -> Result<()> {
pub async fn dump_plugin_data(
pool: &sqlx::Pool<sqlx::Postgres>,
dir: &str,
updated_after: Option<NaiveDateTime>,
) -> Result<()> {
let mut page: u32 = 1;
let page_size = 20;
let mut last_hash = None;
loop {
let plugins =
plugin::batched_get_by_hash_with_mods(pool, page_size, last_hash, "Skyrim.esm", 1, updated_after).await?;
let plugins = plugin::batched_get_by_hash_with_mods(
pool,
page_size,
last_hash,
"Skyrim.esm",
1,
updated_after,
)
.await?;
if plugins.is_empty() {
break;
}
for plugin in plugins {
let path = Path::new(&dir);
create_dir_all(&path)?;
create_dir_all(path)?;
let path = path.join(format!("{}.json", format_radix(plugin.hash as u64, 36)));
info!(page = page, hash = plugin.hash, "dumping plugin data to {}", path.display());
info!(
page = page,
hash = plugin.hash,
"dumping plugin data to {}",
path.display()
);
let mut file = File::create(path)?;
let json_val = serde_json::to_string(&plugin)?;
write!(file, "{}", json_val)?;

8
src/commands/mod.rs
View File

@@ -2,21 +2,21 @@ pub mod backfills;
pub mod download_tiles;
pub mod dump_cell_data;
pub mod dump_cell_edit_counts;
pub mod dump_file_data;
pub mod dump_games;
pub mod dump_mod_cell_counts;
pub mod dump_mod_data;
pub mod dump_mod_search_index;
pub mod dump_plugin_data;
pub mod dump_file_data;
pub mod dump_games;
pub mod update;
pub use download_tiles::download_tiles;
pub use dump_cell_data::dump_cell_data;
pub use dump_cell_edit_counts::dump_cell_edit_counts;
pub use dump_file_data::dump_file_data;
pub use dump_games::dump_games;
pub use dump_mod_cell_counts::dump_mod_cell_counts;
pub use dump_mod_data::dump_mod_data;
pub use dump_mod_search_index::dump_mod_search_index;
pub use dump_plugin_data::dump_plugin_data;
pub use dump_file_data::dump_file_data;
pub use dump_games::dump_games;
pub use update::update;

78
src/commands/update.rs
View File

@@ -1,6 +1,6 @@
use anyhow::Result;
use chrono::{NaiveDateTime, NaiveTime};
use humansize::{file_size_opts, FileSize};
use humansize::{format_size_i, DECIMAL};
use reqwest::StatusCode;
use std::collections::HashSet;
use std::io::SeekFrom;
@@ -36,7 +36,7 @@ pub async fn update(
.build()?;
let game_id = get_game_id(game_name).expect("valid game name");
let game = game::insert(&pool, game_name, game_id).await?;
let game = game::insert(pool, game_name, game_id).await?;
while has_next_page {
if !full && pages_with_no_updates >= 50 {
@@ -46,13 +46,18 @@ pub async fn update(
let page_span = info_span!("page", page, game_name, include_translations);
let _page_span = page_span.enter();
let mod_list_resp =
nexus_scraper::get_mod_list_page(&client, page, game.nexus_game_id, include_translations).await?;
let mod_list_resp = nexus_scraper::get_mod_list_page(
&client,
page,
game.nexus_game_id,
include_translations,
)
.await?;
let scraped = mod_list_resp.scrape_mods()?;
has_next_page = scraped.has_next_page;
let processed_mods = game_mod::bulk_get_last_updated_by_nexus_mod_ids(
&pool,
pool,
game.id,
&scraped
.mods
@@ -101,7 +106,7 @@ pub async fn update(
})
.collect();
let mods = game_mod::batched_insert(&pool, &mods_to_create_or_update).await?;
let mods = game_mod::batched_insert(pool, &mods_to_create_or_update).await?;
if mods.is_empty() {
pages_with_no_updates += 1;
@@ -112,7 +117,8 @@ pub async fn update(
for db_mod in mods {
let mod_span = info_span!("mod", name = ?&db_mod.name, id = &db_mod.nexus_mod_id);
let _mod_span = mod_span.enter();
let files_resp = nexus_api::files::get(&client, game_name, db_mod.nexus_mod_id).await?;
let files_resp =
nexus_api::files::get(&client, game_name, db_mod.nexus_mod_id).await?;
debug!(duration = ?files_resp.wait, "sleeping");
sleep(files_resp.wait).await;
@@ -135,7 +141,7 @@ pub async fn update(
});
let processed_file_ids: HashSet<i32> =
file::get_processed_nexus_file_ids_by_mod_id(&pool, db_mod.id)
file::get_processed_nexus_file_ids_by_mod_id(pool, db_mod.id)
.await?
.into_iter()
.collect();
@@ -150,7 +156,7 @@ pub async fn update(
continue;
}
let db_file = file::insert(
&pool,
pool,
&file::UnsavedFile {
name: api_file.name,
file_name: api_file.file_name,
@@ -174,7 +180,7 @@ pub async fn update(
info!(
"file metadata does not contain a plugin, skip downloading"
);
file::update_has_plugin(&pool, db_file.id, false).await?;
file::update_has_plugin(pool, db_file.id, false).await?;
continue;
}
} else {
@@ -186,10 +192,7 @@ pub async fn update(
}
};
let humanized_size = api_file
.size
.file_size(file_size_opts::CONVENTIONAL)
.expect("unable to create human-readable file size");
let humanized_size = format_size_i(api_file.size, DECIMAL);
info!(size = %humanized_size, "decided to download file");
let download_link_resp = nexus_api::download_link::get(
&client,
@@ -205,7 +208,7 @@ pub async fn update(
status = ?reqwest_err.status(),
"failed to get download link for file, skipping file"
);
file::update_has_download_link(&pool, db_file.id, false).await?;
file::update_has_download_link(pool, db_file.id, false).await?;
continue;
}
}
@@ -215,7 +218,7 @@ pub async fn update(
let mut tokio_file = match download_link_resp.download_file(&client).await {
Ok(file) => {
info!(bytes = api_file.size, "download finished");
file::update_downloaded_at(&pool, db_file.id).await?;
file::update_downloaded_at(pool, db_file.id).await?;
file
}
Err(err) => {
@@ -228,14 +231,14 @@ pub async fn update(
tokio_file.seek(SeekFrom::Start(0)).await?;
if let Err(err) = tokio_file.read_exact(&mut initial_bytes).await {
warn!(error = %err, "failed to read initial bytes, skipping file");
file::update_unable_to_extract_plugins(&pool, db_file.id, true).await?;
file::update_unable_to_extract_plugins(pool, db_file.id, true).await?;
continue;
}
let kind = match infer::get(&initial_bytes) {
Some(kind) => kind,
None => {
warn!(initial_bytes = ?initial_bytes, "unable to determine file type of archive, skipping file");
file::update_unable_to_extract_plugins(&pool, db_file.id, true).await?;
file::update_unable_to_extract_plugins(pool, db_file.id, true).await?;
continue;
}
};
@@ -253,7 +256,7 @@ pub async fn update(
let mut file = tokio_file.try_clone().await?.into_std().await;
match extract_with_unrar(
&mut file,
&pool,
pool,
&db_file,
&db_mod,
game_name,
@@ -266,7 +269,15 @@ pub async fn update(
// unrar failed to extract rar file (e.g. archive has unicode filenames)
// Attempt to uncompress the archive using `7z` unix command instead
warn!(error = %err, "failed to extract file with unrar, extracting whole archive with 7z instead");
extract_with_7zip(&mut file, &pool, &db_file, &db_mod, game_name, checked_metadata).await
extract_with_7zip(
&mut file,
pool,
&db_file,
&db_mod,
game_name,
checked_metadata,
)
.await
}
}?;
}
@@ -274,8 +285,10 @@ pub async fn update(
tokio_file.seek(SeekFrom::Start(0)).await?;
let mut file = tokio_file.try_clone().await?.into_std().await;
match extract_with_compress_tools(&mut file, &pool, &db_file, &db_mod, game_name)
.await
match extract_with_compress_tools(
&mut file, pool, &db_file, &db_mod, game_name,
)
.await
{
Ok(_) => Ok(()),
Err(err) => {
@@ -289,11 +302,24 @@ pub async fn update(
// compress_tools or libarchive failed to extract zip/7z file (e.g. archive is deflate64 compressed)
// Attempt to uncompress the archive using `7z` unix command instead
warn!(error = %err, "failed to extract file with compress_tools, extracting whole archive with 7z instead");
extract_with_7zip(&mut file, &pool, &db_file, &db_mod, game_name, checked_metadata).await
} else if kind.mime_type() == "application/vnd.microsoft.portable-executable" {
extract_with_7zip(
&mut file,
pool,
&db_file,
&db_mod,
game_name,
checked_metadata,
)
.await
} else if kind.mime_type()
== "application/vnd.microsoft.portable-executable"
{
// we tried to extract this .exe file, but it's not an archive so there's nothing we can do
warn!("archive is an .exe file that cannot be extracted, skipping file");
file::update_unable_to_extract_plugins(&pool, db_file.id, true).await?;
file::update_unable_to_extract_plugins(
pool, db_file.id, true,
)
.await?;
continue;
} else {
Err(err)
@@ -307,7 +333,7 @@ pub async fn update(
sleep(download_link_resp.wait).await;
}
game_mod::update_last_updated_files_at(&pool, db_mod.id).await?;
game_mod::update_last_updated_files_at(pool, db_mod.id).await?;
}
page += 1;