thallada/modmapper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Replace dbg macros with tracing events

Browse Source
This commit is contained in:
Tyler Hallada
2021-07-11 19:45:26 -04:00
parent 22757bc475
commit 792e78391c
13 changed files with 199 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
src/main.rs
View File

@@ -12,6 +12,7 @@ use std::time::Duration;
use tempfile::tempdir;
use tokio::io::{AsyncReadExt, AsyncSeekExt};
use tokio::time::sleep;
use tracing::{debug, error, info, instrument, warn};
use unrar::Archive;
use zip::write::{FileOptions, ZipWriter};
@@ -27,6 +28,7 @@ use models::plugin::insert_plugin;
use models::plugin_cell::insert_plugin_cell;
use nexus_api::{GAME_ID, GAME_NAME};
#[instrument(level = "debug", skip(plugin_buf, pool, plugin_archive, db_file, mod_obj), fields(name = ?mod_obj.name, id = mod_obj.nexus_mod_id))]
async fn process_plugin<W>(
plugin_buf: &mut [u8],
pool: &sqlx::Pool<sqlx::Postgres>,
@@ -39,6 +41,7 @@ where
W: std::io::Write + std::io::Seek,
{
let plugin = parse_plugin(&plugin_buf)?;
info!(file_name, num_cells = plugin.cells.len(), "parsed plugin");
let hash = seahash::hash(&plugin_buf);
let plugin_row = insert_plugin(
&pool,
@@ -100,6 +103,9 @@ fn initialize_plugins_archive(mod_id: i32, file_id: i32) -> Result<()> {
#[tokio::main]
pub async fn main() -> Result<()> {
dotenv().ok();
tracing_subscriber::fmt::init();
let pool = PgPoolOptions::new()
.max_connections(5)
.connect(&env::var("DATABASE_URL")?)
@@ -132,10 +138,13 @@ pub async fn main() -> Result<()> {
);
}
}
dbg!(mods.len());
for db_mod in mods {
dbg!(&db_mod.name);
info!(
mod_name = ?&db_mod.name,
mod_id = ?&db_mod.nexus_mod_id,
"fetching files for mod"
);
let files_resp = nexus_api::files::get(&client, db_mod.nexus_mod_id).await?;
// TODO: download other files than just MAIN files
// let files = files.into_iter().filter(|file| {
@@ -146,6 +155,7 @@ pub async fn main() -> Result<()> {
// }
// });
if let Some(duration) = files_resp.wait {
debug!(?duration, "sleeping");
sleep(duration).await;
}
@@ -163,6 +173,8 @@ pub async fn main() -> Result<()> {
)
.await?;
// TODO: check the file metadata to see if there are any plugin files in the archive before bothering to download the file (checking metadata does not count against rate-limit)
let download_link_resp =
nexus_api::download_link::get(&client, db_mod.nexus_mod_id, api_file.file_id)
.await?;
@@ -180,7 +192,10 @@ pub async fn main() -> Result<()> {
tokio_file.seek(SeekFrom::Start(0)).await?;
tokio_file.read_exact(&mut initial_bytes).await?;
let kind = infer::get(&initial_bytes).expect("unknown file type of file download");
dbg!(kind.mime_type());
info!(
mime_type = kind.mime_type(),
"inferred mime_type of downloaded archive"
);
tokio_file.seek(SeekFrom::Start(0)).await?;
let mut file = tokio_file.try_clone().await?.into_std().await;
@@ -194,10 +209,17 @@ pub async fn main() -> Result<()> {
plugin_file_paths.push(file_name);
}
}
info!(
num_plugin_files = plugin_file_paths.len(),
"listed plugins in downloaded archive"
);
for file_name in plugin_file_paths.iter() {
file.seek(SeekFrom::Start(0))?;
dbg!(file_name);
info!(
?file_name,
"attempting to uncompress file from downloaded archive"
);
let mut buf = Vec::default();
match uncompress_archive_file(&mut file, &mut buf, file_name) {
Ok(_) => {
@@ -212,10 +234,14 @@ pub async fn main() -> Result<()> {
.await?;
}
Err(error) => {
dbg!(error);
warn!(
?error,
"error occurred while attempting to uncompress archive file"
);
if kind.mime_type() == "application/x-rar-compressed"
|| kind.mime_type() == "application/vnd.rar"
{
info!("downloaded archive is RAR archive, attempt to uncompress entire archive instead");
// Use unrar to uncompress the entire .rar file to avoid a bug with compress_tools panicking when uncompressing
// certain .rar files: https://github.com/libarchive/libarchive/issues/373
tokio_file.seek(SeekFrom::Start(0)).await?;
@@ -253,7 +279,10 @@ pub async fn main() -> Result<()> {
.process()
.expect("failed to extract");
for file_name in plugin_file_paths.iter() {
dbg!(file_name);
info!(
?file_name,
"processing uncompressed file from downloaded archive"
);
let mut plugin_buf =
std::fs::read(temp_dir.path().join(file_name))?;
process_plugin(
@@ -269,20 +298,21 @@ pub async fn main() -> Result<()> {
}
temp_dir.close()?;
}
error!(mime_type = ?kind.mime_type(), "downloaded archive is not RAR archive, skipping processing of this file");
}
}
}
plugins_archive.finish()?;
if let Some(duration) = download_link_resp.wait {
debug!(?duration, "sleeping");
sleep(duration).await;
}
}
}
page += 1;
dbg!(page);
dbg!(has_next_page);
debug!(?page, ?has_next_page, "sleeping 1 second");
sleep(Duration::new(1, 0)).await;
}

2
src/models/cell.rs
View File

@@ -1,6 +1,7 @@
use anyhow::{Context, Result};
use chrono::NaiveDateTime;
use serde::{Deserialize, Serialize};
use tracing::instrument;
#[derive(Debug, Serialize, Deserialize)]
pub struct Cell {
@@ -13,6 +14,7 @@ pub struct Cell {
pub created_at: NaiveDateTime,
}
#[instrument(level = "debug", skip(pool))]
pub async fn insert_cell(
pool: &sqlx::Pool<sqlx::Postgres>,
form_id: i32,

2
src/models/file.rs
View File

@@ -1,6 +1,7 @@
use anyhow::{Context, Result};
use chrono::NaiveDateTime;
use serde::{Deserialize, Serialize};
use tracing::instrument;
#[derive(Debug, Serialize, Deserialize)]
pub struct File {
@@ -17,6 +18,7 @@ pub struct File {
pub created_at: NaiveDateTime,
}
#[instrument(level = "debug", skip(pool))]
pub async fn insert_file(
pool: &sqlx::Pool<sqlx::Postgres>,
name: &str,

2
src/models/game.rs
View File

@@ -1,6 +1,7 @@
use anyhow::{Context, Result};
use chrono::NaiveDateTime;
use serde::{Deserialize, Serialize};
use tracing::instrument;
#[derive(Debug, Serialize, Deserialize)]
pub struct Game {
@@ -11,6 +12,7 @@ pub struct Game {
pub created_at: NaiveDateTime,
}
#[instrument(level = "debug", skip(pool))]
pub async fn insert_game(
pool: &sqlx::Pool<sqlx::Postgres>,
name: &str,

3
src/models/game_mod.rs
View File

@@ -1,6 +1,7 @@
use anyhow::{Context, Result};
use chrono::NaiveDateTime;
use serde::{Deserialize, Serialize};
use tracing::instrument;
#[derive(Debug, Serialize, Deserialize)]
pub struct Mod {
@@ -15,6 +16,7 @@ pub struct Mod {
pub created_at: NaiveDateTime,
}
#[instrument(level = "debug", skip(pool))]
pub async fn get_mod_by_nexus_mod_id(
pool: &sqlx::Pool<sqlx::Postgres>,
nexus_mod_id: i32,
@@ -29,6 +31,7 @@ pub async fn get_mod_by_nexus_mod_id(
.context("Failed to get mod")
}
#[instrument(level = "debug", skip(pool))]
pub async fn insert_mod(
pool: &sqlx::Pool<sqlx::Postgres>,
name: &str,

4
src/models/plugin.rs
View File

@@ -1,6 +1,7 @@
use anyhow::{Context, Result};
use chrono::NaiveDateTime;
use serde::{Deserialize, Serialize};
use tracing::instrument;
#[derive(Debug, Serialize, Deserialize)]
pub struct Plugin {
@@ -16,6 +17,7 @@ pub struct Plugin {
pub created_at: NaiveDateTime,
}
#[instrument(level = "debug", skip(pool))]
pub async fn insert_plugin(
pool: &sqlx::Pool<sqlx::Postgres>,
name: &str,
@@ -46,4 +48,4 @@ pub async fn insert_plugin(
.fetch_one(pool)
.await
.context("Failed to insert plugin")
}
}

5
src/models/plugin_cell.rs
View File

@@ -1,7 +1,7 @@
use anyhow::{Context, Result};
use chrono::NaiveDateTime;
use serde::{Deserialize, Serialize};
use tracing::instrument;
#[derive(Debug, Serialize, Deserialize)]
pub struct PluginCell {
pub id: i32,
@@ -12,6 +12,7 @@ pub struct PluginCell {
pub created_at: NaiveDateTime,
}
#[instrument(level = "debug", skip(pool))]
pub async fn insert_plugin_cell(
pool: &sqlx::Pool<sqlx::Postgres>,
plugin_id: i32,
@@ -33,4 +34,4 @@ pub async fn insert_plugin_cell(
.fetch_one(pool)
.await
.context("Failed to insert cell")
}
}

7
src/nexus_api/download_link.rs
View File

@@ -6,6 +6,7 @@ use std::{env, time::Duration};
use tempfile::tempfile;
use tokio::fs::File;
use tokio_util::compat::FuturesAsyncReadCompatExt;
use tracing::{info, instrument};
use super::{rate_limit_wait_duration, GAME_NAME, USER_AGENT};
@@ -14,6 +15,7 @@ pub struct DownloadLinkResponse {
json: Value,
}
#[instrument(skip(client))]
pub async fn get(client: &Client, mod_id: i32, file_id: i64) -> Result<DownloadLinkResponse> {
let res = client
.get(format!(
@@ -27,6 +29,7 @@ pub async fn get(client: &Client, mod_id: i32, file_id: i64) -> Result<DownloadL
.await?
.error_for_status()?;
info!(status = %res.status(), "fetched file download link from API");
let wait = rate_limit_wait_duration(&res)?;
let json = res.json::<Value>().await?;
@@ -34,6 +37,7 @@ pub async fn get(client: &Client, mod_id: i32, file_id: i64) -> Result<DownloadL
}
impl DownloadLinkResponse {
#[instrument(skip(self))]
pub fn link<'a>(&'a self) -> Result<&'a str> {
let link = self
.json
@@ -43,9 +47,11 @@ impl DownloadLinkResponse {
.ok_or_else(|| anyhow!("Missing URI key in link in API response"))?
.as_str()
.ok_or_else(|| anyhow!("URI value in API response link is not a string"))?;
info!(link = %link, "parsed download link from API response");
Ok(link)
}
#[instrument(skip(self, client))]
pub async fn download_file(&self, client: &Client) -> Result<File> {
let mut tokio_file = File::from_std(tempfile()?);
let res = client
@@ -55,6 +61,7 @@ impl DownloadLinkResponse {
.send()
.await?
.error_for_status()?;
info!(status = %res.status(), "downloaded file from nexus");
// See: https://github.com/benkay86/async-applied/blob/master/reqwest-tokio-compat/src/main.rs
let mut byte_stream = res

11
src/nexus_api/files.rs
View File

@@ -3,6 +3,7 @@ use chrono::NaiveDateTime;
use reqwest::Client;
use serde_json::Value;
use std::{env, time::Duration};
use tracing::{info, instrument};
use super::{rate_limit_wait_duration, GAME_NAME, USER_AGENT};
@@ -21,6 +22,7 @@ pub struct ApiFile<'a> {
pub uploaded_at: NaiveDateTime,
}
#[instrument(skip(client))]
pub async fn get(client: &Client, nexus_mod_id: i32) -> Result<FilesResponse> {
let res = client
.get(format!(
@@ -34,6 +36,7 @@ pub async fn get(client: &Client, nexus_mod_id: i32) -> Result<FilesResponse> {
.await?
.error_for_status()?;
info!(status = %res.status(), "fetched files for mod from API");
let wait = rate_limit_wait_duration(&res)?;
let json = res.json::<Value>().await?;
@@ -41,6 +44,7 @@ pub async fn get(client: &Client, nexus_mod_id: i32) -> Result<FilesResponse> {
}
impl FilesResponse {
#[instrument(skip(self))]
pub fn files<'a>(&'a self) -> Result<Vec<ApiFile<'a>>> {
let files = self
.json
@@ -48,7 +52,7 @@ impl FilesResponse {
.ok_or_else(|| anyhow!("Missing files key in API response"))?
.as_array()
.ok_or_else(|| anyhow!("files value in API response is not an array"))?;
files
let files: Vec<ApiFile> = files
.into_iter()
.map(|file| {
let file_id = file
@@ -56,7 +60,6 @@ impl FilesResponse {
.ok_or_else(|| anyhow!("Missing file_id key in file in API response"))?
.as_i64()
.ok_or_else(|| anyhow!("file_id value in API response file is not a number"))?;
dbg!(file_id);
let name = file
.get("name")
.ok_or_else(|| anyhow!("Missing name key in file in API response"))?
@@ -102,6 +105,8 @@ impl FilesResponse {
uploaded_at,
})
})
.collect()
.collect::<Result<Vec<ApiFile>>>()?;
info!(num_files = files.len(), "parsed files out of API response");
Ok(files)
}
}

10
src/nexus_api/mod.rs
View File

@@ -3,6 +3,7 @@ use chrono::DateTime;
use chrono::Duration;
use chrono::Utc;
use reqwest::Response;
use tracing::info;
pub mod download_link;
pub mod files;
@@ -24,8 +25,7 @@ pub fn rate_limit_wait_duration(res: &Response) -> Result<Option<std::time::Dura
.headers()
.get("x-rl-hourly-reset")
.expect("No hourly reset in response headers");
dbg!(daily_remaining);
dbg!(hourly_remaining);
info!(daily_remaining = ?daily_remaining, hourly_remaining = ?hourly_remaining, "rate limit check");
if hourly_remaining == "0" {
let hourly_reset = hourly_reset.to_str()?.trim();
@@ -33,9 +33,11 @@ pub fn rate_limit_wait_duration(res: &Response) -> Result<Option<std::time::Dura
(DateTime::parse_from_str(hourly_reset, "%Y-%m-%d %H:%M:%S %z")?
+ Duration::seconds(5))
.into();
dbg!(hourly_reset);
let duration = (hourly_reset - Utc::now()).to_std()?;
dbg!(duration);
info!(
hourly_reset = ?hourly_reset,
duration = ?duration, "need to wait until rate-limit hourly reset"
);
return Ok(Some(duration));
}

9
src/nexus_scraper.rs
View File

@@ -1,6 +1,7 @@
use anyhow::Result;
use reqwest::Client;
use scraper::{Html, Selector};
use tracing::{info, instrument};
use crate::nexus_api::GAME_ID;
@@ -20,6 +21,7 @@ pub struct ModListScrape<'a> {
pub has_next_page: bool,
}
#[instrument(skip(client))]
pub async fn get_mod_list_page(client: &Client, page: i32) -> Result<ModListResponse> {
let res = client
.get(format!(
@@ -30,6 +32,7 @@ pub async fn get_mod_list_page(client: &Client, page: i32) -> Result<ModListResp
.send()
.await?
.error_for_status()?;
info!(status = %res.status(), "fetched mod list page");
let text = res.text().await?;
let html = Html::parse_document(&text);
@@ -37,6 +40,7 @@ pub async fn get_mod_list_page(client: &Client, page: i32) -> Result<ModListResp
}
impl ModListResponse {
#[instrument(skip(self))]
pub fn scrape_mods<'a>(&'a self) -> Result<ModListScrape> {
let mod_select = Selector::parse("li.mod-tile").expect("failed to parse CSS selector");
let left_select =
@@ -110,7 +114,10 @@ impl ModListResponse {
}
})
.collect();
dbg!(mods.len());
info!(
len = mods.len(),
has_next_page, "scraped mods from mod list page"
);
Ok(ModListScrape {
mods,
has_next_page,