diff --git a/Cargo.lock b/Cargo.lock index e37ffe1..54fe15f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -75,7 +75,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "616896e05fc0e2649463a93a15183c6a16bf03413a7af88ef1285ddedfa9cda5" dependencies = [ - "num-traits", + "num-traits 0.2.14", ] [[package]] @@ -182,7 +182,7 @@ checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" dependencies = [ "libc", "num-integer", - "num-traits", + "num-traits 0.2.14", "serde", "time", "winapi", @@ -399,6 +399,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "enum_primitive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4551092f4d519593039259a9ed8daedf0da12e5109c5280338073eaeb81180" +dependencies = [ + "num-traits 0.1.43", +] + [[package]] name = "flate2" version = "1.0.20" @@ -442,6 +451,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + [[package]] name = "funty" version = "1.1.0" @@ -972,6 +987,7 @@ dependencies = [ "tempfile", "tokio", "tokio-util", + "unrar", "zip", ] @@ -1027,6 +1043,42 @@ dependencies = [ "winapi", ] +[[package]] +name = "num" +version = "0.1.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4703ad64153382334aa8db57c637364c322d3372e097840c72000dabdcf6156e" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits 0.2.14", +] + +[[package]] +name = "num-bigint" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e63899ad0da84ce718c14936262a41cee2c79c981fc0a0e7c7beb47d5a07e8c1" +dependencies = [ + "num-integer", + "num-traits 0.2.14", + "rand 0.4.6", + "rustc-serialize", +] + +[[package]] +name = "num-complex" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b288631d7878aaf59442cffd36910ea604ecd7745c36054328595114001c9656" +dependencies = [ + "num-traits 0.2.14", + "rustc-serialize", +] + [[package]] name = "num-integer" version = "0.1.44" @@ -1034,7 +1086,39 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" dependencies = [ "autocfg", - "num-traits", + "num-traits 0.2.14", +] + +[[package]] +name = "num-iter" +version = "0.1.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2021c8337a54d21aca0d59a92577a029af9431cb59b909b03252b9c164fad59" +dependencies = [ + "autocfg", + "num-integer", + "num-traits 0.2.14", +] + +[[package]] +name = "num-rational" +version = "0.1.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee314c74bd753fc86b4780aa9475da469155f3848473a261d2d18e35245a784e" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits 0.2.14", + "rustc-serialize", +] + +[[package]] +name = "num-traits" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31" +dependencies = [ + "num-traits 0.2.14", ] [[package]] @@ -1272,6 +1356,19 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "941ba9d78d8e2f7ce474c015eea4d9c6d25b6a3327f9832ee29a4de27f91bbb8" +[[package]] +name = "rand" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" +dependencies = [ + "fuchsia-cprng", + "libc", + "rand_core 0.3.1", + "rdrand", + "winapi", +] + [[package]] name = "rand" version = "0.7.3" @@ -1318,6 +1415,21 @@ dependencies = [ "rand_core 0.6.2", ] +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + [[package]] name = "rand_core" version = "0.5.1" @@ -1363,6 +1475,15 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + [[package]] name = "redox_syscall" version = "0.2.8" @@ -1443,6 +1564,12 @@ dependencies = [ "winreg", ] +[[package]] +name = "rustc-serialize" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda" + [[package]] name = "ryu" version = "1.0.5" @@ -2055,6 +2182,31 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" +[[package]] +name = "unrar" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "433cea4f0b7bec88d47becb380887b8786a3cfb1c82e1ef9d32a682ba6801814" +dependencies = [ + "bitflags", + "enum_primitive", + "lazy_static", + "num", + "regex", + "unrar_sys", +] + +[[package]] +name = "unrar_sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0009399408dc0bcc5c8910672544fceceeba18b91f741ff943916e917d982c60" +dependencies = [ + "cc", + "libc", + "winapi", +] + [[package]] name = "url" version = "2.2.2" diff --git a/Cargo.toml b/Cargo.toml index be91e00..c164d8d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,5 +26,6 @@ skyrim-cell-dump = "0.1.3" tempfile = "3.2" tokio = { version = "1.5.0", features = ["full"] } tokio-util = { version = "0.6", features = ["compat"] } +unrar = "0.4" # Need `ZipWriter::append_new` from https://github.com/zip-rs/zip/commit/ce272616ac69b798bb7b0925147a8a710dc2bb65 zip = { git = "https://github.com/zip-rs/zip.git" } \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..face05f --- /dev/null +++ b/README.md @@ -0,0 +1,37 @@ +# modmapper + +Downloads mods from nexus, parses the plugins inside, and saves data to a postgres database. + +## Development Install + +1. Install and run postgres. +2. Create postgres user and database (and add uuid extension while you're there + ): + +``` +createuser modmapper +createdb modmapper +sudo -u postgres -i psql +postgres=# ALTER DATABASE modmapper OWNER TO modmapper; +\password modmapper + +# Or, on Windows in PowerShell: + +& 'C:\Program Files\PostgreSQL\13\bin\createuser.exe' -U postgres modmapper +& 'C:\Program Files\PostgreSQL\13\bin\createdb.exe' -U postgres modmapper +& 'C:\Program Files\PostgreSQL\13\bin\psql.exe' -U postgres +postgres=# ALTER DATABASE modmapper OWNER TO modmapper; +\password modmapper +``` + +3. Save password somewhere safe and then and add a `.env` file to the project + directory with the contents: + +``` +DATABASE_URL=postgresql://modmapper:@localhost/modmapper +``` + +4. Install + [`sqlx_cli`](https://github.com/launchbadge/sqlx/tree/master/sqlx-cli) with + `cargo install --version=0.1.0-beta.1 sqlx-cli --no-default-features --features postgres` +5. Run `sqlx migrate --source migrations run` which will run all the database migrations. \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index ede33e4..a6f9cd9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,12 +16,14 @@ use sqlx::postgres::PgPoolOptions; use std::convert::TryInto; use std::env; use std::fs::OpenOptions; +use std::io::Read; use std::io::Seek; use std::io::SeekFrom; -use tempfile::tempfile; +use tempfile::{tempfile, tempdir}; use tokio::io::{AsyncReadExt, AsyncSeekExt}; use tokio::time::sleep; use tokio_util::compat::FuturesAsyncReadCompatExt; +use unrar::Archive; use zip::write::{FileOptions, ZipWriter}; static USER_AGENT: &str = "mod-mapper/0.1"; @@ -266,17 +268,18 @@ async fn insert_plugin_cell( } fn rate_limit_wait_duration(res: &Response) -> Result> { + dbg!(res.headers().get("x-rl-daily-remaining")); let daily_remaining = res .headers() - .get("X-RL-Daily-Remaining") - .expect("No daily limit in response headers"); + .get("x-rl-daily-remaining") + .expect("No daily remaining in response headers"); let hourly_remaining = res .headers() - .get("X-RL-Hourly-Remaining") + .get("x-rl-hourly-remaining") .expect("No hourly limit in response headers"); let hourly_reset = res .headers() - .get("X-RL-Hourly-Reset") + .get("x-rl-hourly-reset") .expect("No hourly reset in response headers"); dbg!(daily_remaining); dbg!(hourly_remaining); @@ -297,6 +300,66 @@ fn rate_limit_wait_duration(res: &Response) -> Result( + plugin_buf: &[u8], + pool: &sqlx::Pool, + plugin_archive: &mut ZipWriter, + name: &str, + db_file: &File, + mod_obj: &Mod, + file_id: i64, + file_name: &str, +) -> Result<()> + where W: std::io::Write + std::io::Seek +{ + let plugin = parse_plugin(&plugin_buf)?; + let hash = seahash::hash(&plugin_buf); + let plugin_row = insert_plugin( + &pool, + name, + hash as i64, + db_file.id, + Some(plugin.header.version as f64), + plugin.header.author, + plugin.header.description, + Some( + &plugin + .header + .masters + .iter() + .map(|s| s.to_string()) + .collect::>(), + ), + ) + .await?; + for cell in plugin.cells { + let cell_row = insert_cell( + &pool, + cell.form_id.try_into().unwrap(), + cell.x, + cell.y, + cell.is_persistent, + ) + .await?; + insert_plugin_cell( + &pool, + plugin_row.id, + cell_row.id, + cell.editor_id, + ) + .await?; + } + plugin_archive.start_file( + format!( + "{}/{}/{}/{}", + GAME_NAME, mod_obj.nexus_mod_id, file_id, file_name + ), + FileOptions::default(), + )?; + std::io::copy(plugin_buf, plugin_archive)?; + Ok(()) +} + #[tokio::main] pub async fn main() -> Result<()> { dotenv().ok(); @@ -433,6 +496,7 @@ pub async fn main() -> Result<()> { .ok_or_else(|| anyhow!("Missing file_id key in file in API response"))? .as_i64() .ok_or_else(|| anyhow!("file_id value in API response file is not a number"))?; + let file_id = 18422; // DELETEME: temp test bad rar file dbg!(file_id); let name = file .get("name") @@ -492,6 +556,8 @@ pub async fn main() -> Result<()> { .await? .error_for_status()?; + let duration = rate_limit_wait_duration(&res)?; + let links = res.json::().await?; let link = links .get(0) @@ -510,8 +576,6 @@ pub async fn main() -> Result<()> { .await? .error_for_status()?; - let duration = rate_limit_wait_duration(&res)?; - // See: https://github.com/benkay86/async-applied/blob/master/reqwest-tokio-compat/src/main.rs let mut byte_stream = res .bytes_stream() @@ -571,10 +635,51 @@ pub async fn main() -> Result<()> { // std::io::copy(&mut file, &mut plugin_archive)?; // } // } + + // Use unrar to uncompress the entire .rar file to avoid a bug with compress_tools panicking when uncompressing + // certain .rar files: https://github.com/libarchive/libarchive/issues/373 + "application/x-rar-compressed" => { + tokio_file.seek(SeekFrom::Start(0)).await?; + let mut file = tokio_file.into_std().await; + let temp_dir = tempdir()?; + let temp_file_path = temp_dir.path().join("download.rar"); + let mut temp_file = std::fs::File::create(temp_file_path)?; + std::io::copy(&mut file, &mut temp_file)?; + + let mut plugin_file_paths = Vec::new(); + let list = Archive::new(temp_file_path.to_string_lossy().to_string()).list(); + if let Ok(list) = list { + for entry in list { + if let Ok(entry) = entry { + if entry.filename.ends_with(".esp") + || entry.filename.ends_with(".esm") + || entry.filename.ends_with(".esl") + { + plugin_file_paths.push(entry.filename); + } + } + } + } + + if plugin_file_paths.len() > 0 { + let extract = Archive::new(temp_file_path.to_string_lossy().to_string()).extract_to(temp_dir.path().to_string_lossy().to_string()); + extract.expect("failed to extract").process().expect("failed to extract"); + for file_name in plugin_file_paths.iter() { + dbg!(file_name); + let plugin_file = std::fs::File::open(temp_dir.path().join(file_name))?; + let mut plugin_buf = Vec::new(); + plugin_file.read(&mut plugin_buf)?; + process_plugin(&plugin_buf, &pool, &mut plugin_archive, name, &db_file, &mod_obj, file_id, file_name).await?; + } + dbg!("uncompressed!"); + } + temp_dir.close()?; + }, _ => { tokio_file.seek(SeekFrom::Start(0)).await?; let mut file = tokio_file.into_std().await; let mut plugin_file_paths = Vec::new(); + for file_name in list_archive_files(&file)? { if file_name.ends_with(".esp") || file_name.ends_with(".esm") @@ -583,56 +688,13 @@ pub async fn main() -> Result<()> { plugin_file_paths.push(file_name); } } + for file_name in plugin_file_paths.iter() { file.seek(SeekFrom::Start(0))?; dbg!(file_name); let mut buf = Vec::default(); uncompress_archive_file(&mut file, &mut buf, file_name)?; - let plugin = parse_plugin(&buf)?; - let hash = seahash::hash(&buf); - let plugin_row = insert_plugin( - &pool, - name, - hash as i64, - db_file.id, - Some(plugin.header.version as f64), - plugin.header.author, - plugin.header.description, - Some( - &plugin - .header - .masters - .iter() - .map(|s| s.to_string()) - .collect::>(), - ), - ) - .await?; - for cell in plugin.cells { - let cell_row = insert_cell( - &pool, - cell.form_id.try_into().unwrap(), - cell.x, - cell.y, - cell.is_persistent, - ) - .await?; - insert_plugin_cell( - &pool, - plugin_row.id, - cell_row.id, - cell.editor_id, - ) - .await?; - } - plugin_archive.start_file( - format!( - "{}/{}/{}/{}", - GAME_NAME, mod_obj.nexus_mod_id, file_id, file_name - ), - FileOptions::default(), - )?; - std::io::copy(&mut buf.as_slice(), &mut plugin_archive)?; + process_plugin(&buf, &pool, &mut plugin_archive, name, &db_file, &mod_obj, file_id, file_name).await?; } } }; @@ -641,10 +703,13 @@ pub async fn main() -> Result<()> { if let Some(duration) = duration { sleep(duration).await; } + break; } + break; } page += 1; + break; } Ok(())