Mostly working download loop done
Still need to fix a panic that happens on some .rar archive files.
This commit is contained in:
parent
b132a94c64
commit
d6b8f4e74a
4
Cargo.lock
generated
4
Cargo.lock
generated
@ -1626,9 +1626,9 @@ checksum = "cbce6d4507c7e4a3962091436e56e95290cb71fa302d0d270e32130b75fbff27"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "skyrim-cell-dump"
|
name = "skyrim-cell-dump"
|
||||||
version = "0.1.2"
|
version = "0.1.3"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "372b96816596c25ba82afdc4819aae92e3750c9f4d965aa99d46f25fe53bbc3f"
|
checksum = "b8ff27163eeca52326be9a89a4adc15dd7ed3d7c0c44dd981aa2bbacff10aede"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"bitflags",
|
"bitflags",
|
||||||
|
@ -22,7 +22,7 @@ seahash = "4.1"
|
|||||||
serde = { version = "1.0", features = ["derive"] }
|
serde = { version = "1.0", features = ["derive"] }
|
||||||
serde_json = "1.0"
|
serde_json = "1.0"
|
||||||
sqlx = { version = "0.5", features = ["runtime-tokio-native-tls", "postgres", "migrate", "chrono"] }
|
sqlx = { version = "0.5", features = ["runtime-tokio-native-tls", "postgres", "migrate", "chrono"] }
|
||||||
skyrim-cell-dump = "0.1.2"
|
skyrim-cell-dump = "0.1.3"
|
||||||
tempfile = "3.2"
|
tempfile = "3.2"
|
||||||
tokio = { version = "1.5.0", features = ["full"] }
|
tokio = { version = "1.5.0", features = ["full"] }
|
||||||
tokio-util = { version = "0.6", features = ["compat"] }
|
tokio-util = { version = "0.6", features = ["compat"] }
|
||||||
|
150
src/main.rs
150
src/main.rs
@ -1,9 +1,13 @@
|
|||||||
use anyhow::{anyhow, Context, Result};
|
use anyhow::{anyhow, Context, Result};
|
||||||
|
use chrono::DateTime;
|
||||||
|
use chrono::Duration;
|
||||||
use chrono::NaiveDateTime;
|
use chrono::NaiveDateTime;
|
||||||
|
use chrono::Utc;
|
||||||
use compress_tools::{list_archive_files, uncompress_archive_file};
|
use compress_tools::{list_archive_files, uncompress_archive_file};
|
||||||
use dotenv::dotenv;
|
use dotenv::dotenv;
|
||||||
use futures::future::try_join_all;
|
use futures::future::try_join_all;
|
||||||
use futures::stream::TryStreamExt;
|
use futures::stream::TryStreamExt;
|
||||||
|
use reqwest::Response;
|
||||||
use scraper::{Html, Selector};
|
use scraper::{Html, Selector};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
@ -16,6 +20,7 @@ use std::io::Seek;
|
|||||||
use std::io::SeekFrom;
|
use std::io::SeekFrom;
|
||||||
use tempfile::tempfile;
|
use tempfile::tempfile;
|
||||||
use tokio::io::{AsyncReadExt, AsyncSeekExt};
|
use tokio::io::{AsyncReadExt, AsyncSeekExt};
|
||||||
|
use tokio::time::sleep;
|
||||||
use tokio_util::compat::FuturesAsyncReadCompatExt;
|
use tokio_util::compat::FuturesAsyncReadCompatExt;
|
||||||
use zip::write::{FileOptions, ZipWriter};
|
use zip::write::{FileOptions, ZipWriter};
|
||||||
|
|
||||||
@ -223,7 +228,9 @@ async fn insert_cell(
|
|||||||
"INSERT INTO cells
|
"INSERT INTO cells
|
||||||
(form_id, x, y, is_persistent, created_at, updated_at)
|
(form_id, x, y, is_persistent, created_at, updated_at)
|
||||||
VALUES ($1, $2, $3, $4, now(), now())
|
VALUES ($1, $2, $3, $4, now(), now())
|
||||||
ON CONFLICT DO NOTHING
|
ON CONFLICT (form_id) DO UPDATE
|
||||||
|
SET (x, y, is_persistent, updated_at) =
|
||||||
|
(EXCLUDED.x, EXCLUDED.y, EXCLUDED.is_persistent, now())
|
||||||
RETURNING *",
|
RETURNING *",
|
||||||
form_id,
|
form_id,
|
||||||
x,
|
x,
|
||||||
@ -258,6 +265,38 @@ async fn insert_plugin_cell(
|
|||||||
.context("Failed to insert cell")
|
.context("Failed to insert cell")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn rate_limit_wait_duration(res: &Response) -> Result<Option<std::time::Duration>> {
|
||||||
|
let daily_remaining = res
|
||||||
|
.headers()
|
||||||
|
.get("X-RL-Daily-Remaining")
|
||||||
|
.expect("No daily limit in response headers");
|
||||||
|
let hourly_remaining = res
|
||||||
|
.headers()
|
||||||
|
.get("X-RL-Hourly-Remaining")
|
||||||
|
.expect("No hourly limit in response headers");
|
||||||
|
let hourly_reset = res
|
||||||
|
.headers()
|
||||||
|
.get("X-RL-Hourly-Reset")
|
||||||
|
.expect("No hourly reset in response headers");
|
||||||
|
dbg!(daily_remaining);
|
||||||
|
dbg!(hourly_remaining);
|
||||||
|
|
||||||
|
if hourly_remaining == "0" {
|
||||||
|
let hourly_reset = hourly_reset.to_str()?.trim();
|
||||||
|
let hourly_reset: DateTime<Utc> =
|
||||||
|
(DateTime::parse_from_str(hourly_reset, "%Y-%m-%d %H:%M:%S %z")?
|
||||||
|
+ Duration::seconds(5))
|
||||||
|
.into();
|
||||||
|
dbg!(hourly_reset);
|
||||||
|
let duration = (hourly_reset - Utc::now()).to_std()?;
|
||||||
|
dbg!(duration);
|
||||||
|
|
||||||
|
return Ok(Some(duration));
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(None)
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
pub async fn main() -> Result<()> {
|
pub async fn main() -> Result<()> {
|
||||||
dotenv().ok();
|
dotenv().ok();
|
||||||
@ -268,23 +307,46 @@ pub async fn main() -> Result<()> {
|
|||||||
let game = insert_game(&pool, GAME_NAME, GAME_ID as i32).await?;
|
let game = insert_game(&pool, GAME_NAME, GAME_ID as i32).await?;
|
||||||
let client = reqwest::Client::new();
|
let client = reqwest::Client::new();
|
||||||
|
|
||||||
|
let mut page: i32 = 1;
|
||||||
|
let mut last_page: i32 = 1;
|
||||||
|
|
||||||
|
while page <= last_page {
|
||||||
let res = client
|
let res = client
|
||||||
.get(format!(
|
.get(format!(
|
||||||
"https://www.nexusmods.com/Core/Libs/Common/Widgets/ModList?RH_ModList=nav:true,home:false,type:0,user_id:0,game_id:{},advfilt:true,include_adult:true,page_size:80,show_game_filter:false,open:false,page:1,sort_by:OLD_u_downloads",
|
"https://www.nexusmods.com/Core/Libs/Common/Widgets/ModList?RH_ModList=nav:true,home:false,type:0,user_id:0,game_id:{},advfilt:true,include_adult:true,page_size:80,show_game_filter:false,open:false,page:{},sort_by:OLD_u_downloads",
|
||||||
GAME_ID
|
GAME_ID,
|
||||||
|
page
|
||||||
))
|
))
|
||||||
.send()
|
.send()
|
||||||
.await?
|
.await?
|
||||||
.error_for_status()?;
|
.error_for_status()?;
|
||||||
let html = res.text().await?;
|
let html = res.text().await?;
|
||||||
let document = Html::parse_document(&html);
|
let document = Html::parse_document(&html);
|
||||||
let mod_select = Selector::parse("div.mod-tile").expect("failed to parse CSS selector");
|
let mod_select = Selector::parse("li.mod-tile").expect("failed to parse CSS selector");
|
||||||
let left_select = Selector::parse("div.mod-tile-left").expect("failed to parse CSS selector");
|
let left_select =
|
||||||
let right_select = Selector::parse("div.mod-tile-right").expect("failed to parse CSS selector");
|
Selector::parse("div.mod-tile-left").expect("failed to parse CSS selector");
|
||||||
|
let right_select =
|
||||||
|
Selector::parse("div.mod-tile-right").expect("failed to parse CSS selector");
|
||||||
let name_select = Selector::parse("p.tile-name a").expect("failed to parse CSS selector");
|
let name_select = Selector::parse("p.tile-name a").expect("failed to parse CSS selector");
|
||||||
let category_select = Selector::parse("div.category a").expect("failed to parse CSS selector");
|
let category_select =
|
||||||
|
Selector::parse("div.category a").expect("failed to parse CSS selector");
|
||||||
let author_select = Selector::parse("div.author a").expect("failed to parse CSS selector");
|
let author_select = Selector::parse("div.author a").expect("failed to parse CSS selector");
|
||||||
let desc_select = Selector::parse("p.desc").expect("failed to parse CSS selector");
|
let desc_select = Selector::parse("p.desc").expect("failed to parse CSS selector");
|
||||||
|
let last_page_select =
|
||||||
|
Selector::parse("div.pagination li.extra a").expect("failed to parse CSS selector");
|
||||||
|
|
||||||
|
let last_page_elem = document
|
||||||
|
.select(&last_page_select)
|
||||||
|
.next()
|
||||||
|
.expect("Missing last page link");
|
||||||
|
last_page = last_page_elem
|
||||||
|
.text()
|
||||||
|
.next()
|
||||||
|
.expect("Missing last page text")
|
||||||
|
.trim()
|
||||||
|
.parse::<i32>()
|
||||||
|
.ok()
|
||||||
|
.expect("Failed to parse last page");
|
||||||
|
|
||||||
let mods = try_join_all(document.select(&mod_select).map(|element| {
|
let mods = try_join_all(document.select(&mod_select).map(|element| {
|
||||||
let left = element
|
let left = element
|
||||||
@ -328,18 +390,16 @@ pub async fn main() -> Result<()> {
|
|||||||
.next()
|
.next()
|
||||||
.expect("Missing desc elem for mod");
|
.expect("Missing desc elem for mod");
|
||||||
let desc = desc_elem.text().next();
|
let desc = desc_elem.text().next();
|
||||||
dbg!(name, nexus_mod_id, author, category, desc, game.id);
|
|
||||||
insert_mod(&pool, name, nexus_mod_id, author, category, desc, game.id)
|
insert_mod(&pool, name, nexus_mod_id, author, category, desc, game.id)
|
||||||
}))
|
}))
|
||||||
.await?;
|
.await?;
|
||||||
dbg!(&mods);
|
|
||||||
|
|
||||||
for mod_obj in mods {
|
for mod_obj in mods {
|
||||||
dbg!(mod_obj.id);
|
dbg!(&mod_obj);
|
||||||
let res = client
|
let res = client
|
||||||
.get(format!(
|
.get(format!(
|
||||||
"https://api.nexusmods.com/v1/games/{}/mods/{}/files.json",
|
"https://api.nexusmods.com/v1/games/{}/mods/{}/files.json",
|
||||||
GAME_NAME, mod_obj.id
|
GAME_NAME, mod_obj.nexus_mod_id
|
||||||
))
|
))
|
||||||
.header("accept", "application/json")
|
.header("accept", "application/json")
|
||||||
.header("apikey", env::var("NEXUS_API_KEY")?)
|
.header("apikey", env::var("NEXUS_API_KEY")?)
|
||||||
@ -347,6 +407,11 @@ pub async fn main() -> Result<()> {
|
|||||||
.send()
|
.send()
|
||||||
.await?
|
.await?
|
||||||
.error_for_status()?;
|
.error_for_status()?;
|
||||||
|
|
||||||
|
if let Some(duration) = rate_limit_wait_duration(&res)? {
|
||||||
|
sleep(duration).await;
|
||||||
|
}
|
||||||
|
|
||||||
let files = res.json::<Value>().await?;
|
let files = res.json::<Value>().await?;
|
||||||
let files = files
|
let files = files
|
||||||
.get("files")
|
.get("files")
|
||||||
@ -378,9 +443,11 @@ pub async fn main() -> Result<()> {
|
|||||||
.get("file_name")
|
.get("file_name")
|
||||||
.ok_or_else(|| anyhow!("Missing file_name key in file in API response"))?
|
.ok_or_else(|| anyhow!("Missing file_name key in file in API response"))?
|
||||||
.as_str()
|
.as_str()
|
||||||
.ok_or_else(|| anyhow!("file_name value in API response file is not a string"))?;
|
.ok_or_else(|| {
|
||||||
|
anyhow!("file_name value in API response file is not a string")
|
||||||
|
})?;
|
||||||
let category = file
|
let category = file
|
||||||
.get("category")
|
.get("category_name")
|
||||||
.ok_or_else(|| anyhow!("Missing category key in file in API response"))?
|
.ok_or_else(|| anyhow!("Missing category key in file in API response"))?
|
||||||
.as_str();
|
.as_str();
|
||||||
let version = file
|
let version = file
|
||||||
@ -393,13 +460,15 @@ pub async fn main() -> Result<()> {
|
|||||||
.as_str();
|
.as_str();
|
||||||
let uploaded_timestamp = file
|
let uploaded_timestamp = file
|
||||||
.get("uploaded_timestamp")
|
.get("uploaded_timestamp")
|
||||||
.ok_or_else(|| anyhow!("Missing uploaded_timestamp key in file in API response"))?
|
.ok_or_else(|| {
|
||||||
|
anyhow!("Missing uploaded_timestamp key in file in API response")
|
||||||
|
})?
|
||||||
.as_i64()
|
.as_i64()
|
||||||
.ok_or_else(|| {
|
.ok_or_else(|| {
|
||||||
anyhow!("uploaded_timestamp value in API response file is not a number")
|
anyhow!("uploaded_timestamp value in API response file is not a number")
|
||||||
})?;
|
})?;
|
||||||
let uploaded_at = NaiveDateTime::from_timestamp(uploaded_timestamp, 0);
|
let uploaded_at = NaiveDateTime::from_timestamp(uploaded_timestamp, 0);
|
||||||
insert_file(
|
let db_file = insert_file(
|
||||||
&pool,
|
&pool,
|
||||||
name,
|
name,
|
||||||
file_name,
|
file_name,
|
||||||
@ -414,7 +483,7 @@ pub async fn main() -> Result<()> {
|
|||||||
let res = client
|
let res = client
|
||||||
.get(format!(
|
.get(format!(
|
||||||
"https://api.nexusmods.com/v1/games/{}/mods/{}/files/{}/download_link.json",
|
"https://api.nexusmods.com/v1/games/{}/mods/{}/files/{}/download_link.json",
|
||||||
GAME_NAME, mod_obj.id, file_id
|
GAME_NAME, mod_obj.nexus_mod_id, file_id
|
||||||
))
|
))
|
||||||
.header("accept", "application/json")
|
.header("accept", "application/json")
|
||||||
.header("apikey", env::var("NEXUS_API_KEY")?)
|
.header("apikey", env::var("NEXUS_API_KEY")?)
|
||||||
@ -422,6 +491,7 @@ pub async fn main() -> Result<()> {
|
|||||||
.send()
|
.send()
|
||||||
.await?
|
.await?
|
||||||
.error_for_status()?;
|
.error_for_status()?;
|
||||||
|
|
||||||
let links = res.json::<Value>().await?;
|
let links = res.json::<Value>().await?;
|
||||||
let link = links
|
let link = links
|
||||||
.get(0)
|
.get(0)
|
||||||
@ -440,6 +510,8 @@ pub async fn main() -> Result<()> {
|
|||||||
.await?
|
.await?
|
||||||
.error_for_status()?;
|
.error_for_status()?;
|
||||||
|
|
||||||
|
let duration = rate_limit_wait_duration(&res)?;
|
||||||
|
|
||||||
// See: https://github.com/benkay86/async-applied/blob/master/reqwest-tokio-compat/src/main.rs
|
// See: https://github.com/benkay86/async-applied/blob/master/reqwest-tokio-compat/src/main.rs
|
||||||
let mut byte_stream = res
|
let mut byte_stream = res
|
||||||
.bytes_stream()
|
.bytes_stream()
|
||||||
@ -459,7 +531,7 @@ pub async fn main() -> Result<()> {
|
|||||||
.open("plugins.zip")?,
|
.open("plugins.zip")?,
|
||||||
);
|
);
|
||||||
plugin_archive.add_directory(
|
plugin_archive.add_directory(
|
||||||
format!("{}/{}/{}", GAME_NAME, mod_obj.id, file_id),
|
format!("{}/{}/{}", GAME_NAME, mod_obj.nexus_mod_id, file_id),
|
||||||
FileOptions::default(),
|
FileOptions::default(),
|
||||||
)?;
|
)?;
|
||||||
plugin_archive.finish()?;
|
plugin_archive.finish()?;
|
||||||
@ -473,7 +545,6 @@ pub async fn main() -> Result<()> {
|
|||||||
let mut initial_bytes = [0; 8];
|
let mut initial_bytes = [0; 8];
|
||||||
tokio_file.seek(SeekFrom::Start(0)).await?;
|
tokio_file.seek(SeekFrom::Start(0)).await?;
|
||||||
tokio_file.read_exact(&mut initial_bytes).await?;
|
tokio_file.read_exact(&mut initial_bytes).await?;
|
||||||
dbg!(&initial_bytes);
|
|
||||||
let kind = infer::get(&initial_bytes).expect("unknown file type of file download");
|
let kind = infer::get(&initial_bytes).expect("unknown file type of file download");
|
||||||
match kind.mime_type() {
|
match kind.mime_type() {
|
||||||
// "application/zip" => {
|
// "application/zip" => {
|
||||||
@ -505,7 +576,6 @@ pub async fn main() -> Result<()> {
|
|||||||
let mut file = tokio_file.into_std().await;
|
let mut file = tokio_file.into_std().await;
|
||||||
let mut plugin_file_paths = Vec::new();
|
let mut plugin_file_paths = Vec::new();
|
||||||
for file_name in list_archive_files(&file)? {
|
for file_name in list_archive_files(&file)? {
|
||||||
dbg!(&file_name);
|
|
||||||
if file_name.ends_with(".esp")
|
if file_name.ends_with(".esp")
|
||||||
|| file_name.ends_with(".esm")
|
|| file_name.ends_with(".esm")
|
||||||
|| file_name.ends_with(".esl")
|
|| file_name.ends_with(".esl")
|
||||||
@ -513,21 +583,18 @@ pub async fn main() -> Result<()> {
|
|||||||
plugin_file_paths.push(file_name);
|
plugin_file_paths.push(file_name);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
file.seek(SeekFrom::Start(0))?;
|
|
||||||
for file_name in plugin_file_paths.iter() {
|
for file_name in plugin_file_paths.iter() {
|
||||||
|
file.seek(SeekFrom::Start(0))?;
|
||||||
dbg!(file_name);
|
dbg!(file_name);
|
||||||
let mut buf = Vec::default();
|
let mut buf = Vec::default();
|
||||||
uncompress_archive_file(&mut file, &mut buf, file_name)?;
|
uncompress_archive_file(&mut file, &mut buf, file_name)?;
|
||||||
let plugin = parse_plugin(&buf)?;
|
let plugin = parse_plugin(&buf)?;
|
||||||
dbg!(&plugin);
|
|
||||||
let hash = seahash::hash(&buf);
|
let hash = seahash::hash(&buf);
|
||||||
dbg!(&hash);
|
|
||||||
let plugin_row = insert_plugin(
|
let plugin_row = insert_plugin(
|
||||||
&pool,
|
&pool,
|
||||||
name,
|
name,
|
||||||
// TODO: how to make i64 hash?
|
hash as i64,
|
||||||
hash.try_into()?,
|
db_file.id,
|
||||||
file_id as i32,
|
|
||||||
Some(plugin.header.version as f64),
|
Some(plugin.header.version as f64),
|
||||||
plugin.header.author,
|
plugin.header.author,
|
||||||
plugin.header.description,
|
plugin.header.description,
|
||||||
@ -550,11 +617,19 @@ pub async fn main() -> Result<()> {
|
|||||||
cell.is_persistent,
|
cell.is_persistent,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
insert_plugin_cell(&pool, plugin_row.id, cell_row.id, cell.editor_id)
|
insert_plugin_cell(
|
||||||
|
&pool,
|
||||||
|
plugin_row.id,
|
||||||
|
cell_row.id,
|
||||||
|
cell.editor_id,
|
||||||
|
)
|
||||||
.await?;
|
.await?;
|
||||||
}
|
}
|
||||||
plugin_archive.start_file(
|
plugin_archive.start_file(
|
||||||
format!("{}/{}/{}/{}", GAME_NAME, mod_obj.id, file_id, file_name),
|
format!(
|
||||||
|
"{}/{}/{}/{}",
|
||||||
|
GAME_NAME, mod_obj.nexus_mod_id, file_id, file_name
|
||||||
|
),
|
||||||
FileOptions::default(),
|
FileOptions::default(),
|
||||||
)?;
|
)?;
|
||||||
std::io::copy(&mut buf.as_slice(), &mut plugin_archive)?;
|
std::io::copy(&mut buf.as_slice(), &mut plugin_archive)?;
|
||||||
@ -563,21 +638,14 @@ pub async fn main() -> Result<()> {
|
|||||||
};
|
};
|
||||||
|
|
||||||
plugin_archive.finish()?;
|
plugin_archive.finish()?;
|
||||||
break; // temporarily just grabbing first file
|
if let Some(duration) = duration {
|
||||||
|
sleep(duration).await;
|
||||||
}
|
}
|
||||||
break; // temporarily just grabbing first mod
|
|
||||||
}
|
}
|
||||||
// let mod_id = 4119; // hardcoded temporarily
|
}
|
||||||
// let res = client
|
|
||||||
// .get("https://cf-files.nexusmods.com/cdn/1704/351/Kynesgrove-351-2-0-8-1602105523.7z?md5=hUgu4epNAuzlp8yTUMNPgQ&expires=1621585205&user_id=512579&rip=24.218.205.137")
|
page += 1;
|
||||||
// .header("apikey", env::var("NEXUS_API_KEY")?)
|
}
|
||||||
// .header("user-agent", USER_AGENT)
|
|
||||||
// .send()
|
|
||||||
// .await?;
|
|
||||||
// dbg!(&res);
|
|
||||||
// let bytes = res.bytes().await?;
|
|
||||||
// let mut bytes = read("C:\\Users\\tyler\\Downloads\\Crime Overhaul Expanded 1.1-19188-1-1.rar")?;
|
|
||||||
// let mut bytes = read("C:\\Users\\tyler\\Downloads\\YourMarketStall-15814-1-4-2.zip")?;
|
|
||||||
// let mut reader = std::io::Cursor::new(&bytes);
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user