thallada/modmapper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Scrape additional fields to mod table

Browse Source
This commit is contained in:
Tyler Hallada
2022-01-17 23:37:58 -05:00
parent f23cf526e5
commit b97689b7fd
4 changed files with 147 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
src/models/game_mod.rs
View File

@@ -11,12 +11,17 @@ pub struct Mod {
pub id: i32,
pub name: String,
pub nexus_mod_id: i32,
pub author: String,
pub category: Option<String>,
pub author_name: String,
pub author_id: Option<i32>,
pub category_name: Option<String>,
pub category_id: Option<i32>,
pub description: Option<String>,
pub thumbnail_link: Option<String>,
pub game_id: i32,
pub updated_at: NaiveDateTime,
pub created_at: NaiveDateTime,
pub last_update_at: Option<NaiveDateTime>,
pub first_upload_at: Option<NaiveDateTime>,
pub last_updated_files_at: Option<NaiveDateTime>,
}
@@ -24,10 +29,15 @@ pub struct Mod {
pub struct UnsavedMod<'a> {
pub name: &'a str,
pub nexus_mod_id: i32,
pub author: &'a str,
pub category: Option<&'a str>,
pub author_name: &'a str,
pub author_id: Option<i32>,
pub category_name: Option<&'a str>,
pub category_id: Option<i32>,
pub description: Option<&'a str>,
pub thumbnail_link: Option<&'a str>,
pub game_id: i32,
pub last_update_at: Option<NaiveDateTime>,
pub first_upload_at: Option<NaiveDateTime>,
}
#[instrument(level = "debug", skip(pool))]
@@ -77,26 +87,36 @@ pub async fn insert(
pool: &sqlx::Pool<sqlx::Postgres>,
name: &str,
nexus_mod_id: i32,
author: &str,
category: Option<&str>,
author_name: &str,
author_id: i32,
category_name: Option<&str>,
category_id: Option<i32>,
description: Option<&str>,
thumbnail_link: Option<&str>,
game_id: i32,
last_update_at: Option<NaiveDateTime>,
first_upload_at: Option<NaiveDateTime>,
) -> Result<Mod> {
sqlx::query_as!(
Mod,
"INSERT INTO mods
(name, nexus_mod_id, author, category, description, game_id, created_at, updated_at)
VALUES ($1, $2, $3, $4, $5, $6, now(), now())
(name, nexus_mod_id, author_name, author_id, category_name, category_id, description, thumbnail_link, game_id, last_update_at, first_upload_at, created_at, updated_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, now(), now())
ON CONFLICT (game_id, nexus_mod_id) DO UPDATE
SET (name, author, category, description, updated_at) =
(EXCLUDED.name, EXCLUDED.author, EXCLUDED.category, EXCLUDED.description, now())
SET (name, author_name, author_id, category_name, category_id, description, thumbnail_link, last_update_at, first_upload_at, updated_at) =
(EXCLUDED.name, EXCLUDED.author_name, EXCLUDED.author_id, EXCLUDED.category_name, EXCLUDED.category_id, EXCLUDED.description, EXCLUDED.thumbnail_link, EXCLUDED.last_update_at, EXCLUDED.first_upload_at, now())
RETURNING *",
name,
nexus_mod_id,
author,
category,
author_name,
author_id,
category_name,
category_id,
description,
game_id
thumbnail_link,
game_id,
last_update_at,
first_upload_at
)
.fetch_one(pool)
.await
@@ -112,36 +132,51 @@ pub async fn batched_insert<'a>(
for batch in mods.chunks(BATCH_SIZE) {
let mut names: Vec<&str> = vec![];
let mut nexus_mod_ids: Vec<i32> = vec![];
let mut authors: Vec<&str> = vec![];
let mut categories: Vec<Option<&str>> = vec![];
let mut author_names: Vec<&str> = vec![];
let mut author_ids: Vec<Option<i32>> = vec![];
let mut category_names: Vec<Option<&str>> = vec![];
let mut category_ids: Vec<Option<i32>> = vec![];
let mut descriptions: Vec<Option<&str>> = vec![];
let mut thumbnail_links: Vec<Option<&str>> = vec![];
let mut game_ids: Vec<i32> = vec![];
let mut last_update_ats: Vec<Option<NaiveDateTime>> = vec![];
let mut first_upload_ats: Vec<Option<NaiveDateTime>> = vec![];
batch.iter().for_each(|unsaved_mod| {
names.push(unsaved_mod.name);
nexus_mod_ids.push(unsaved_mod.nexus_mod_id);
authors.push(unsaved_mod.author);
categories.push(unsaved_mod.category);
author_names.push(unsaved_mod.author_name);
author_ids.push(unsaved_mod.author_id);
category_names.push(unsaved_mod.category_name);
category_ids.push(unsaved_mod.category_id);
descriptions.push(unsaved_mod.description);
thumbnail_links.push(unsaved_mod.thumbnail_link);
game_ids.push(unsaved_mod.game_id);
last_update_ats.push(unsaved_mod.last_update_at);
first_upload_ats.push(unsaved_mod.first_upload_at);
});
saved_mods.append(
// sqlx doesn't understand arrays of Options with the query_as! macro
&mut sqlx::query_as(
r#"INSERT INTO mods
(name, nexus_mod_id, author, category, description, game_id, created_at, updated_at)
(name, nexus_mod_id, author_name, author_id, category_name, category_id, description, thumbnail_link, game_id, last_update_at, first_upload_at, created_at, updated_at)
SELECT *, now(), now()
FROM UNNEST($1::text[], $2::int[], $3::text[], $4::text[], $5::text[], $6::int[])
FROM UNNEST($1::text[], $2::int[], $3::text[], $4::int[], $5::text[], $6::int[], $7::text[], $8::text[], $9::int[], $10::timestamp(3)[], $11::timestamp(3)[])
ON CONFLICT (game_id, nexus_mod_id) DO UPDATE
SET (name, author, category, description, updated_at) =
(EXCLUDED.name, EXCLUDED.author, EXCLUDED.category, EXCLUDED.description, now())
SET (name, author_name, author_id, category_name, category_id, description, thumbnail_link, last_update_at, first_upload_at, updated_at) =
(EXCLUDED.name, EXCLUDED.author_name, EXCLUDED.author_id, EXCLUDED.category_name, EXCLUDED.category_id, EXCLUDED.description, EXCLUDED.thumbnail_link, EXCLUDED.last_update_at, EXCLUDED.first_upload_at, now())
RETURNING *"#,
)
.bind(&names)
.bind(&nexus_mod_ids)
.bind(&authors)
.bind(&categories)
.bind(&author_names)
.bind(&author_ids)
.bind(&category_names)
.bind(&category_ids)
.bind(&descriptions)
.bind(&thumbnail_links)
.bind(&game_ids)
.bind(&last_update_ats)
.bind(&first_upload_ats)
.fetch_all(pool)
.await
.context("Failed to insert mods")?,