thallada/modmapper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

More efficient batched_get_with_cells

Browse Source
This commit is contained in:
Tyler Hallada 2022-01-24 22:33:50 -05:00
parent 17facae842
commit 0c3d85d697
2 changed files with 11 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/main.rs
View File

@ -249,9 +249,9 @@ pub async fn main() -> Result<()> {
if let Some(mod_data_dir) = args.mod_data { if let Some(mod_data_dir) = args.mod_data {
let page_size = 20; let page_size = 20;
let mut page = 0; let mut last_id = None;
loop { loop {
let mods = game_mod::batched_get_with_cells(&pool, page_size, page).await?; let mods = game_mod::batched_get_with_cells(&pool, page_size, last_id).await?;
if mods.is_empty() { if mods.is_empty() {
break; break;
} }
@ -261,8 +261,8 @@ pub async fn main() -> Result<()> {
let path = path.join(format!("{}.json", mod_with_cells.nexus_mod_id)); let path = path.join(format!("{}.json", mod_with_cells.nexus_mod_id));
let mut file = std::fs::File::create(path)?; let mut file = std::fs::File::create(path)?;
write!(file, "{}", serde_json::to_string(&mod_with_cells)?)?; write!(file, "{}", serde_json::to_string(&mod_with_cells)?)?;
last_id = Some(mod_with_cells.id);
} }
page += 1;
} }
return Ok(()); return Ok(());
} }

17
src/models/game_mod.rs
View File

@ -322,24 +322,23 @@ pub async fn update_from_api_response<'a>(
pub async fn batched_get_with_cells( pub async fn batched_get_with_cells(
pool: &sqlx::Pool<sqlx::Postgres>, pool: &sqlx::Pool<sqlx::Postgres>,
page_size: i64, page_size: i64,
page: i64, last_id: Option<i32>,
) -> Result<Vec<ModWithCells>> { ) -> Result<Vec<ModWithCells>> {
let offset = page_size * page; let last_id = last_id.unwrap_or(0);
sqlx::query_as!( sqlx::query_as!(
ModWithCells, ModWithCells,
"SELECT "SELECT
mods.*, mods.*,
json_agg(cells.*) AS cells COALESCE(json_agg(DISTINCT jsonb_build_object('x', cells.x, 'y', cells.y)) FILTER (WHERE cells.x IS NOT NULL AND cells.y IS NOT NULL), '[]') AS cells
FROM mods FROM mods
JOIN files ON files.mod_id = mods.id JOIN plugin_cells ON plugin_cells.mod_id = mods.id
JOIN plugins ON plugins.file_id = files.id
JOIN plugin_cells ON plugin_cells.plugin_id = plugins.id
JOIN cells ON cells.id = plugin_cells.cell_id JOIN cells ON cells.id = plugin_cells.cell_id
WHERE mods.id > $2
GROUP BY mods.id GROUP BY mods.id
LIMIT $1 ORDER BY mods.id ASC
OFFSET $2", LIMIT $1",
page_size, page_size,
offset, last_id,
) )
.fetch_all(pool) .fetch_all(pool)
.await .await