thallada/modmapper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add ancestor id columns to plugins and plugin_cells

Browse Source 
With temp backfill queries. Second stage will to make the columns nullable and add foreign keys.

I'm hoping that this will improve query performance.
This commit is contained in:
Tyler Hallada 2022-01-24 21:13:07 -05:00
parent 4a8e3cad6c
commit e5936683fa
6 changed files with 67 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
migrations/20220125012642_add_ancestor_ids_to_plugins_and_plugin_cells.sql Normal file
View File

@ -0,0 +1,4 @@
/* TODO: make these non-nullable and add foreign keys */
ALTER TABLE "plugins" ADD COLUMN "mod_id" INTEGER;
ALTER TABLE "plugin_cells" ADD COLUMN "file_id" INTEGER;
ALTER TABLE "plugin_cells" ADD COLUMN "mod_id" INTEGER;

29
src/main.rs
View File

@ -215,6 +215,35 @@ pub async fn main() -> Result<()> {
.connect(&env::var("DATABASE_URL")?)
.await?;
// Temporary backfill
sqlx::query!(
r#"UPDATE plugins
SET mod_id = files.mod_id
FROM files
WHERE
files.id = plugins.file_id AND
plugins.mod_id IS NULL
"#,
)
.execute(&pool)
.await?;
sqlx::query!(
r#"UPDATE plugin_cells
SET
file_id = plugins.file_id,
mod_id = files.mod_id
FROM plugins
JOIN files ON plugins.file_id = files.id
WHERE
plugins.id = plugin_cells.plugin_id AND
plugin_cells.file_id IS NULL AND
plugin_cells.mod_id IS NULL
"#,
)
.execute(&pool)
.await?;
return Ok(());
let args: Args = argh::from_env();
if let Some(dump_edits) = args.dump_edits {

8
src/models/cell.rs
View File

@ -130,8 +130,8 @@ pub async fn count_mod_edits(
FROM cells
JOIN plugin_cells on cells.id = cell_id
JOIN plugins ON plugins.id = plugin_id
JOIN files ON files.id = file_id
JOIN mods ON mods.id = mod_id
JOIN files ON files.id = plugins.file_id
JOIN mods ON mods.id = files.mod_id
WHERE master = $1 AND world_id = $2 AND x = $3 and y = $4",
master,
world_id,
@ -166,8 +166,8 @@ pub async fn get_cell_data(
FROM cells
JOIN plugin_cells on cells.id = cell_id
JOIN plugins ON plugins.id = plugin_id
JOIN files ON files.id = file_id
JOIN mods ON mods.id = mod_id
JOIN files ON files.id = plugins.file_id
JOIN mods ON mods.id = files.mod_id
WHERE master = $1 AND world_id = $2 AND x = $3 and y = $4
GROUP BY cells.x, cells.y, cells.is_persistent, cells.form_id"#,
master,

11
src/models/plugin.rs
View File

@ -10,6 +10,7 @@ pub struct Plugin {
pub name: String,
pub hash: i64,
pub file_id: i32,
pub mod_id: Option<i32>,
pub version: f64,
pub size: i64,
pub author: Option<String>,
@ -26,6 +27,7 @@ pub struct UnsavedPlugin<'a> {
pub name: &'a str,
pub hash: i64,
pub file_id: i32,
pub mod_id: Option<i32>,
pub version: f64,
pub size: i64,
pub author: Option<&'a str>,
@ -43,16 +45,17 @@ pub async fn insert<'a>(
// sqlx doesn't understand slices of &str with the query_as! macro: https://github.com/launchbadge/sqlx/issues/280
sqlx::query_as(
r#"INSERT INTO plugins
(name, hash, file_id, version, size, author, description, masters, file_name, file_path, created_at, updated_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, now(), now())
(name, hash, file_id, mod_id, version, size, author, description, masters, file_name, file_path, created_at, updated_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, now(), now())
ON CONFLICT (file_id, file_path) DO UPDATE
SET (name, hash, version, author, description, masters, file_name, updated_at) =
(EXCLUDED.name, EXCLUDED.hash, EXCLUDED.version, EXCLUDED.author, EXCLUDED.description, EXCLUDED.masters, EXCLUDED.file_name, now())
SET (name, hash, mod_id, version, author, description, masters, file_name, updated_at) =
(EXCLUDED.name, EXCLUDED.hash, EXCLUDED.mod_id, EXCLUDED.version, EXCLUDED.author, EXCLUDED.description, EXCLUDED.masters, EXCLUDED.file_name, now())
RETURNING *"#,
)
.bind(unsaved_plugin.name)
.bind(unsaved_plugin.hash)
.bind(unsaved_plugin.file_id)
.bind(unsaved_plugin.mod_id)
.bind(unsaved_plugin.version)
.bind(unsaved_plugin.size)
.bind(unsaved_plugin.author)

24
src/models/plugin_cell.rs
View File

@ -11,6 +11,8 @@ pub struct PluginCell {
pub id: i32,
pub plugin_id: i32,
pub cell_id: i32,
pub file_id: Option<i32>,
pub mod_id: Option<i32>,
pub editor_id: Option<String>,
pub updated_at: NaiveDateTime,
pub created_at: NaiveDateTime,
@ -20,6 +22,8 @@ pub struct PluginCell {
pub struct UnsavedPluginCell<'a> {
pub plugin_id: i32,
pub cell_id: i32,
pub file_id: Option<i32>,
pub mod_id: Option<i32>,
pub editor_id: Option<&'a str>,
}
@ -28,18 +32,22 @@ pub async fn insert(
pool: &sqlx::Pool<sqlx::Postgres>,
plugin_id: i32,
cell_id: i32,
file_id: Option<i32>,
mod_id: Option<i32>,
editor_id: Option<String>,
) -> Result<PluginCell> {
sqlx::query_as!(
PluginCell,
"INSERT INTO plugin_cells
(plugin_id, cell_id, editor_id, created_at, updated_at)
VALUES ($1, $2, $3, now(), now())
(plugin_id, cell_id, file_id, mod_id, editor_id, created_at, updated_at)
VALUES ($1, $2, $3, $4, $5, now(), now())
ON CONFLICT (plugin_id, cell_id) DO UPDATE
SET (editor_id, updated_at) = (EXCLUDED.editor_id, now())
RETURNING *",
plugin_id,
cell_id,
file_id,
mod_id,
editor_id,
)
.fetch_one(pool)
@ -56,23 +64,29 @@ pub async fn batched_insert<'a>(
for batch in plugin_cells.chunks(BATCH_SIZE) {
let mut plugin_ids: Vec<i32> = vec![];
let mut cell_ids: Vec<i32> = vec![];
let mut file_ids: Vec<Option<i32>> = vec![];
let mut mod_ids: Vec<Option<i32>> = vec![];
let mut editor_ids: Vec<Option<&str>> = vec![];
batch.iter().for_each(|unsaved_plugin_cell| {
plugin_ids.push(unsaved_plugin_cell.plugin_id);
cell_ids.push(unsaved_plugin_cell.cell_id);
file_ids.push(unsaved_plugin_cell.file_id);
mod_ids.push(unsaved_plugin_cell.mod_id);
editor_ids.push(unsaved_plugin_cell.editor_id);
});
saved_plugin_cells.append(
// sqlx doesn't understand arrays of Options with the query_as! macro
&mut sqlx::query_as(
r#"INSERT INTO plugin_cells (plugin_id, cell_id, editor_id, created_at, updated_at)
SELECT *, now(), now() FROM UNNEST($1::int[], $2::int[], $3::text[])
r#"INSERT INTO plugin_cells (plugin_id, cell_id, file_id, mod_id, editor_id, created_at, updated_at)
SELECT *, now(), now() FROM UNNEST($1::int[], $2::int[], $3::int[], $4::int[], $5::text[])
ON CONFLICT (plugin_id, cell_id) DO UPDATE
SET (editor_id, updated_at) = (EXCLUDED.editor_id, now())
SET (file_id, mod_id, editor_id, updated_at) = (EXCLUDED.file_id, EXCLUDED.mod_id, EXCLUDED.editor_id, now())
RETURNING *"#,
)
.bind(&plugin_ids)
.bind(&cell_ids)
.bind(&file_ids)
.bind(&mod_ids)
.bind(&editor_ids)
.fetch_all(pool)
.await

5
src/plugin_processor.rs
View File

@ -9,8 +9,8 @@ use tracing::{info, warn};
use crate::models::file::File;
use crate::models::game_mod::Mod;
use crate::models::{plugin, plugin::UnsavedPlugin};
use crate::models::{cell, cell::UnsavedCell};
use crate::models::{plugin, plugin::UnsavedPlugin};
use crate::models::{plugin_cell, plugin_cell::UnsavedPluginCell};
use crate::models::{plugin_world, plugin_world::UnsavedPluginWorld};
use crate::models::{world, world::UnsavedWorld};
@ -62,6 +62,7 @@ pub async fn process_plugin(
name: &db_file.name,
hash: hash as i64,
file_id: db_file.id,
mod_id: Some(db_mod.id),
version: plugin.header.version as f64,
size: plugin_buf.len() as i64,
author,
@ -133,6 +134,8 @@ pub async fn process_plugin(
.map(|(db_cell, plugin_cell)| UnsavedPluginCell {
plugin_id: plugin_row.id,
cell_id: db_cell.id,
file_id: Some(db_file.id),
mod_id: Some(db_mod.id),
editor_id: plugin_cell.editor_id.as_ref().map(|id| id.as_ref()),
})
.collect();