Add ancestor id columns to plugins and plugin_cells

With temp backfill queries. Second stage will to make the columns nullable and add foreign keys.

I'm hoping that this will improve query performance.

migrations/20220125012642_add_ancestor_ids_to_plugins_and_plugin_cells.sql

@@ -0,0 +1,4 @@
+/* TODO: make these non-nullable and add foreign keys */
+ALTER TABLE "plugins" ADD COLUMN "mod_id" INTEGER;
+ALTER TABLE "plugin_cells" ADD COLUMN "file_id" INTEGER;
+ALTER TABLE "plugin_cells" ADD COLUMN "mod_id" INTEGER;

src/main.rs

@@ -215,6 +215,35 @@ pub async fn main() -> Result<()> {
         .connect(&env::var("DATABASE_URL")?)
         .await?;
 
+    // Temporary backfill
+    sqlx::query!(
+        r#"UPDATE plugins
+        SET mod_id = files.mod_id
+        FROM files
+        WHERE
+            files.id = plugins.file_id AND
+            plugins.mod_id IS NULL
+        "#,
+    )
+    .execute(&pool)
+    .await?;
+    sqlx::query!(
+        r#"UPDATE plugin_cells
+        SET
+            file_id = plugins.file_id,
+            mod_id = files.mod_id
+        FROM plugins
+        JOIN files ON plugins.file_id = files.id
+        WHERE
+            plugins.id = plugin_cells.plugin_id AND
+            plugin_cells.file_id IS NULL AND
+            plugin_cells.mod_id IS NULL
+        "#,
+    )
+    .execute(&pool)
+    .await?;
+    return Ok(());
+
     let args: Args = argh::from_env();
 
     if let Some(dump_edits) = args.dump_edits {

src/models/cell.rs

@@ -130,8 +130,8 @@ pub async fn count_mod_edits(
             FROM cells
             JOIN plugin_cells on cells.id = cell_id
             JOIN plugins ON plugins.id = plugin_id
-            JOIN files ON files.id = file_id
+            JOIN files ON files.id = plugins.file_id
-            JOIN mods ON mods.id = mod_id
+            JOIN mods ON mods.id = files.mod_id
             WHERE master = $1 AND world_id = $2 AND x = $3 and y = $4",
         master,
         world_id,
@@ -166,8 +166,8 @@ pub async fn get_cell_data(
             FROM cells
             JOIN plugin_cells on cells.id = cell_id
             JOIN plugins ON plugins.id = plugin_id
-            JOIN files ON files.id = file_id
+            JOIN files ON files.id = plugins.file_id
-            JOIN mods ON mods.id = mod_id
+            JOIN mods ON mods.id = files.mod_id
             WHERE master = $1 AND world_id = $2 AND x = $3 and y = $4
             GROUP BY cells.x, cells.y, cells.is_persistent, cells.form_id"#,
         master,

src/models/plugin.rs

@@ -10,6 +10,7 @@ pub struct Plugin {
     pub name: String,
     pub hash: i64,
     pub file_id: i32,
+    pub mod_id: Option<i32>,
     pub version: f64,
     pub size: i64,
     pub author: Option<String>,
@@ -26,6 +27,7 @@ pub struct UnsavedPlugin<'a> {
     pub name: &'a str,
     pub hash: i64,
     pub file_id: i32,
+    pub mod_id: Option<i32>,
     pub version: f64,
     pub size: i64,
     pub author: Option<&'a str>,
@@ -43,16 +45,17 @@ pub async fn insert<'a>(
     // sqlx doesn't understand slices of &str with the query_as! macro: https://github.com/launchbadge/sqlx/issues/280
     sqlx::query_as(
         r#"INSERT INTO plugins
-            (name, hash, file_id, version, size, author, description, masters, file_name, file_path, created_at, updated_at)
+            (name, hash, file_id, mod_id, version, size, author, description, masters, file_name, file_path, created_at, updated_at)
-            VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, now(), now())
+            VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, now(), now())
             ON CONFLICT (file_id, file_path) DO UPDATE
-            SET (name, hash, version, author, description, masters, file_name, updated_at) =
+            SET (name, hash, mod_id, version, author, description, masters, file_name, updated_at) =
-            (EXCLUDED.name, EXCLUDED.hash, EXCLUDED.version, EXCLUDED.author, EXCLUDED.description, EXCLUDED.masters, EXCLUDED.file_name, now())
+            (EXCLUDED.name, EXCLUDED.hash, EXCLUDED.mod_id, EXCLUDED.version, EXCLUDED.author, EXCLUDED.description, EXCLUDED.masters, EXCLUDED.file_name, now())
             RETURNING *"#,
     )
     .bind(unsaved_plugin.name)
     .bind(unsaved_plugin.hash)
     .bind(unsaved_plugin.file_id)
+    .bind(unsaved_plugin.mod_id)
     .bind(unsaved_plugin.version)
     .bind(unsaved_plugin.size)
     .bind(unsaved_plugin.author)

src/models/plugin_cell.rs

@@ -11,6 +11,8 @@ pub struct PluginCell {
     pub id: i32,
     pub plugin_id: i32,
     pub cell_id: i32,
+    pub file_id: Option<i32>,
+    pub mod_id: Option<i32>,
     pub editor_id: Option<String>,
     pub updated_at: NaiveDateTime,
     pub created_at: NaiveDateTime,
@@ -20,6 +22,8 @@ pub struct PluginCell {
 pub struct UnsavedPluginCell<'a> {
     pub plugin_id: i32,
     pub cell_id: i32,
+    pub file_id: Option<i32>,
+    pub mod_id: Option<i32>,
     pub editor_id: Option<&'a str>,
 }
 
@@ -28,18 +32,22 @@ pub async fn insert(
     pool: &sqlx::Pool<sqlx::Postgres>,
     plugin_id: i32,
     cell_id: i32,
+    file_id: Option<i32>,
+    mod_id: Option<i32>,
     editor_id: Option<String>,
 ) -> Result<PluginCell> {
     sqlx::query_as!(
         PluginCell,
         "INSERT INTO plugin_cells
-            (plugin_id, cell_id, editor_id, created_at, updated_at)
+            (plugin_id, cell_id, file_id, mod_id, editor_id, created_at, updated_at)
-            VALUES ($1, $2, $3, now(), now())
+            VALUES ($1, $2, $3, $4, $5, now(), now())
             ON CONFLICT (plugin_id, cell_id) DO UPDATE
             SET (editor_id, updated_at) = (EXCLUDED.editor_id, now())
             RETURNING *",
         plugin_id,
         cell_id,
+        file_id,
+        mod_id,
         editor_id,
     )
     .fetch_one(pool)
@@ -56,23 +64,29 @@ pub async fn batched_insert<'a>(
     for batch in plugin_cells.chunks(BATCH_SIZE) {
         let mut plugin_ids: Vec<i32> = vec![];
         let mut cell_ids: Vec<i32> = vec![];
+        let mut file_ids: Vec<Option<i32>> = vec![];
+        let mut mod_ids: Vec<Option<i32>> = vec![];
         let mut editor_ids: Vec<Option<&str>> = vec![];
         batch.iter().for_each(|unsaved_plugin_cell| {
             plugin_ids.push(unsaved_plugin_cell.plugin_id);
             cell_ids.push(unsaved_plugin_cell.cell_id);
+            file_ids.push(unsaved_plugin_cell.file_id);
+            mod_ids.push(unsaved_plugin_cell.mod_id);
             editor_ids.push(unsaved_plugin_cell.editor_id);
         });
         saved_plugin_cells.append(
             // sqlx doesn't understand arrays of Options with the query_as! macro
             &mut sqlx::query_as(
-                r#"INSERT INTO plugin_cells (plugin_id, cell_id, editor_id, created_at, updated_at)
+                r#"INSERT INTO plugin_cells (plugin_id, cell_id, file_id, mod_id, editor_id, created_at, updated_at)
-                SELECT *, now(), now() FROM UNNEST($1::int[], $2::int[], $3::text[])
+                SELECT *, now(), now() FROM UNNEST($1::int[], $2::int[], $3::int[], $4::int[], $5::text[])
                 ON CONFLICT (plugin_id, cell_id) DO UPDATE
-                SET (editor_id, updated_at) = (EXCLUDED.editor_id, now())
+                SET (file_id, mod_id, editor_id, updated_at) = (EXCLUDED.file_id, EXCLUDED.mod_id, EXCLUDED.editor_id, now())
                 RETURNING *"#,
             )
             .bind(&plugin_ids)
             .bind(&cell_ids)
+            .bind(&file_ids)
+            .bind(&mod_ids)
             .bind(&editor_ids)
             .fetch_all(pool)
             .await

src/plugin_processor.rs

@@ -9,8 +9,8 @@ use tracing::{info, warn};
 
 use crate::models::file::File;
 use crate::models::game_mod::Mod;
-use crate::models::{plugin, plugin::UnsavedPlugin};
 use crate::models::{cell, cell::UnsavedCell};
+use crate::models::{plugin, plugin::UnsavedPlugin};
 use crate::models::{plugin_cell, plugin_cell::UnsavedPluginCell};
 use crate::models::{plugin_world, plugin_world::UnsavedPluginWorld};
 use crate::models::{world, world::UnsavedWorld};
@@ -62,6 +62,7 @@ pub async fn process_plugin(
                     name: &db_file.name,
                     hash: hash as i64,
                     file_id: db_file.id,
+                    mod_id: Some(db_mod.id),
                     version: plugin.header.version as f64,
                     size: plugin_buf.len() as i64,
                     author,
@@ -133,6 +134,8 @@ pub async fn process_plugin(
                 .map(|(db_cell, plugin_cell)| UnsavedPluginCell {
                     plugin_id: plugin_row.id,
                     cell_id: db_cell.id,
+                    file_id: Some(db_file.id),
+                    mod_id: Some(db_mod.id),
                     editor_id: plugin_cell.editor_id.as_ref().map(|id| id.as_ref()),
                 })
                 .collect();