WIP adding interior_refs endpoints

Ran into some limitations of sqlx while trying to bulk create interior_refs. I
also discovered how slow creating hundreds of rows in postgres is and I'm
planning on saving interior_refs data in a jsonb column instead which seems to
be much faster.

Cargo.lock

@@ -1615,6 +1615,7 @@ dependencies = [
  "pretty_env_logger 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "refinery 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_json 1.0.56 (registry+https://github.com/rust-lang/crates.io-index)",
  "sqlx 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)",
  "url 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1706,6 +1707,8 @@ dependencies = [
  "memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_json 1.0.56 (registry+https://github.com/rust-lang/crates.io-index)",
  "sha-1 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "sha2 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "sqlformat 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1725,6 +1728,7 @@ dependencies = [
  "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "proc-macro2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)",
  "quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_json 1.0.56 (registry+https://github.com/rust-lang/crates.io-index)",
  "sqlx-core 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "syn 1.0.34 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)",

Cargo.toml

@@ -16,12 +16,14 @@ listenfd = "0.3"
 log = "0.4"
 pretty_env_logger = "0.4"
 tokio = { version = "0.2", features = ["macros"] }
-sqlx = { version = "0.3", default-features = false, features = [ "runtime-tokio", "macros", "postgres", "chrono", "uuid", "ipnetwork" ] }
+sqlx = { version = "0.3", default-features = false, features = [ "runtime-tokio", "macros", "postgres", "chrono", 
+"uuid", "ipnetwork", "json" ] }
 warp = { version = "0.2", features = ["compression"] }
 refinery = { version = "0.3.0", features = [ "tokio-postgres", "tokio" ] }
 barrel = { version = "0.6.5", features = [ "pg" ] }
 clap = "3.0.0-beta.1"
 serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
 uuid = { version = "0.8", features = ["serde", "v4"] }
 ipnetwork = "0.16"
 url = "2.1"

src/db/migrations/V1__initial.rs

@@ -26,6 +26,7 @@ pub fn migration() -> String {
         t.add_column("sell_buy_list_id", types::integer().default(0));
         t.add_column("vendor_id", types::integer());
         t.add_column("vendor_gold", types::integer());
+        t.add_column("interior_refs", types::custom("jsonb"));
         t.add_column("created_at", types::custom("timestamp(3)"));
         t.add_column("updated_at", types::custom("timestamp(3)"));
         t.add_index(

src/filters/mod.rs

@@ -3,7 +3,7 @@ use std::convert::Infallible;
 use warp::{Filter, Rejection, Reply};
 
 use super::handlers;
-use super::models::{ListParams, Owner, Shop};
+use super::models::{InteriorRef, ListParams, Owner, Shop};
 use super::Environment;
 
 pub fn get_shop(env: Environment) -> impl Filter<Extract = impl Reply, Error = Rejection> + Clone {
@@ -61,6 +61,45 @@ pub fn list_owners(
         .and_then(handlers::list_owners)
 }
 
+pub fn get_interior_ref(
+    env: Environment,
+) -> impl Filter<Extract = impl Reply, Error = Rejection> + Clone {
+    warp::path!("interior_refs" / i32)
+        .and(warp::get())
+        .and(with_env(env))
+        .and_then(handlers::get_interior_ref)
+}
+
+pub fn create_interior_ref(
+    env: Environment,
+) -> impl Filter<Extract = impl Reply, Error = Rejection> + Clone {
+    warp::path!("interior_refs")
+        .and(warp::post())
+        .and(json_body::<InteriorRef>())
+        .and(with_env(env))
+        .and_then(handlers::create_interior_ref)
+}
+
+pub fn list_interior_refs(
+    env: Environment,
+) -> impl Filter<Extract = impl Reply, Error = Rejection> + Clone {
+    warp::path!("interior_refs")
+        .and(warp::get())
+        .and(warp::query::<ListParams>())
+        .and(with_env(env))
+        .and_then(handlers::list_interior_refs)
+}
+
+pub fn bulk_create_interior_refs(
+    env: Environment,
+) -> impl Filter<Extract = impl Reply, Error = Rejection> + Clone {
+    warp::path!("interior_refs" / "bulk")
+        .and(warp::post())
+        .and(json_body::<Vec<InteriorRef>>())
+        .and(with_env(env))
+        .and_then(handlers::bulk_create_interior_refs)
+}
+
 fn with_env(env: Environment) -> impl Filter<Extract = (Environment,), Error = Infallible> + Clone {
     warp::any().map(move || env.clone())
 }
@@ -69,5 +108,5 @@ fn json_body<T>() -> impl Filter<Extract = (T,), Error = warp::Rejection> + Clon
 where
     T: Send + DeserializeOwned,
 {
-    warp::body::content_length_limit(1024 * 16).and(warp::body::json())
+    warp::body::content_length_limit(1024 * 64).and(warp::body::json())
 }

src/handlers/mod.rs

@@ -4,7 +4,7 @@ use warp::http::StatusCode;
 use warp::reply::{json, with_header, with_status};
 use warp::{Rejection, Reply};
 
-use super::models::{ListParams, Model, Owner, Shop};
+use super::models::{InteriorRef, ListParams, Model, Owner, Shop};
 use super::problem::reject_anyhow;
 use super::Environment;
 
@@ -74,3 +74,46 @@ pub async fn create_owner(
     let reply = with_status(reply, StatusCode::CREATED);
     Ok(reply)
 }
+
+pub async fn get_interior_ref(id: i32, env: Environment) -> Result<impl Reply, Rejection> {
+    let interior_ref = InteriorRef::get(&env.db, id).await.map_err(reject_anyhow)?;
+    let reply = json(&interior_ref);
+    let reply = with_status(reply, StatusCode::OK);
+    Ok(reply)
+}
+
+pub async fn list_interior_refs(
+    list_params: ListParams,
+    env: Environment,
+) -> Result<impl Reply, Rejection> {
+    let interior_refs = InteriorRef::list(&env.db, list_params)
+        .await
+        .map_err(reject_anyhow)?;
+    let reply = json(&interior_refs);
+    let reply = with_status(reply, StatusCode::OK);
+    Ok(reply)
+}
+
+pub async fn create_interior_ref(
+    interior_ref: InteriorRef,
+    env: Environment,
+) -> Result<impl Reply, Rejection> {
+    let saved_interior_ref = interior_ref.save(&env.db).await.map_err(reject_anyhow)?;
+    let url = saved_interior_ref
+        .url(&env.api_url)
+        .map_err(reject_anyhow)?;
+    let reply = json(&saved_interior_ref);
+    let reply = with_header(reply, "Location", url.as_str());
+    let reply = with_status(reply, StatusCode::CREATED);
+    Ok(reply)
+}
+
+pub async fn bulk_create_interior_refs(
+    interior_refs: Vec<InteriorRef>,
+    env: Environment,
+) -> Result<impl Reply, Rejection> {
+    InteriorRef::bulk_save(&env.db, interior_refs)
+        .await
+        .map_err(reject_anyhow)?;
+    Ok(StatusCode::CREATED)
+}

src/main.rs

@@ -70,20 +70,30 @@ async fn main() -> Result<()> {
     let api_url = host_url.join("/api/v1")?;
     let env = Environment::new(api_url).await?;
 
-    let home = warp::path!("api" / "v1").map(|| "Shopkeeper home page");
+    let base = warp::path("api").and(warp::path("v1"));
     let get_shop = filters::get_shop(env.clone());
     let create_shop = filters::create_shop(env.clone());
     let list_shops = filters::list_shops(env.clone());
     let get_owner = filters::get_owner(env.clone());
     let create_owner = filters::create_owner(env.clone());
     let list_owners = filters::list_owners(env.clone());
-    let routes = create_shop
+    let get_interior_ref = filters::get_interior_ref(env.clone());
+    let create_interior_ref = filters::create_interior_ref(env.clone());
+    let list_interior_refs = filters::list_interior_refs(env.clone());
+    let bulk_create_interior_refs = filters::bulk_create_interior_refs(env.clone());
+    let routes = base
+        .and(
+            create_shop
                 .or(get_shop)
                 .or(list_shops)
                 .or(create_owner)
                 .or(get_owner)
                 .or(list_owners)
-        .or(home)
+                .or(create_interior_ref)
+                .or(get_interior_ref)
+                .or(list_interior_refs)
+                .or(bulk_create_interior_refs),
+        )
         .recover(problem::unpack_problem)
         .with(warp::compression::gzip())
         .with(warp::log("shopkeeper"));

src/models/interior_ref.rs

@@ -0,0 +1,151 @@
+use anyhow::Result;
+use async_trait::async_trait;
+use chrono::prelude::*;
+use serde::{Deserialize, Serialize};
+use serde_json;
+use sqlx::postgres::PgPool;
+
+use super::ListParams;
+use super::Model;
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct InteriorRef {
+    pub id: Option<i32>,
+    pub shop_id: i32,
+    pub mod_name: String,
+    pub local_form_id: i32,
+    pub position_x: f64,
+    pub position_y: f64,
+    pub position_z: f64,
+    pub angle_x: f64,
+    pub angle_y: f64,
+    pub angle_z: f64,
+    pub scale: f64,
+    pub created_at: Option<NaiveDateTime>,
+}
+
+#[async_trait]
+impl Model for InteriorRef {
+    fn resource_name() -> &'static str {
+        "interior_ref"
+    }
+
+    fn pk(&self) -> Option<i32> {
+        self.id
+    }
+
+    async fn get(db: &PgPool, id: i32) -> Result<Self> {
+        let timer = std::time::Instant::now();
+        let result = sqlx::query_as!(Self, "SELECT * FROM interior_refs WHERE id = $1", id)
+            .fetch_one(db)
+            .await?;
+        let elapsed = timer.elapsed();
+        debug!("SELECT * FROM interior_refs ... {:.3?}", elapsed);
+        Ok(result)
+    }
+
+    async fn save(self, db: &PgPool) -> Result<Self> {
+        let timer = std::time::Instant::now();
+        let result = sqlx::query_as!(
+            Self,
+            "INSERT INTO interior_refs
+            (shop_id, mod_name, local_form_id, position_x, position_y, position_z, angle_x,
+            angle_y, angle_z, scale, created_at)
+            VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, now())
+            RETURNING *",
+            self.shop_id,
+            self.mod_name,
+            self.local_form_id,
+            self.position_x,
+            self.position_y,
+            self.position_z,
+            self.angle_x,
+            self.angle_y,
+            self.angle_z,
+            self.scale,
+        )
+        .fetch_one(db)
+        .await?;
+        let elapsed = timer.elapsed();
+        debug!("INSERT INTO interior_refs ... {:.3?}", elapsed);
+        Ok(result)
+    }
+
+    async fn list(db: &PgPool, list_params: ListParams) -> Result<Vec<Self>> {
+        let timer = std::time::Instant::now();
+        let result = if let Some(order_by) = list_params.get_order_by() {
+            sqlx::query_as!(
+                Self,
+                "SELECT * FROM interior_refs
+                ORDER BY $1
+                LIMIT $2
+                OFFSET $3",
+                order_by,
+                list_params.limit.unwrap_or(10),
+                list_params.offset.unwrap_or(0),
+            )
+            .fetch_all(db)
+            .await?
+        } else {
+            sqlx::query_as!(
+                Self,
+                "SELECT * FROM interior_refs
+                LIMIT $1
+                OFFSET $2",
+                list_params.limit.unwrap_or(10),
+                list_params.offset.unwrap_or(0),
+            )
+            .fetch_all(db)
+            .await?
+        };
+        let elapsed = timer.elapsed();
+        debug!("SELECT * FROM interior_refs ... {:.3?}", elapsed);
+        Ok(result)
+    }
+
+    // TODO: figure out a way bulk insert in a single query
+    // see: https://github.com/launchbadge/sqlx/issues/294
+    async fn bulk_save(db: &PgPool, interior_refs: Vec<Self>) -> Result<()> {
+        let timer = std::time::Instant::now();
+        // Testing whether setting a jsonb column with an array of 200 items is faster than
+        // inserting 200 rows. Answer: it is a hell of a lot faster!
+        // TODO:
+        // 1. remove interior_refs column from shops
+        // 2. replace all columns in interior_refs table with single `refs` jsonb column and
+        //    shops_id foreign_key
+        // 3. This function will now create the row in that table
+        // 4. Decide if I'll need to make the same changes to merchandise and transactions
+        //      - answer depends on how many rows of each I expect to insert in one go
+        sqlx::query!(
+            "UPDATE shops SET interior_refs = $1::jsonb",
+            serde_json::to_value(&interior_refs)?,
+        )
+        .execute(db)
+        .await?;
+        // let mut transaction = db.begin().await?;
+        // for interior_ref in interior_refs {
+        // sqlx::query!(
+        // "INSERT INTO interior_refs
+        // (shop_id, mod_name, local_form_id, position_x, position_y, position_z, angle_x,
+        // angle_y, angle_z, scale, created_at)
+        // VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, now())",
+        // interior_ref.shop_id,
+        // interior_ref.mod_name,
+        // interior_ref.local_form_id,
+        // interior_ref.position_x,
+        // interior_ref.position_y,
+        // interior_ref.position_z,
+        // interior_ref.angle_x,
+        // interior_ref.angle_y,
+        // interior_ref.angle_z,
+        // interior_ref.scale,
+        // )
+        // .execute(&mut transaction)
+        // .await?;
+        // }
+        // transaction.commit().await?;
+        let elapsed = timer.elapsed();
+        debug!("INSERT INTO interior_refs ... {:.3?}", elapsed);
+        Ok(())
+    }
+}

src/models/mod.rs

@@ -4,10 +4,12 @@ use std::fmt;
 pub mod model;
 pub mod owner;
 pub mod shop;
+pub mod interior_ref;
 
 pub use model::Model;
 pub use owner::Owner;
 pub use shop::Shop;
+pub use interior_ref::InteriorRef;
 
 #[derive(Debug, Deserialize)]
 pub enum Order {

src/models/model.rs

@@ -25,4 +25,7 @@ where
     async fn get(db: &PgPool, id: i32) -> Result<Self>;
     async fn save(self, db: &PgPool) -> Result<Self>;
     async fn list(db: &PgPool, list_params: ListParams) -> Result<Vec<Self>>;
+    async fn bulk_save(_db: &PgPool, _models: Vec<Self>) -> Result<()> {
+        unimplemented!()
+    }
 }

src/models/shop.rs

@@ -2,6 +2,7 @@ use anyhow::Result;
 use async_trait::async_trait;
 use chrono::prelude::*;
 use serde::{Deserialize, Serialize};
+use serde_json;
 use sqlx::postgres::PgPool;
 
 use super::ListParams;
@@ -17,6 +18,7 @@ pub struct Shop {
     pub sell_buy_list_id: i32,
     pub vendor_id: i32,
     pub vendor_gold: i32,
+    pub interior_refs: serde_json::value::Value,
     pub created_at: Option<NaiveDateTime>,
     pub updated_at: Option<NaiveDateTime>,
 }
@@ -47,8 +49,8 @@ impl Model for Shop {
             Self,
             "INSERT INTO shops
             (name, owner_id, description, is_not_sell_buy, sell_buy_list_id, vendor_id,
-             vendor_gold, created_at, updated_at)
+             vendor_gold, interior_refs, created_at, updated_at)
-            VALUES ($1, $2, $3, $4, $5, $6, $7, now(), now())
+            VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now(), now())
             RETURNING *",
             self.name,
             self.owner_id,
@@ -57,6 +59,7 @@ impl Model for Shop {
             self.sell_buy_list_id,
             self.vendor_id,
             self.vendor_gold,
+            self.interior_refs,
         )
         .fetch_one(db)
         .await?;