thallada/BazaarRealmAPI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP adding interior_refs endpoints

Browse Source 
Ran into some limitations of sqlx while trying to bulk create interior_refs. I
also discovered how slow creating hundreds of rows in postgres is and I'm
planning on saving interior_refs data in a jsonb column instead which seems to
be much faster.
This commit is contained in:
Tyler Hallada 2020-07-19 03:01:20 -04:00
parent 9985f123c9
commit 65e6ba1f8a
10 changed files with 272 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Cargo.lock generated
View File

@ -1615,6 +1615,7 @@ dependencies = [
"pretty_env_logger 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"refinery 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.56 (registry+https://github.com/rust-lang/crates.io-index)",
"sqlx 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)",
"url 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1706,6 +1707,8 @@ dependencies = [
"memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
"percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.56 (registry+https://github.com/rust-lang/crates.io-index)",
"sha-1 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)",
"sha2 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)",
"sqlformat 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1725,6 +1728,7 @@ dependencies = [
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"proc-macro2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.56 (registry+https://github.com/rust-lang/crates.io-index)",
"sqlx-core 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 1.0.34 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)",

4
Cargo.toml
View File

@ -16,12 +16,14 @@ listenfd = "0.3"
log = "0.4"
pretty_env_logger = "0.4"
tokio = { version = "0.2", features = ["macros"] }
sqlx = { version = "0.3", default-features = false, features = [ "runtime-tokio", "macros", "postgres", "chrono", "uuid", "ipnetwork" ] }
sqlx = { version = "0.3", default-features = false, features = [ "runtime-tokio", "macros", "postgres", "chrono",
"uuid", "ipnetwork", "json" ] }
warp = { version = "0.2", features = ["compression"] }
refinery = { version = "0.3.0", features = [ "tokio-postgres", "tokio" ] }
barrel = { version = "0.6.5", features = [ "pg" ] }
clap = "3.0.0-beta.1"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
uuid = { version = "0.8", features = ["serde", "v4"] }
ipnetwork = "0.16"
url = "2.1"

1
src/db/migrations/V1__initial.rs
View File

@ -26,6 +26,7 @@ pub fn migration() -> String {
t.add_column("sell_buy_list_id", types::integer().default(0));
t.add_column("vendor_id", types::integer());
t.add_column("vendor_gold", types::integer());
t.add_column("interior_refs", types::custom("jsonb"));
t.add_column("created_at", types::custom("timestamp(3)"));
t.add_column("updated_at", types::custom("timestamp(3)"));
t.add_index(

43
src/filters/mod.rs
View File

@ -3,7 +3,7 @@ use std::convert::Infallible;
use warp::{Filter, Rejection, Reply};
use super::handlers;
use super::models::{ListParams, Owner, Shop};
use super::models::{InteriorRef, ListParams, Owner, Shop};
use super::Environment;
pub fn get_shop(env: Environment) -> impl Filter<Extract = impl Reply, Error = Rejection> + Clone {
@ -61,6 +61,45 @@ pub fn list_owners(
.and_then(handlers::list_owners)
}
pub fn get_interior_ref(
env: Environment,
) -> impl Filter<Extract = impl Reply, Error = Rejection> + Clone {
warp::path!("interior_refs" / i32)
.and(warp::get())
.and(with_env(env))
.and_then(handlers::get_interior_ref)
}
pub fn create_interior_ref(
env: Environment,
) -> impl Filter<Extract = impl Reply, Error = Rejection> + Clone {
warp::path!("interior_refs")
.and(warp::post())
.and(json_body::<InteriorRef>())
.and(with_env(env))
.and_then(handlers::create_interior_ref)
}
pub fn list_interior_refs(
env: Environment,
) -> impl Filter<Extract = impl Reply, Error = Rejection> + Clone {
warp::path!("interior_refs")
.and(warp::get())
.and(warp::query::<ListParams>())
.and(with_env(env))
.and_then(handlers::list_interior_refs)
}
pub fn bulk_create_interior_refs(
env: Environment,
) -> impl Filter<Extract = impl Reply, Error = Rejection> + Clone {
warp::path!("interior_refs" / "bulk")
.and(warp::post())
.and(json_body::<Vec<InteriorRef>>())
.and(with_env(env))
.and_then(handlers::bulk_create_interior_refs)
}
fn with_env(env: Environment) -> impl Filter<Extract = (Environment,), Error = Infallible> + Clone {
warp::any().map(move || env.clone())
}
@ -69,5 +108,5 @@ fn json_body<T>() -> impl Filter<Extract = (T,), Error = warp::Rejection> + Clon
where
T: Send + DeserializeOwned,
{
warp::body::content_length_limit(1024 * 16).and(warp::body::json())
warp::body::content_length_limit(1024 * 64).and(warp::body::json())
}

45
src/handlers/mod.rs
View File

@ -4,7 +4,7 @@ use warp::http::StatusCode;
use warp::reply::{json, with_header, with_status};
use warp::{Rejection, Reply};
use super::models::{ListParams, Model, Owner, Shop};
use super::models::{InteriorRef, ListParams, Model, Owner, Shop};
use super::problem::reject_anyhow;
use super::Environment;
@ -74,3 +74,46 @@ pub async fn create_owner(
let reply = with_status(reply, StatusCode::CREATED);
Ok(reply)
}
pub async fn get_interior_ref(id: i32, env: Environment) -> Result<impl Reply, Rejection> {
let interior_ref = InteriorRef::get(&env.db, id).await.map_err(reject_anyhow)?;
let reply = json(&interior_ref);
let reply = with_status(reply, StatusCode::OK);
Ok(reply)
}
pub async fn list_interior_refs(
list_params: ListParams,
env: Environment,
) -> Result<impl Reply, Rejection> {
let interior_refs = InteriorRef::list(&env.db, list_params)
.await
.map_err(reject_anyhow)?;
let reply = json(&interior_refs);
let reply = with_status(reply, StatusCode::OK);
Ok(reply)
}
pub async fn create_interior_ref(
interior_ref: InteriorRef,
env: Environment,
) -> Result<impl Reply, Rejection> {
let saved_interior_ref = interior_ref.save(&env.db).await.map_err(reject_anyhow)?;
let url = saved_interior_ref
.url(&env.api_url)
.map_err(reject_anyhow)?;
let reply = json(&saved_interior_ref);
let reply = with_header(reply, "Location", url.as_str());
let reply = with_status(reply, StatusCode::CREATED);
Ok(reply)
}
pub async fn bulk_create_interior_refs(
interior_refs: Vec<InteriorRef>,
env: Environment,
) -> Result<impl Reply, Rejection> {
InteriorRef::bulk_save(&env.db, interior_refs)
.await
.map_err(reject_anyhow)?;
Ok(StatusCode::CREATED)
}

16
src/main.rs
View File

@ -70,20 +70,30 @@ async fn main() -> Result<()> {
let api_url = host_url.join("/api/v1")?;
let env = Environment::new(api_url).await?;
let home = warp::path!("api" / "v1").map(|| "Shopkeeper home page");
let base = warp::path("api").and(warp::path("v1"));
let get_shop = filters::get_shop(env.clone());
let create_shop = filters::create_shop(env.clone());
let list_shops = filters::list_shops(env.clone());
let get_owner = filters::get_owner(env.clone());
let create_owner = filters::create_owner(env.clone());
let list_owners = filters::list_owners(env.clone());
let routes = create_shop
let get_interior_ref = filters::get_interior_ref(env.clone());
let create_interior_ref = filters::create_interior_ref(env.clone());
let list_interior_refs = filters::list_interior_refs(env.clone());
let bulk_create_interior_refs = filters::bulk_create_interior_refs(env.clone());
let routes = base
.and(
create_shop
.or(get_shop)
.or(list_shops)
.or(create_owner)
.or(get_owner)
.or(list_owners)
.or(home)
.or(create_interior_ref)
.or(get_interior_ref)
.or(list_interior_refs)
.or(bulk_create_interior_refs),
)
.recover(problem::unpack_problem)
.with(warp::compression::gzip())
.with(warp::log("shopkeeper"));

151
src/models/interior_ref.rs Normal file
View File

@ -0,0 +1,151 @@
use anyhow::Result;
use async_trait::async_trait;
use chrono::prelude::*;
use serde::{Deserialize, Serialize};
use serde_json;
use sqlx::postgres::PgPool;
use super::ListParams;
use super::Model;
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct InteriorRef {
pub id: Option<i32>,
pub shop_id: i32,
pub mod_name: String,
pub local_form_id: i32,
pub position_x: f64,
pub position_y: f64,
pub position_z: f64,
pub angle_x: f64,
pub angle_y: f64,
pub angle_z: f64,
pub scale: f64,
pub created_at: Option<NaiveDateTime>,
}
#[async_trait]
impl Model for InteriorRef {
fn resource_name() -> &'static str {
"interior_ref"
}
fn pk(&self) -> Option<i32> {
self.id
}
async fn get(db: &PgPool, id: i32) -> Result<Self> {
let timer = std::time::Instant::now();
let result = sqlx::query_as!(Self, "SELECT * FROM interior_refs WHERE id = $1", id)
.fetch_one(db)
.await?;
let elapsed = timer.elapsed();
debug!("SELECT * FROM interior_refs ... {:.3?}", elapsed);
Ok(result)
}
async fn save(self, db: &PgPool) -> Result<Self> {
let timer = std::time::Instant::now();
let result = sqlx::query_as!(
Self,
"INSERT INTO interior_refs
(shop_id, mod_name, local_form_id, position_x, position_y, position_z, angle_x,
angle_y, angle_z, scale, created_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, now())
RETURNING *",
self.shop_id,
self.mod_name,
self.local_form_id,
self.position_x,
self.position_y,
self.position_z,
self.angle_x,
self.angle_y,
self.angle_z,
self.scale,
)
.fetch_one(db)
.await?;
let elapsed = timer.elapsed();
debug!("INSERT INTO interior_refs ... {:.3?}", elapsed);
Ok(result)
}
async fn list(db: &PgPool, list_params: ListParams) -> Result<Vec<Self>> {
let timer = std::time::Instant::now();
let result = if let Some(order_by) = list_params.get_order_by() {
sqlx::query_as!(
Self,
"SELECT * FROM interior_refs
ORDER BY $1
LIMIT $2
OFFSET $3",
order_by,
list_params.limit.unwrap_or(10),
list_params.offset.unwrap_or(0),
)
.fetch_all(db)
.await?
} else {
sqlx::query_as!(
Self,
"SELECT * FROM interior_refs
LIMIT $1
OFFSET $2",
list_params.limit.unwrap_or(10),
list_params.offset.unwrap_or(0),
)
.fetch_all(db)
.await?
};
let elapsed = timer.elapsed();
debug!("SELECT * FROM interior_refs ... {:.3?}", elapsed);
Ok(result)
}
// TODO: figure out a way bulk insert in a single query
// see: https://github.com/launchbadge/sqlx/issues/294
async fn bulk_save(db: &PgPool, interior_refs: Vec<Self>) -> Result<()> {
let timer = std::time::Instant::now();
// Testing whether setting a jsonb column with an array of 200 items is faster than
// inserting 200 rows. Answer: it is a hell of a lot faster!
// TODO:
// 1. remove interior_refs column from shops
// 2. replace all columns in interior_refs table with single `refs` jsonb column and
// shops_id foreign_key
// 3. This function will now create the row in that table
// 4. Decide if I'll need to make the same changes to merchandise and transactions
// - answer depends on how many rows of each I expect to insert in one go
sqlx::query!(
"UPDATE shops SET interior_refs = $1::jsonb",
serde_json::to_value(&interior_refs)?,
)
.execute(db)
.await?;
// let mut transaction = db.begin().await?;
// for interior_ref in interior_refs {
// sqlx::query!(
// "INSERT INTO interior_refs
// (shop_id, mod_name, local_form_id, position_x, position_y, position_z, angle_x,
// angle_y, angle_z, scale, created_at)
// VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, now())",
// interior_ref.shop_id,
// interior_ref.mod_name,
// interior_ref.local_form_id,
// interior_ref.position_x,
// interior_ref.position_y,
// interior_ref.position_z,
// interior_ref.angle_x,
// interior_ref.angle_y,
// interior_ref.angle_z,
// interior_ref.scale,
// )
// .execute(&mut transaction)
// .await?;
// }
// transaction.commit().await?;
let elapsed = timer.elapsed();
debug!("INSERT INTO interior_refs ... {:.3?}", elapsed);
Ok(())
}
}

2
src/models/mod.rs
View File

@ -4,10 +4,12 @@ use std::fmt;
pub mod model;
pub mod owner;
pub mod shop;
pub mod interior_ref;
pub use model::Model;
pub use owner::Owner;
pub use shop::Shop;
pub use interior_ref::InteriorRef;
#[derive(Debug, Deserialize)]
pub enum Order {

3
src/models/model.rs
View File

@ -25,4 +25,7 @@ where
async fn get(db: &PgPool, id: i32) -> Result<Self>;
async fn save(self, db: &PgPool) -> Result<Self>;
async fn list(db: &PgPool, list_params: ListParams) -> Result<Vec<Self>>;
async fn bulk_save(_db: &PgPool, _models: Vec<Self>) -> Result<()> {
unimplemented!()
}
}

7
src/models/shop.rs
View File

@ -2,6 +2,7 @@ use anyhow::Result;
use async_trait::async_trait;
use chrono::prelude::*;
use serde::{Deserialize, Serialize};
use serde_json;
use sqlx::postgres::PgPool;
use super::ListParams;
@ -17,6 +18,7 @@ pub struct Shop {
pub sell_buy_list_id: i32,
pub vendor_id: i32,
pub vendor_gold: i32,
pub interior_refs: serde_json::value::Value,
pub created_at: Option<NaiveDateTime>,
pub updated_at: Option<NaiveDateTime>,
}
@ -47,8 +49,8 @@ impl Model for Shop {
Self,
"INSERT INTO shops
(name, owner_id, description, is_not_sell_buy, sell_buy_list_id, vendor_id,
vendor_gold, created_at, updated_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, now(), now())
vendor_gold, interior_refs, created_at, updated_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now(), now())
RETURNING *",
self.name,
self.owner_id,
@ -57,6 +59,7 @@ impl Model for Shop {
self.sell_buy_list_id,
self.vendor_id,
self.vendor_gold,
self.interior_refs,
)
.fetch_one(db)
.await?;