Add ETag headers to get/list endpoints

Now the client can opt out of receiving the whole JSON body if it hasn't changed since they last requested.

Right now, only `ETag` and `If-None-Match` headers are implemeted which isn't very RFC-spec compliant but it's all I need so I don't care.

Cargo.lock

@@ -144,6 +144,7 @@ dependencies = [
  "ipnetwork",
  "listenfd",
  "lru",
+ "seahash",
  "serde",
  "serde_json",
  "sqlx",
@@ -1368,6 +1369,12 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2"
 
+[[package]]
+name = "seahash"
+version = "4.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39ee459cae272d224928ca09a1df5406da984f263dc544f9f8bde92a8c3dc916"
+
 [[package]]
 name = "security-framework"
 version = "0.4.4"

Cargo.toml

@@ -22,6 +22,7 @@ uuid = { version = "0.8", features = ["serde", "v4"] }
 ipnetwork = "0.16"
 url = "2.1"
 async-trait = "0.1"
+seahash = "4.0"
 tracing = "0.1"
 tracing-subscriber = "0.2"
 tracing-futures = "0.2"

src/caches/cache.rs

@@ -106,21 +106,23 @@ where
 
         self.log_with_key(&key, "get_response: miss");
         let reply = getter().await.map_err(reject_anyhow);
-        let cached_response = match reply {
+        Ok(match reply {
-            Ok(reply) => CachedResponse::from_reply(reply)
+            Ok(reply) => {
-                .await
+                let cached_response = CachedResponse::from_reply(reply)
-                .map_err(reject_anyhow)?,
+                    .await
+                    .map_err(reject_anyhow)?;
+                let mut guard = self.lru_mutex.lock().await;
+                guard.put(key, cached_response.clone());
+                cached_response
+            }
             Err(rejection) => {
+                self.log_with_key(&key, "get_response: getter returned rejection, not caching");
                 let reply = unpack_problem(rejection).await?;
                 CachedResponse::from_reply(reply)
                     .await
                     .map_err(reject_anyhow)?
             }
-        };
+        })
-        let mut guard = self.lru_mutex.lock().await;
-        guard.put(key, cached_response.clone());
-
-        Ok(cached_response)
     }
 
     pub async fn delete_response(&self, key: K) -> Option<CachedResponse> {

src/caches/cached_response.rs

@@ -1,4 +1,5 @@
 use anyhow::Result;
+use http::header::ETAG;
 use http::{HeaderMap, HeaderValue, Response, StatusCode, Version};
 use hyper::body::{to_bytes, Body, Bytes};
 use warp::Reply;
@@ -24,6 +25,17 @@ impl CachedResponse {
             body: to_bytes(response.body_mut()).await?,
         })
     }
+
+    pub fn not_modified(etag: HeaderValue) -> Self {
+        let mut headers = HeaderMap::new();
+        headers.insert(ETAG, etag);
+        Self {
+            status: StatusCode::NOT_MODIFIED,
+            version: Version::HTTP_11,
+            headers,
+            body: Bytes::new(),
+        }
+    }
 }
 
 impl Reply for CachedResponse {

src/handlers/interior_ref_list.rs

@@ -8,42 +8,56 @@ use crate::models::{InteriorRefList, ListParams};
 use crate::problem::reject_anyhow;
 use crate::Environment;
 
-use super::authenticate;
+use super::{authenticate, check_etag, JsonWithETag};
 
-pub async fn get(id: i32, env: Environment) -> Result<impl Reply, Rejection> {
+pub async fn get(id: i32, etag: Option<String>, env: Environment) -> Result<impl Reply, Rejection> {
-    env.caches
+    let response = env
+        .caches
         .interior_ref_list
         .get_response(id, || async {
             let interior_ref_list = InteriorRefList::get(&env.db, id).await?;
-            let reply = json(&interior_ref_list);
+            let reply = JsonWithETag::from_serializable(&interior_ref_list)?;
             let reply = with_status(reply, StatusCode::OK);
             Ok(reply)
         })
-        .await
+        .await?;
+    Ok(check_etag(etag, response))
 }
 
-pub async fn get_by_shop_id(shop_id: i32, env: Environment) -> Result<impl Reply, Rejection> {
+pub async fn get_by_shop_id(
-    env.caches
+    shop_id: i32,
+    etag: Option<String>,
+    env: Environment,
+) -> Result<impl Reply, Rejection> {
+    let response = env
+        .caches
         .interior_ref_list_by_shop_id
         .get_response(shop_id, || async {
             let interior_ref_list = InteriorRefList::get_by_shop_id(&env.db, shop_id).await?;
-            let reply = json(&interior_ref_list);
+            let reply = JsonWithETag::from_serializable(&interior_ref_list)?;
             let reply = with_status(reply, StatusCode::OK);
             Ok(reply)
         })
-        .await
+        .await?;
+    Ok(check_etag(etag, response))
 }
 
-pub async fn list(list_params: ListParams, env: Environment) -> Result<impl Reply, Rejection> {
+pub async fn list(
-    env.caches
+    list_params: ListParams,
+    etag: Option<String>,
+    env: Environment,
+) -> Result<impl Reply, Rejection> {
+    let response = env
+        .caches
         .list_interior_ref_lists
         .get_response(list_params.clone(), || async {
             let interior_ref_lists = InteriorRefList::list(&env.db, &list_params).await?;
-            let reply = json(&interior_ref_lists);
+            let reply = JsonWithETag::from_serializable(&interior_ref_lists)?;
             let reply = with_status(reply, StatusCode::OK);
             Ok(reply)
         })
-        .await
+        .await?;
+    Ok(check_etag(etag, response))
 }
 
 pub async fn create(

src/handlers/merchandise_list.rs

@@ -8,42 +8,56 @@ use crate::models::{ListParams, MerchandiseList};
 use crate::problem::reject_anyhow;
 use crate::Environment;
 
-use super::authenticate;
+use super::{authenticate, check_etag, JsonWithETag};
 
-pub async fn get(id: i32, env: Environment) -> Result<impl Reply, Rejection> {
+pub async fn get(id: i32, etag: Option<String>, env: Environment) -> Result<impl Reply, Rejection> {
-    env.caches
+    let response = env
+        .caches
         .merchandise_list
         .get_response(id, || async {
             let merchandise_list = MerchandiseList::get(&env.db, id).await?;
-            let reply = json(&merchandise_list);
+            let reply = JsonWithETag::from_serializable(&merchandise_list)?;
             let reply = with_status(reply, StatusCode::OK);
             Ok(reply)
         })
-        .await
+        .await?;
+    Ok(check_etag(etag, response))
 }
 
-pub async fn get_by_shop_id(shop_id: i32, env: Environment) -> Result<impl Reply, Rejection> {
+pub async fn get_by_shop_id(
-    env.caches
+    shop_id: i32,
+    etag: Option<String>,
+    env: Environment,
+) -> Result<impl Reply, Rejection> {
+    let response = env
+        .caches
         .merchandise_list_by_shop_id
         .get_response(shop_id, || async {
             let merchandise_list = MerchandiseList::get_by_shop_id(&env.db, shop_id).await?;
-            let reply = json(&merchandise_list);
+            let reply = JsonWithETag::from_serializable(&merchandise_list)?;
             let reply = with_status(reply, StatusCode::OK);
             Ok(reply)
         })
-        .await
+        .await?;
+    Ok(check_etag(etag, response))
 }
 
-pub async fn list(list_params: ListParams, env: Environment) -> Result<impl Reply, Rejection> {
+pub async fn list(
-    env.caches
+    list_params: ListParams,
+    etag: Option<String>,
+    env: Environment,
+) -> Result<impl Reply, Rejection> {
+    let response = env
+        .caches
         .list_merchandise_lists
         .get_response(list_params.clone(), || async {
             let merchandise_lists = MerchandiseList::list(&env.db, &list_params).await?;
-            let reply = json(&merchandise_lists);
+            let reply = JsonWithETag::from_serializable(&merchandise_lists)?;
             let reply = with_status(reply, StatusCode::OK);
             Ok(reply)
         })
-        .await
+        .await?;
+    Ok(check_etag(etag, response))
 }
 
 pub async fn create(

src/handlers/mod.rs

@@ -1,6 +1,13 @@
 use anyhow::{anyhow, Result};
-use tracing::instrument;
+use http::header::{HeaderValue, CONTENT_TYPE, ETAG};
+use http::StatusCode;
+use http_api_problem::HttpApiProblem;
+use seahash::hash;
+use serde::Serialize;
+use tracing::{error, instrument, warn};
 use uuid::Uuid;
+use warp::reply::Response;
+use warp::Reply;
 
 pub mod interior_ref_list;
 pub mod merchandise_list;
@@ -8,6 +15,7 @@ pub mod owner;
 pub mod shop;
 pub mod transaction;
 
+use super::caches::CachedResponse;
 use super::problem::{unauthorized_no_api_key, unauthorized_no_owner};
 use super::Environment;
 
@@ -35,3 +43,54 @@ pub async fn authenticate(env: &Environment, api_key: Option<Uuid>) -> Result<i3
         Err(unauthorized_no_api_key())
     }
 }
+
+// Similar to `warp::reply::Json`, but stores hash of body content for the ETag header created in `into_response`.
+// Also, it does not store a serialize `Result`. Instead it returns the error to the caller immediately in `from_serializable`.
+// It's purpose is to avoid serializing the body content twice and to encapsulate ETag logic in one place.
+pub struct JsonWithETag {
+    body: Vec<u8>,
+    etag: String,
+}
+
+impl Reply for JsonWithETag {
+    fn into_response(self) -> Response {
+        let mut res = Response::new(self.body.into());
+        res.headers_mut()
+            .insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
+        if let Ok(val) = HeaderValue::from_str(&self.etag) {
+            res.headers_mut().insert(ETAG, val);
+        } else {
+            // This should never happen in practice since etag values should only be hex-encoded strings
+            warn!("omitting etag header with invalid ASCII characters")
+        }
+        res
+    }
+}
+
+impl JsonWithETag {
+    pub fn from_serializable<T: Serialize>(val: &T) -> Result<Self> {
+        let bytes = serde_json::to_vec(val).map_err(|err| {
+            error!("Failed to serialize database value to JSON: {}", err);
+            anyhow!(HttpApiProblem::with_title_and_type_from_status(
+                StatusCode::INTERNAL_SERVER_ERROR
+            )
+            .set_detail(format!(
+                "Failed to serialize database value to JSON: {}",
+                err
+            )))
+        })?;
+        let etag = format!("{:x}", hash(&bytes));
+        Ok(Self { body: bytes, etag })
+    }
+}
+
+pub fn check_etag(etag: Option<String>, response: CachedResponse) -> CachedResponse {
+    if let Some(request_etag) = etag {
+        if let Some(response_etag) = response.headers.get("etag") {
+            if request_etag == *response_etag {
+                return CachedResponse::not_modified(response_etag.clone());
+            }
+        }
+    }
+    response
+}

src/handlers/owner.rs

@@ -10,30 +10,38 @@ use crate::models::{ListParams, Owner};
 use crate::problem::{reject_anyhow, unauthorized_no_api_key};
 use crate::Environment;
 
-use super::authenticate;
+use super::{authenticate, check_etag, JsonWithETag};
 
-pub async fn get(id: i32, env: Environment) -> Result<impl Reply, Rejection> {
+pub async fn get(id: i32, etag: Option<String>, env: Environment) -> Result<impl Reply, Rejection> {
-    env.caches
+    let response = env
+        .caches
         .owner
         .get_response(id, || async {
             let owner = Owner::get(&env.db, id).await?;
-            let reply = json(&owner);
+            let reply = JsonWithETag::from_serializable(&owner)?;
             let reply = with_status(reply, StatusCode::OK);
             Ok(reply)
         })
-        .await
+        .await?;
+    Ok(check_etag(etag, response))
 }
 
-pub async fn list(list_params: ListParams, env: Environment) -> Result<impl Reply, Rejection> {
+pub async fn list(
-    env.caches
+    list_params: ListParams,
+    etag: Option<String>,
+    env: Environment,
+) -> Result<impl Reply, Rejection> {
+    let response = env
+        .caches
         .list_owners
         .get_response(list_params.clone(), || async {
             let owners = Owner::list(&env.db, &list_params).await?;
-            let reply = json(&owners);
+            let reply = JsonWithETag::from_serializable(&owners)?;
             let reply = with_status(reply, StatusCode::OK);
             Ok(reply)
         })
-        .await
+        .await?;
+    Ok(check_etag(etag, response))
 }
 
 pub async fn create(

src/handlers/shop.rs

@@ -9,30 +9,38 @@ use crate::models::{InteriorRefList, ListParams, MerchandiseList, Shop};
 use crate::problem::reject_anyhow;
 use crate::Environment;
 
-use super::authenticate;
+use super::{authenticate, check_etag, JsonWithETag};
 
-pub async fn get(id: i32, env: Environment) -> Result<impl Reply, Rejection> {
+pub async fn get(id: i32, etag: Option<String>, env: Environment) -> Result<impl Reply, Rejection> {
-    env.caches
+    let response = env
+        .caches
         .shop
         .get_response(id, || async {
             let shop = Shop::get(&env.db, id).await?;
-            let reply = json(&shop);
+            let reply = JsonWithETag::from_serializable(&shop)?;
             let reply = with_status(reply, StatusCode::OK);
             Ok(reply)
         })
-        .await
+        .await?;
+    Ok(check_etag(etag, response))
 }
 
-pub async fn list(list_params: ListParams, env: Environment) -> Result<impl Reply, Rejection> {
+pub async fn list(
-    env.caches
+    list_params: ListParams,
+    etag: Option<String>,
+    env: Environment,
+) -> Result<impl Reply, Rejection> {
+    let response = env
+        .caches
         .list_shops
         .get_response(list_params.clone(), || async {
             let shops = Shop::list(&env.db, &list_params).await?;
-            let reply = json(&shops);
+            let reply = JsonWithETag::from_serializable(&shops)?;
             let reply = with_status(reply, StatusCode::OK);
             Ok(reply)
         })
-        .await
+        .await?;
+    Ok(check_etag(etag, response))
 }
 
 pub async fn create(

src/handlers/transaction.rs

@@ -8,46 +8,57 @@ use crate::models::{ListParams, MerchandiseList, Transaction};
 use crate::problem::reject_anyhow;
 use crate::Environment;
 
-use super::authenticate;
+use super::{authenticate, check_etag, JsonWithETag};
 
-pub async fn get(id: i32, env: Environment) -> Result<impl Reply, Rejection> {
+pub async fn get(id: i32, etag: Option<String>, env: Environment) -> Result<impl Reply, Rejection> {
-    env.caches
+    let response = env
+        .caches
         .transaction
         .get_response(id, || async {
             let transaction = Transaction::get(&env.db, id).await?;
-            let reply = json(&transaction);
+            let reply = JsonWithETag::from_serializable(&transaction)?;
             let reply = with_status(reply, StatusCode::OK);
             Ok(reply)
         })
-        .await
+        .await?;
+    Ok(check_etag(etag, response))
 }
 
-pub async fn list(list_params: ListParams, env: Environment) -> Result<impl Reply, Rejection> {
+pub async fn list(
-    env.caches
+    list_params: ListParams,
+    etag: Option<String>,
+    env: Environment,
+) -> Result<impl Reply, Rejection> {
+    let response = env
+        .caches
         .list_transactions
         .get_response(list_params.clone(), || async {
             let transactions = Transaction::list(&env.db, &list_params).await?;
-            let reply = json(&transactions);
+            let reply = JsonWithETag::from_serializable(&transactions)?;
             let reply = with_status(reply, StatusCode::OK);
             Ok(reply)
         })
-        .await
+        .await?;
+    Ok(check_etag(etag, response))
 }
 
 pub async fn list_by_shop_id(
     shop_id: i32,
     list_params: ListParams,
+    etag: Option<String>,
     env: Environment,
 ) -> Result<impl Reply, Rejection> {
-    env.caches
+    let response = env
+        .caches
         .list_transactions_by_shop_id
         .get_response((shop_id, list_params.clone()), || async {
             let transactions = Transaction::list_by_shop_id(&env.db, shop_id, &list_params).await?;
-            let reply = json(&transactions);
+            let reply = JsonWithETag::from_serializable(&transactions)?;
             let reply = with_status(reply, StatusCode::OK);
             Ok(reply)
         })
-        .await
+        .await?;
+    Ok(check_etag(etag, response))
 }
 
 pub async fn create(

src/main.rs

@@ -81,6 +81,7 @@ async fn main() -> Result<()> {
         warp::path::param()
             .and(warp::path::end())
             .and(warp::get())
+            .and(warp::header::optional("if-none-match"))
             .and(with_env(env.clone()))
             .and_then(handlers::owner::get),
     );
@@ -115,6 +116,7 @@ async fn main() -> Result<()> {
         warp::path::end()
             .and(warp::get())
             .and(warp::query::<ListParams>())
+            .and(warp::header::optional("if-none-match"))
             .and(with_env(env.clone()))
             .and_then(handlers::owner::list),
     );
@@ -122,6 +124,7 @@ async fn main() -> Result<()> {
         warp::path::param()
             .and(warp::path::end())
             .and(warp::get())
+            .and(warp::header::optional("if-none-match"))
             .and(with_env(env.clone()))
             .and_then(handlers::shop::get),
     );
@@ -154,6 +157,7 @@ async fn main() -> Result<()> {
         warp::path::end()
             .and(warp::get())
             .and(warp::query::<ListParams>())
+            .and(warp::header::optional("if-none-match"))
             .and(with_env(env.clone()))
             .and_then(handlers::shop::list),
     );
@@ -161,6 +165,7 @@ async fn main() -> Result<()> {
         warp::path::param()
             .and(warp::path::end())
             .and(warp::get())
+            .and(warp::header::optional("if-none-match"))
             .and(with_env(env.clone()))
             .and_then(handlers::interior_ref_list::get),
     );
@@ -203,6 +208,7 @@ async fn main() -> Result<()> {
         warp::path::end()
             .and(warp::get())
             .and(warp::query::<ListParams>())
+            .and(warp::header::optional("if-none-match"))
             .and(with_env(env.clone()))
             .and_then(handlers::interior_ref_list::list),
     );
@@ -211,6 +217,7 @@ async fn main() -> Result<()> {
             .and(warp::path("interior_ref_list"))
             .and(warp::path::end())
             .and(warp::get())
+            .and(warp::header::optional("if-none-match"))
             .and(with_env(env.clone()))
             .and_then(handlers::interior_ref_list::get_by_shop_id),
     );
@@ -218,6 +225,7 @@ async fn main() -> Result<()> {
         warp::path::param()
             .and(warp::path::end())
             .and(warp::get())
+            .and(warp::header::optional("if-none-match"))
             .and(with_env(env.clone()))
             .and_then(handlers::merchandise_list::get),
     );
@@ -260,6 +268,7 @@ async fn main() -> Result<()> {
         warp::path::end()
             .and(warp::get())
             .and(warp::query::<ListParams>())
+            .and(warp::header::optional("if-none-match"))
             .and(with_env(env.clone()))
             .and_then(handlers::merchandise_list::list),
     );
@@ -268,6 +277,7 @@ async fn main() -> Result<()> {
             .and(warp::path("merchandise_list"))
             .and(warp::path::end())
             .and(warp::get())
+            .and(warp::header::optional("if-none-match"))
             .and(with_env(env.clone()))
             .and_then(handlers::merchandise_list::get_by_shop_id),
     );
@@ -275,6 +285,7 @@ async fn main() -> Result<()> {
         warp::path::param()
             .and(warp::path::end())
             .and(warp::get())
+            .and(warp::header::optional("if-none-match"))
             .and(with_env(env.clone()))
             .and_then(handlers::transaction::get),
     );
@@ -298,6 +309,7 @@ async fn main() -> Result<()> {
         warp::path::end()
             .and(warp::get())
             .and(warp::query::<ListParams>())
+            .and(warp::header::optional("if-none-match"))
             .and(with_env(env.clone()))
             .and_then(handlers::transaction::list),
     );
@@ -307,6 +319,7 @@ async fn main() -> Result<()> {
             .and(warp::path::end())
             .and(warp::get())
             .and(warp::query::<ListParams>())
+            .and(warp::header::optional("if-none-match"))
             .and(with_env(env.clone()))
             .and_then(handlers::transaction::list_by_shop_id),
     );