thallada/BazaarRealmClient
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Defer writing to file cache to separate thread

Browse Source 
This allows the plugin to get the data faster while the cache update continues in the background.
This commit is contained in:
Tyler Hallada 2020-11-07 00:34:09 -05:00
parent 2b82d09d17
commit 52b1a64d7e
6 changed files with 78 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
src/cache.rs
View File

@ -1,18 +1,20 @@
use std::{fs::create_dir_all, fs::File, io::BufReader, io::Write, path::Path, path::PathBuf}; use std::{
fs::create_dir_all, fs::File, io::BufReader, io::Write, path::Path, path::PathBuf, thread,
};
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use base64::{encode_config, URL_SAFE_NO_PAD}; use base64::{encode_config, URL_SAFE_NO_PAD};
use bytes::Bytes; use bytes::Bytes;
use chrono::{DateTime, Utc}; use chrono::{DateTime, Utc};
use reqwest::{blocking::Response, header::HeaderMap}; use reqwest::header::HeaderMap;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
#[cfg(test)] #[cfg(test)]
use tempfile::tempfile; use tempfile::tempfile;
#[cfg(not(test))] #[cfg(not(test))]
use log::info; use log::{error, info};
#[cfg(test)] #[cfg(test)]
use std::println as info; use std::{println as error, println as info};
use super::API_VERSION; use super::API_VERSION;
@ -60,14 +62,23 @@ pub fn update_metadata_file_cache(cache_path: &Path, headers: &HeaderMap) -> Res
} }
pub fn update_file_caches( pub fn update_file_caches(
body_cache_path: &Path, body_cache_path: PathBuf,
metadata_cache_path: &Path, metadata_cache_path: PathBuf,
response: Response, bytes: Bytes,
) -> Result<Bytes> { headers: HeaderMap,
update_metadata_file_cache(metadata_cache_path, &response.headers())?; ) {
let bytes = response.bytes()?; thread::spawn(move || {
update_file_cache(body_cache_path, &bytes)?; update_file_cache(&body_cache_path, &bytes)
Ok(bytes) .map_err(|err| {
error!("Failed to update body file cache: {}", err);
})
.ok();
update_metadata_file_cache(&metadata_cache_path, &headers)
.map_err(|err| {
error!("Failed to update metadata file cache: {}", err);
})
.ok();
});
} }
pub fn from_file_cache<T: for<'de> Deserialize<'de>>(cache_path: &Path) -> Result<T> { pub fn from_file_cache<T: for<'de> Deserialize<'de>>(cache_path: &Path) -> Result<T> {

27
src/interior_ref_list.rs
View File

@ -11,8 +11,7 @@ use std::{println as info, println as error};
use crate::{ use crate::{
cache::file_cache_dir, cache::from_file_cache, cache::load_metadata_from_file_cache, cache::file_cache_dir, cache::from_file_cache, cache::load_metadata_from_file_cache,
cache::update_file_cache, cache::update_file_caches, cache::update_metadata_file_cache, cache::update_file_caches, log_server_error, result::FFIResult,
log_server_error, result::FFIResult,
}; };
#[derive(Serialize, Deserialize, Debug)] #[derive(Serialize, Deserialize, Debug)]
@ -141,14 +140,10 @@ pub extern "C" fn create_interior_ref_list(
let bytes = resp.bytes()?; let bytes = resp.bytes()?;
let json: InteriorRefList = serde_json::from_slice(&bytes)?; let json: InteriorRefList = serde_json::from_slice(&bytes)?;
if let Some(id) = json.id { if let Some(id) = json.id {
update_file_cache( let body_cache_path = cache_dir.join(format!("interior_ref_list_{}.json", id));
&cache_dir.join(format!("interior_ref_list_{}.json", id)), let metadata_cache_path =
&bytes, cache_dir.join(format!("interior_ref_list_{}_metadata.json", id));
)?; update_file_caches(body_cache_path, metadata_cache_path, bytes, headers);
update_metadata_file_cache(
&cache_dir.join(format!("interior_ref_list_{}_metadata.json", id)),
&headers,
)?;
} }
Ok(json) Ok(json)
} }
@ -223,8 +218,10 @@ pub extern "C" fn update_interior_ref_list(
let body_cache_path = cache_dir.join(format!("shop_{}_interior_ref_list.json", shop_id)); let body_cache_path = cache_dir.join(format!("shop_{}_interior_ref_list.json", shop_id));
let metadata_cache_path = let metadata_cache_path =
cache_dir.join(format!("shop_{}_interior_ref_list_metadata.json", shop_id)); cache_dir.join(format!("shop_{}_interior_ref_list_metadata.json", shop_id));
let bytes = update_file_caches(&body_cache_path, &metadata_cache_path, resp)?; let headers = resp.headers().clone();
let bytes = resp.bytes()?;
let json: InteriorRefList = serde_json::from_slice(&bytes)?; let json: InteriorRefList = serde_json::from_slice(&bytes)?;
update_file_caches(body_cache_path, metadata_cache_path, bytes, headers);
Ok(json) Ok(json)
} }
@ -296,8 +293,10 @@ pub extern "C" fn get_interior_ref_list(
Ok(resp) => { Ok(resp) => {
info!("get_interior_ref_list response from api: {:?}", &resp); info!("get_interior_ref_list response from api: {:?}", &resp);
if resp.status().is_success() { if resp.status().is_success() {
let bytes = update_file_caches(&body_cache_path, &metadata_cache_path, resp)?; let headers = resp.headers().clone();
let bytes = resp.bytes()?;
let json = serde_json::from_slice(&bytes)?; let json = serde_json::from_slice(&bytes)?;
update_file_caches(body_cache_path, metadata_cache_path, bytes, headers);
Ok(json) Ok(json)
} else if resp.status() == StatusCode::NOT_MODIFIED { } else if resp.status() == StatusCode::NOT_MODIFIED {
from_file_cache(&body_cache_path) from_file_cache(&body_cache_path)
@ -396,8 +395,10 @@ pub extern "C" fn get_interior_ref_list_by_shop_id(
&resp &resp
); );
if resp.status().is_success() { if resp.status().is_success() {
let bytes = update_file_caches(&body_cache_path, &metadata_cache_path, resp)?; let headers = resp.headers().clone();
let bytes = resp.bytes()?;
let json = serde_json::from_slice(&bytes)?; let json = serde_json::from_slice(&bytes)?;
update_file_caches(body_cache_path, metadata_cache_path, bytes, headers);
Ok(json) Ok(json)
} else if resp.status() == StatusCode::NOT_MODIFIED { } else if resp.status() == StatusCode::NOT_MODIFIED {
from_file_cache(&body_cache_path) from_file_cache(&body_cache_path)

27
src/merchandise_list.rs
View File

@ -11,8 +11,7 @@ use std::{println as info, println as error};
use crate::{ use crate::{
cache::file_cache_dir, cache::from_file_cache, cache::load_metadata_from_file_cache, cache::file_cache_dir, cache::from_file_cache, cache::load_metadata_from_file_cache,
cache::update_file_cache, cache::update_file_caches, cache::update_metadata_file_cache, cache::update_file_caches, log_server_error, result::FFIResult,
log_server_error, result::FFIResult,
}; };
#[derive(Serialize, Deserialize, Debug, Clone)] #[derive(Serialize, Deserialize, Debug, Clone)]
@ -124,14 +123,10 @@ pub extern "C" fn create_merchandise_list(
let bytes = resp.bytes()?; let bytes = resp.bytes()?;
let json: MerchandiseList = serde_json::from_slice(&bytes)?; let json: MerchandiseList = serde_json::from_slice(&bytes)?;
if let Some(id) = json.id { if let Some(id) = json.id {
update_file_cache( let body_cache_path = cache_dir.join(format!("merchandise_list_{}.json", id));
&cache_dir.join(format!("merchandise_list_{}.json", id)), let metadata_cache_path =
&bytes, cache_dir.join(format!("merchandise_list_{}_metadata.json", id));
)?; update_file_caches(body_cache_path, metadata_cache_path, bytes, headers);
update_metadata_file_cache(
&cache_dir.join(format!("merchandise_list_{}_metadata.json", id)),
&headers,
)?;
} }
Ok(json) Ok(json)
} }
@ -208,8 +203,10 @@ pub extern "C" fn update_merchandise_list(
let body_cache_path = cache_dir.join(format!("shop_{}_merchandise_list.json", shop_id)); let body_cache_path = cache_dir.join(format!("shop_{}_merchandise_list.json", shop_id));
let metadata_cache_path = let metadata_cache_path =
cache_dir.join(format!("shop_{}_merchandise_list_metadata.json", shop_id)); cache_dir.join(format!("shop_{}_merchandise_list_metadata.json", shop_id));
let bytes = update_file_caches(&body_cache_path, &metadata_cache_path, resp)?; let headers = resp.headers().clone();
let bytes = resp.bytes()?;
let json: MerchandiseList = serde_json::from_slice(&bytes)?; let json: MerchandiseList = serde_json::from_slice(&bytes)?;
update_file_caches(body_cache_path, metadata_cache_path, bytes, headers);
Ok(json) Ok(json)
} }
@ -284,8 +281,10 @@ pub extern "C" fn get_merchandise_list(
Ok(resp) => { Ok(resp) => {
info!("get_merchandise_list response from api: {:?}", &resp); info!("get_merchandise_list response from api: {:?}", &resp);
if resp.status().is_success() { if resp.status().is_success() {
let bytes = update_file_caches(&body_cache_path, &metadata_cache_path, resp)?; let headers = resp.headers().clone();
let bytes = resp.bytes()?;
let json = serde_json::from_slice(&bytes)?; let json = serde_json::from_slice(&bytes)?;
update_file_caches(body_cache_path, metadata_cache_path, bytes, headers);
Ok(json) Ok(json)
} else if resp.status() == StatusCode::NOT_MODIFIED { } else if resp.status() == StatusCode::NOT_MODIFIED {
from_file_cache(&body_cache_path) from_file_cache(&body_cache_path)
@ -377,8 +376,10 @@ pub extern "C" fn get_merchandise_list_by_shop_id(
&resp &resp
); );
if resp.status().is_success() { if resp.status().is_success() {
let bytes = update_file_caches(&body_cache_path, &metadata_cache_path, resp)?; let headers = resp.headers().clone();
let bytes = resp.bytes()?;
let json = serde_json::from_slice(&bytes)?; let json = serde_json::from_slice(&bytes)?;
update_file_caches(body_cache_path, metadata_cache_path, bytes, headers);
Ok(json) Ok(json)
} else if resp.status() == StatusCode::NOT_MODIFIED { } else if resp.status() == StatusCode::NOT_MODIFIED {
from_file_cache(&body_cache_path) from_file_cache(&body_cache_path)

17
src/owner.rs
View File

@ -9,10 +9,7 @@ use log::{error, info};
#[cfg(test)] #[cfg(test)]
use std::{println as info, println as error}; use std::{println as info, println as error};
use crate::{ use crate::{cache::file_cache_dir, cache::update_file_caches, result::FFIResult};
cache::file_cache_dir, cache::update_file_cache, cache::update_file_caches,
cache::update_metadata_file_cache, result::FFIResult,
};
#[derive(Serialize, Deserialize, Debug)] #[derive(Serialize, Deserialize, Debug)]
pub struct Owner { pub struct Owner {
@ -78,11 +75,9 @@ pub extern "C" fn create_owner(
let bytes = resp.bytes()?; let bytes = resp.bytes()?;
let json: Owner = serde_json::from_slice(&bytes)?; let json: Owner = serde_json::from_slice(&bytes)?;
if let Some(id) = json.id { if let Some(id) = json.id {
update_file_cache(&cache_dir.join(format!("owner_{}.json", id)), &bytes)?; let body_cache_path = cache_dir.join(format!("owner_{}.json", id));
update_metadata_file_cache( let metadata_cache_path = cache_dir.join(format!("owner_{}_metadata.json", id));
&cache_dir.join(format!("owner_{}_metadata.json", id)), update_file_caches(body_cache_path, metadata_cache_path, bytes, headers);
&headers,
)?;
} }
Ok(json) Ok(json)
} else { } else {
@ -155,8 +150,10 @@ pub extern "C" fn update_owner(
let cache_dir = file_cache_dir(api_url)?; let cache_dir = file_cache_dir(api_url)?;
let body_cache_path = cache_dir.join(format!("owner_{}.json", id)); let body_cache_path = cache_dir.join(format!("owner_{}.json", id));
let metadata_cache_path = cache_dir.join(format!("owner_{}_metadata.json", id)); let metadata_cache_path = cache_dir.join(format!("owner_{}_metadata.json", id));
let bytes = update_file_caches(&body_cache_path, &metadata_cache_path, resp)?; let headers = resp.headers().clone();
let bytes = resp.bytes()?;
let json: Owner = serde_json::from_slice(&bytes)?; let json: Owner = serde_json::from_slice(&bytes)?;
update_file_caches(body_cache_path, metadata_cache_path, bytes, headers);
Ok(json) Ok(json)
} else { } else {
Err(anyhow!("api-key not defined")) Err(anyhow!("api-key not defined"))

23
src/shop.rs
View File

@ -11,8 +11,7 @@ use std::{println as info, println as error};
use crate::{ use crate::{
cache::file_cache_dir, cache::from_file_cache, cache::load_metadata_from_file_cache, cache::file_cache_dir, cache::from_file_cache, cache::load_metadata_from_file_cache,
cache::update_file_cache, cache::update_file_caches, cache::update_metadata_file_cache, cache::update_file_caches, log_server_error, result::FFIResult,
log_server_error, result::FFIResult,
}; };
#[derive(Serialize, Deserialize, Debug)] #[derive(Serialize, Deserialize, Debug)]
@ -117,11 +116,9 @@ pub extern "C" fn create_shop(
let bytes = resp.bytes()?; let bytes = resp.bytes()?;
let json: Shop = serde_json::from_slice(&bytes)?; let json: Shop = serde_json::from_slice(&bytes)?;
if let Some(id) = json.id { if let Some(id) = json.id {
update_file_cache(&cache_dir.join(format!("shop_{}.json", id)), &bytes)?; let body_cache_path = cache_dir.join(format!("shop_{}.json", id));
update_metadata_file_cache( let metadata_cache_path = cache_dir.join(format!("shop_{}_metadata.json", id));
&cache_dir.join(format!("shop_{}_metadata.json", id)), update_file_caches(body_cache_path, metadata_cache_path, bytes, headers);
&headers,
)?;
} }
Ok(json) Ok(json)
} }
@ -188,8 +185,10 @@ pub extern "C" fn update_shop(
let cache_dir = file_cache_dir(api_url)?; let cache_dir = file_cache_dir(api_url)?;
let body_cache_path = cache_dir.join(format!("shop_{}.json", id)); let body_cache_path = cache_dir.join(format!("shop_{}.json", id));
let metadata_cache_path = cache_dir.join(format!("shop_{}_metadata.json", id)); let metadata_cache_path = cache_dir.join(format!("shop_{}_metadata.json", id));
let bytes = update_file_caches(&body_cache_path, &metadata_cache_path, resp)?; let headers = resp.headers().clone();
let bytes = resp.bytes()?;
let json: Shop = serde_json::from_slice(&bytes)?; let json: Shop = serde_json::from_slice(&bytes)?;
update_file_caches(body_cache_path, metadata_cache_path, bytes, headers);
Ok(json) Ok(json)
} }
@ -255,8 +254,10 @@ pub extern "C" fn get_shop(
Ok(resp) => { Ok(resp) => {
info!("get_shop response from api: {:?}", &resp); info!("get_shop response from api: {:?}", &resp);
if resp.status().is_success() { if resp.status().is_success() {
let bytes = update_file_caches(&body_cache_path, &metadata_cache_path, resp)?; let headers = resp.headers().clone();
let bytes = resp.bytes()?;
let json = serde_json::from_slice(&bytes)?; let json = serde_json::from_slice(&bytes)?;
update_file_caches(body_cache_path, metadata_cache_path, bytes, headers);
Ok(json) Ok(json)
} else if resp.status() == StatusCode::NOT_MODIFIED { } else if resp.status() == StatusCode::NOT_MODIFIED {
from_file_cache(&body_cache_path) from_file_cache(&body_cache_path)
@ -329,8 +330,10 @@ pub extern "C" fn list_shops(
Ok(resp) => { Ok(resp) => {
info!("list_shops response from api: {:?}", &resp); info!("list_shops response from api: {:?}", &resp);
if resp.status().is_success() { if resp.status().is_success() {
let bytes = update_file_caches(&body_cache_path, &metadata_cache_path, resp)?; let headers = resp.headers().clone();
let bytes = resp.bytes()?;
let json = serde_json::from_slice(&bytes)?; let json = serde_json::from_slice(&bytes)?;
update_file_caches(body_cache_path, metadata_cache_path, bytes, headers);
Ok(json) Ok(json)
} else if resp.status() == StatusCode::NOT_MODIFIED { } else if resp.status() == StatusCode::NOT_MODIFIED {
from_file_cache(&body_cache_path) from_file_cache(&body_cache_path)

15
src/transaction.rs
View File

@ -1,4 +1,4 @@
use std::{convert::TryFrom, ffi::CStr, ffi::CString, os::raw::c_char, slice, str}; use std::{convert::TryFrom, ffi::CStr, ffi::CString, os::raw::c_char, slice, str, thread};
use anyhow::{anyhow, Result}; use anyhow::{anyhow, Result};
use http_api_problem::HttpApiProblem; use http_api_problem::HttpApiProblem;
@ -11,8 +11,8 @@ use log::{error, info};
use std::{println as info, println as error}; use std::{println as info, println as error};
use crate::{ use crate::{
cache::file_cache_dir, cache::from_file_cache, cache::update_file_cache, cache::file_cache_dir, cache::from_file_cache, cache::update_file_caches, log_server_error,
cache::update_metadata_file_cache, log_server_error, result::FFIResult, result::FFIResult,
}; };
#[derive(Serialize, Deserialize, Debug)] #[derive(Serialize, Deserialize, Debug)]
@ -165,11 +165,10 @@ pub extern "C" fn create_transaction(
if status.is_success() { if status.is_success() {
let json: Transaction = serde_json::from_slice(&bytes)?; let json: Transaction = serde_json::from_slice(&bytes)?;
if let Some(id) = json.id { if let Some(id) = json.id {
update_file_cache(&cache_dir.join(format!("transaction_{}.json", id)), &bytes)?; let body_cache_path = cache_dir.join(format!("transaction_{}.json", id));
update_metadata_file_cache( let metadata_cache_path =
&cache_dir.join(format!("transaction_{}_metadata.json", id)), cache_dir.join(format!("transaction_{}_metadata.json", id));
&headers, update_file_caches(body_cache_path, metadata_cache_path, bytes, headers);
)?;
} }
Ok(json) Ok(json)
} else { } else {