Increase max size of downloaded article images
- Download cap raised from 5 MiB to 100 MiB in src/images/fetch.rs:14. - Added 16,384×16,384 source dimension limits. - Added an explicit 512 MiB per-image decoder allocation limit in src/images/encode.rs:15. - Existing output resizing and 25 MiB final issue budget remain unchanged. - Added regression coverage for a large image source size and oversized image dimensions. - Improved overflow safety when accumulating streamed image bytes.
This commit is contained in:
+31
-3
@@ -6,12 +6,16 @@
|
|||||||
|
|
||||||
use std::io::Cursor;
|
use std::io::Cursor;
|
||||||
|
|
||||||
use image::{DynamicImage, GenericImageView, ImageFormat};
|
use image::{DynamicImage, GenericImageView, ImageFormat, ImageReader, Limits};
|
||||||
|
|
||||||
use crate::types::Edition;
|
use crate::types::Edition;
|
||||||
|
|
||||||
/// Images smaller than this in either dimension are decorative — skipped (§3.10).
|
/// Images smaller than this in either dimension are decorative — skipped (§3.10).
|
||||||
pub const MIN_DIMENSION_PX: u32 = 24;
|
pub const MIN_DIMENSION_PX: u32 = 24;
|
||||||
|
/// Maximum width or height accepted from a decoded raster source.
|
||||||
|
pub const MAX_DECODED_DIMENSION_PX: u32 = 16_384;
|
||||||
|
/// Maximum memory the image decoder may allocate for one source image.
|
||||||
|
pub const MAX_DECODE_ALLOC_BYTES: u64 = 512 * 1024 * 1024;
|
||||||
/// Width an SVG is rendered at when the profile asks for less than this.
|
/// Width an SVG is rendered at when the profile asks for less than this.
|
||||||
const SVG_FALLBACK_SIZE: u32 = 1000;
|
const SVG_FALLBACK_SIZE: u32 = 1000;
|
||||||
|
|
||||||
@@ -60,8 +64,18 @@ pub fn reencode(bytes: &[u8], profile: ImageProfile) -> Option<(Vec<u8>, &'stati
|
|||||||
let raster = rasterize_svg(bytes, profile)?;
|
let raster = rasterize_svg(bytes, profile)?;
|
||||||
return reencode(&raster, profile);
|
return reencode(&raster, profile);
|
||||||
}
|
}
|
||||||
let format = image::guess_format(bytes).ok();
|
let mut reader = ImageReader::new(Cursor::new(bytes))
|
||||||
let decoded = image::load_from_memory(bytes)
|
.with_guessed_format()
|
||||||
|
.map_err(|e| tracing::debug!("unrecognized image: {e}"))
|
||||||
|
.ok()?;
|
||||||
|
let format = reader.format();
|
||||||
|
let mut limits = Limits::default();
|
||||||
|
limits.max_image_width = Some(MAX_DECODED_DIMENSION_PX);
|
||||||
|
limits.max_image_height = Some(MAX_DECODED_DIMENSION_PX);
|
||||||
|
limits.max_alloc = Some(MAX_DECODE_ALLOC_BYTES);
|
||||||
|
reader.limits(limits);
|
||||||
|
let decoded = reader
|
||||||
|
.decode()
|
||||||
.map_err(|e| tracing::debug!("undecodable image: {e}"))
|
.map_err(|e| tracing::debug!("undecodable image: {e}"))
|
||||||
.ok()?;
|
.ok()?;
|
||||||
|
|
||||||
@@ -267,6 +281,20 @@ mod tests {
|
|||||||
assert!(reencode(b"not an image at all", ImageProfile::STANDARD).is_none());
|
assert!(reencode(b"not an image at all", ImageProfile::STANDARD).is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn reencode_rejects_excessive_source_dimensions() {
|
||||||
|
// A narrow image keeps the fixture cheap while exercising the strict
|
||||||
|
// source-dimension limit before any large decoded buffer is allocated.
|
||||||
|
let img = image::GrayImage::new(MAX_DECODED_DIMENSION_PX + 1, MIN_DIMENSION_PX);
|
||||||
|
let mut png = Cursor::new(Vec::new());
|
||||||
|
DynamicImage::ImageLuma8(img)
|
||||||
|
.write_to(&mut png, ImageFormat::Png)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert!(reencode(&png.into_inner(), ImageProfile::STANDARD).is_none());
|
||||||
|
assert_eq!(MAX_DECODE_ALLOC_BYTES, 512 * 1024 * 1024);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn svg_charts_are_rasterized_rather_than_dropped() {
|
fn svg_charts_are_rasterized_rather_than_dropped() {
|
||||||
let svg = br##"<svg xmlns="http://www.w3.org/2000/svg" width="400" height="300">
|
let svg = br##"<svg xmlns="http://www.w3.org/2000/svg" width="400" height="300">
|
||||||
|
|||||||
+25
-4
@@ -11,13 +11,21 @@ use super::refs::{ImgRef, extract_img_refs};
|
|||||||
|
|
||||||
/// Per-image download timeout (§3.10).
|
/// Per-image download timeout (§3.10).
|
||||||
pub const DOWNLOAD_TIMEOUT_SECS: u64 = 10;
|
pub const DOWNLOAD_TIMEOUT_SECS: u64 = 10;
|
||||||
/// Per-image size cap (§3.10).
|
/// Per-image compressed download cap (§3.10).
|
||||||
pub const MAX_IMAGE_BYTES: usize = 5 * 1024 * 1024;
|
///
|
||||||
|
/// Source screenshots and lossless artwork can be much larger than the asset we
|
||||||
|
/// ultimately embed. Decoded dimensions and allocations are capped separately
|
||||||
|
/// by the encoder, and the original bytes are discarded after re-encoding.
|
||||||
|
pub const MAX_IMAGE_BYTES: usize = 100 * 1024 * 1024;
|
||||||
/// Concurrent downloads (§3.10).
|
/// Concurrent downloads (§3.10).
|
||||||
pub const CONCURRENCY: usize = 8;
|
pub const CONCURRENCY: usize = 8;
|
||||||
/// Whole-issue asset budget (§3.10).
|
/// Whole-issue asset budget (§3.10).
|
||||||
pub const ISSUE_ASSET_BUDGET_BYTES: usize = 25 * 1024 * 1024;
|
pub const ISSUE_ASSET_BUDGET_BYTES: usize = 25 * 1024 * 1024;
|
||||||
|
|
||||||
|
fn exceeds_download_cap(bytes: usize) -> bool {
|
||||||
|
bytes > MAX_IMAGE_BYTES
|
||||||
|
}
|
||||||
|
|
||||||
/// Download one image, honoring the timeout and size cap (§3.10).
|
/// Download one image, honoring the timeout and size cap (§3.10).
|
||||||
pub async fn download(http: &reqwest::Client, url: &str) -> Option<Vec<u8>> {
|
pub async fn download(http: &reqwest::Client, url: &str) -> Option<Vec<u8>> {
|
||||||
let resp = http
|
let resp = http
|
||||||
@@ -34,7 +42,7 @@ pub async fn download(http: &reqwest::Client, url: &str) -> Option<Vec<u8>> {
|
|||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
if let Some(len) = resp.content_length()
|
if let Some(len) = resp.content_length()
|
||||||
&& len as usize > MAX_IMAGE_BYTES
|
&& usize::try_from(len).map_or(true, exceeds_download_cap)
|
||||||
{
|
{
|
||||||
tracing::debug!(url, len, "image exceeds the size cap");
|
tracing::debug!(url, len, "image exceeds the size cap");
|
||||||
return None;
|
return None;
|
||||||
@@ -44,7 +52,7 @@ pub async fn download(http: &reqwest::Client, url: &str) -> Option<Vec<u8>> {
|
|||||||
loop {
|
loop {
|
||||||
match resp.chunk().await {
|
match resp.chunk().await {
|
||||||
Ok(Some(chunk)) => {
|
Ok(Some(chunk)) => {
|
||||||
if buf.len() + chunk.len() > MAX_IMAGE_BYTES {
|
if exceeds_download_cap(buf.len().saturating_add(chunk.len())) {
|
||||||
tracing::debug!(url, "image exceeds the size cap mid-stream");
|
tracing::debug!(url, "image exceeds the size cap mid-stream");
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
@@ -157,3 +165,16 @@ pub async fn collect_for_issue(
|
|||||||
);
|
);
|
||||||
assets
|
assets
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn compressed_download_cap_is_large_but_bounded() {
|
||||||
|
assert_eq!(MAX_IMAGE_BYTES, 100 * 1024 * 1024);
|
||||||
|
assert!(!exceeds_download_cap(MAX_IMAGE_BYTES));
|
||||||
|
assert!(exceeds_download_cap(MAX_IMAGE_BYTES + 1));
|
||||||
|
assert!(!exceeds_download_cap(5_790_082));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user