diff --git a/Cargo.lock b/Cargo.lock index 8139158efe..2f58c628af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4116,7 +4116,7 @@ dependencies = [ name = "lightcraft-geocode" version = "0.21.1" dependencies = [ - "miniz_oxide 0.9.1", + "craft-zip", "serde", ] diff --git a/crates/geocode/Cargo.toml b/crates/geocode/Cargo.toml index 0d30af7626..1169f3e292 100644 --- a/crates/geocode/Cargo.toml +++ b/crates/geocode/Cargo.toml @@ -9,5 +9,5 @@ description = "LightCraft offline reverse geocoding: GPS position to sublocation workspace = true [dependencies] -miniz_oxide = { workspace = true } +craft-zip = { workspace = true } serde = { workspace = true } diff --git a/crates/geocode/src/unzip.rs b/crates/geocode/src/unzip.rs index 49d86bafde..30afed2ecd 100644 --- a/crates/geocode/src/unzip.rs +++ b/crates/geocode/src/unzip.rs @@ -1,113 +1,63 @@ -//! Just enough ZIP to read one file out of a GeoNames dump (`cities15000.zip`): the central -//! directory, stored or deflated entries, sizes capped. -//! -//! Sources: PKWARE APPNOTE (prose spec); own design. +//! Read the city-list entry from a GeoNames ZIP within a caller-owned output limit. + +use craft_zip::{Archive, Limits}; use crate::GeocodeError; -fn u16_at(b: &[u8], i: usize) -> Option { - let s = b.get(i..i.checked_add(2)?)?; - Some(u16::from_le_bytes([*s.first()?, *s.get(1)?])) -} - -fn u32_at(b: &[u8], i: usize) -> Option { - let s = b.get(i..i.checked_add(4)?)?; - Some(u32::from_le_bytes([*s.first()?, *s.get(1)?, *s.get(2)?, *s.get(3)?])) -} - fn bad(what: &str) -> GeocodeError { GeocodeError::Invalid(format!("not a readable ZIP archive ({what})")) } -/// The contents of the entry whose name ends with `name`, at most `max` bytes. +/// The contents of the first entry whose name ends with `name`, at most `max` bytes. pub fn extract(zip: &[u8], name: &str, max: usize) -> Result, GeocodeError> { - // end of central directory: the last "PK\x05\x06" within the final 64 KiB + 22 bytes - let from = zip.len().saturating_sub(65_557); - let eocd = (from..zip.len().saturating_sub(21)) - .rev() - .find(|&i| zip.get(i..i.saturating_add(4)) == Some(b"PK\x05\x06")) - .ok_or_else(|| bad("no directory"))?; - let field16 = |at: usize, what: &str| u16_at(zip, at).map(usize::from).ok_or_else(|| bad(what)); - let field32 = |at: usize, what: &str| u32_at(zip, at).and_then(|v| usize::try_from(v).ok()).ok_or_else(|| bad(what)); - let count = field16(eocd.saturating_add(10), "directory")?; - let mut p = field32(eocd.saturating_add(16), "directory")?; - for _ in 0..count { - if zip.get(p..p.saturating_add(4)) != Some(b"PK\x01\x02") { - return Err(bad("directory entry")); - } - let method = field16(p.saturating_add(10), "entry")?; - let csize = field32(p.saturating_add(20), "entry")?; - let usize_ = field32(p.saturating_add(24), "entry")?; - let nlen = field16(p.saturating_add(28), "entry")?; - let xlen = field16(p.saturating_add(30), "entry")?; - let clen = field16(p.saturating_add(32), "entry")?; - let local = field32(p.saturating_add(42), "entry")?; - let name_at = p.saturating_add(46); - let entry = zip.get(name_at..name_at.saturating_add(nlen)).ok_or_else(|| bad("entry name"))?; - p = name_at.saturating_add(nlen).saturating_add(xlen).saturating_add(clen); - if !String::from_utf8_lossy(entry).ends_with(name) { - continue; - } - if usize_ > max { - return Err(GeocodeError::Invalid(format!("{name} is larger than {max} bytes"))); - } - if zip.get(local..local.saturating_add(4)) != Some(b"PK\x03\x04") { - return Err(bad("local header")); - } - let lnlen = field16(local.saturating_add(26), "local header")?; - let lxlen = field16(local.saturating_add(28), "local header")?; - let start = local.saturating_add(30).saturating_add(lnlen).saturating_add(lxlen); - let data = zip.get(start..start.saturating_add(csize)).ok_or_else(|| bad("truncated"))?; - return match method { - 0 if data.len() <= max => Ok(data.to_vec()), - 0 => Err(GeocodeError::Invalid(format!("{name} is larger than {max} bytes"))), - 8 => { - let decoded = miniz_oxide::inflate::decompress_to_vec_with_limit(data, max).map_err(|e| bad(&format!("deflate: {:?}", e.status)))?; - if decoded.len() != usize_ { - return Err(bad("uncompressed size")); - } - Ok(decoded) - } - m => Err(bad(&format!("compression method {m}"))), - }; + let max_bytes = u64::try_from(max).map_err(|_| bad("output limit"))?; + let mut archive = + Archive::open(zip, Limits { max_entries: u64::from(u16::MAX), max_total_bytes: max_bytes }).map_err(|error| bad(&error.to_string()))?; + // Exact duplicate names make the shared index's selected entry ambiguous. + if archive.has_duplicate_names() { + return Err(bad("duplicate entry name")); } - Err(GeocodeError::Invalid(format!("{name} is not in the archive"))) + // Different raw names can decode to the same text. Keep the first matching index. + let mut selected = None; + for index in 0..archive.len() { + let entry = archive.entry_name(index).map_err(|error| bad(&error.to_string()))?; + if entry.is_some_and(|entry| entry.ends_with(name)) { + selected = Some(index); + break; + } + } + let index = selected.ok_or_else(|| GeocodeError::Invalid(format!("{name} is not in the archive")))?; + let (declared, _) = archive.entry_sizes(index).map_err(|error| bad(&error.to_string()))?.ok_or_else(|| bad("entry size"))?; + let decoded = archive + .read_index(index, max_bytes) + .map_err( + |error| { + if error.is_limit() { GeocodeError::Invalid(format!("{name} is larger than {max} bytes")) } else { bad(&error.to_string()) } + }, + )? + .ok_or_else(|| bad("entry"))?; + // A short read can remain within the output budget, so check the exact size too. + if decoded.len() as u64 != declared { + return Err(bad("uncompressed size")); + } + Ok(decoded) } -/// A one-file deflated ZIP (test fixtures: the engine's download test serves one). +/// A one-file deflated ZIP (test fixtures: the engine's download test serves one). A writer error +/// gives an empty archive, which [`extract`] refuses. #[doc(hidden)] pub fn make(name: &str, data: &[u8]) -> Vec { - let c = miniz_oxide::deflate::compress_to_vec(data, 6); - let mut z = Vec::new(); - z.extend_from_slice(b"PK\x03\x04"); - z.extend_from_slice(&[20, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0]); - z.extend_from_slice(&(c.len() as u32).to_le_bytes()); - z.extend_from_slice(&(data.len() as u32).to_le_bytes()); - z.extend_from_slice(&(name.len() as u16).to_le_bytes()); - z.extend_from_slice(&[0, 0]); - z.extend_from_slice(name.as_bytes()); - z.extend_from_slice(&c); - let cd = z.len(); - z.extend_from_slice(b"PK\x01\x02"); - z.extend_from_slice(&[20, 0, 20, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0]); - z.extend_from_slice(&(c.len() as u32).to_le_bytes()); - z.extend_from_slice(&(data.len() as u32).to_le_bytes()); - z.extend_from_slice(&(name.len() as u16).to_le_bytes()); - z.extend_from_slice(&[0; 12]); - z.extend_from_slice(&0u32.to_le_bytes()); - z.extend_from_slice(name.as_bytes()); - let cdlen = z.len() - cd; - z.extend_from_slice(b"PK\x05\x06"); - z.extend_from_slice(&[0, 0, 0, 0, 1, 0, 1, 0]); - z.extend_from_slice(&(cdlen as u32).to_le_bytes()); - z.extend_from_slice(&(cd as u32).to_le_bytes()); - z.extend_from_slice(&[0, 0]); - z + let mut writer = craft_zip::Writer::new(); + if writer.add(name, data, craft_zip::Compression::Deflated).is_err() { + return Vec::new(); + } + writer.finish().unwrap_or_default() } #[cfg(test)] mod tests { use super::*; + use craft_zip::{Compression, Writer}; #[test] fn round_trip_and_junk() { @@ -137,4 +87,111 @@ mod tests { assert!(extract(&z, "cities15000.txt", 100).is_err()); } + + #[test] + fn reads_stored_and_deflated_entries_at_the_exact_limit() { + for compression in [Compression::Stored, Compression::Deflated] { + let zip = archive(&[("folder/cities15000.txt", b"Paris\n")], compression); + assert_eq!(extract(&zip, "cities15000.txt", 6).unwrap(), b"Paris\n"); + assert!(extract(&zip, "cities15000.txt", 5).is_err()); + assert_eq!(extract(&zip, "missing.txt", 6).unwrap_err().to_string(), "missing.txt is not in the archive"); + } + } + + #[test] + fn reads_the_first_distinct_suffix_match_in_directory_order() { + let zip = archive(&[("z/cities15000.txt", b"first"), ("a/cities15000.txt", b"second")], Compression::Deflated); + assert_eq!(extract(&zip, "cities15000.txt", 100).unwrap(), b"first"); + } + + #[test] + fn reads_first_suffix_match_with_colliding_decoded_names() { + let mut zip = archive(&[("a/cities15000.txt", b"first"), ("é/cities15000.txt", b"second")], Compression::Stored); + let first_directory = directory_entries(&zip)[0]; + // Keep the name length, replace 'a' with CP437 'é' and clear the UTF-8 flags. + zip[30] = 0x82; + zip[7] &= !0x08; + zip[first_directory + 46] = 0x82; + zip[first_directory + 9] &= !0x08; + + assert_eq!(extract(&zip, "cities15000.txt", 100).unwrap(), b"first"); + } + + #[test] + fn rejects_bad_checksums_without_trying_a_later_suffix_match() { + for compression in [Compression::Stored, Compression::Deflated] { + let mut zip = archive(&[("a/cities15000.txt", b"first"), ("b/cities15000.txt", b"second")], compression); + let directory = directory_entries(&zip); + zip[14..18].copy_from_slice(&0u32.to_le_bytes()); + zip[directory[0] + 16..directory[0] + 20].copy_from_slice(&0u32.to_le_bytes()); + assert!(extract(&zip, "cities15000.txt", 100).is_err()); + } + } + + #[test] + fn rejects_duplicate_exact_member_names() { + let mut zip = archive(&[("a/cities15000.txt", b"first"), ("b/cities15000.txt", b"second")], Compression::Stored); + let names: Vec<_> = + zip.windows(b"b/cities15000.txt".len()).enumerate().filter_map(|(at, bytes)| (bytes == b"b/cities15000.txt").then_some(at)).collect(); + assert_eq!(names.len(), 2); + for at in names { + zip[at] = b'a'; + } + assert!(extract(&zip, "cities15000.txt", 100).is_err()); + } + + #[test] + fn rejects_shorter_and_longer_declared_sizes() { + for compression in [Compression::Stored, Compression::Deflated] { + for declared in [4u32, 6] { + let mut zip = archive(&[("cities15000.txt", b"Paris")], compression); + let directory = directory_entries(&zip)[0]; + zip[22..26].copy_from_slice(&declared.to_le_bytes()); + zip[directory + 24..directory + 28].copy_from_slice(&declared.to_le_bytes()); + assert!(extract(&zip, "cities15000.txt", 100).is_err(), "{compression:?}, declared {declared}"); + } + } + } + + #[test] + fn rejects_truncation_and_unsupported_compression() { + let original = archive(&[("cities15000.txt", b"Paris")], Compression::Stored); + for cut in [0, 4, 29, original.len() - 1] { + assert!(extract(&original[..cut], "cities15000.txt", 100).is_err()); + } + let mut unsupported = original.clone(); + let directory = directory_entries(&unsupported)[0]; + unsupported[8..10].copy_from_slice(&99u16.to_le_bytes()); + unsupported[directory + 10..directory + 12].copy_from_slice(&99u16.to_le_bytes()); + assert!(extract(&unsupported, "cities15000.txt", 100).is_err()); + + let mut truncated_payload = original; + truncated_payload[directory + 20..directory + 24].copy_from_slice(&u32::MAX.to_le_bytes()); + assert!(extract(&truncated_payload, "cities15000.txt", 100).is_err()); + } + + fn archive(entries: &[(&str, &[u8])], compression: Compression) -> Vec { + let mut writer = Writer::new(); + for (name, bytes) in entries { + writer.add(name, bytes, compression).unwrap(); + } + writer.finish().unwrap() + } + + fn directory_entries(zip: &[u8]) -> Vec { + let end = zip.len() - 22; + assert_eq!(&zip[end..end + 4], b"PK\x05\x06"); + let count = u16::from_le_bytes(zip[end + 10..end + 12].try_into().unwrap()); + let mut at = u32::from_le_bytes(zip[end + 16..end + 20].try_into().unwrap()) as usize; + let mut entries = Vec::new(); + for _ in 0..count { + assert_eq!(&zip[at..at + 4], b"PK\x01\x02"); + entries.push(at); + let name = u16::from_le_bytes(zip[at + 28..at + 30].try_into().unwrap()) as usize; + let extra = u16::from_le_bytes(zip[at + 30..at + 32].try_into().unwrap()) as usize; + let comment = u16::from_le_bytes(zip[at + 32..at + 34].try_into().unwrap()) as usize; + at += 46 + name + extra + comment; + } + entries + } }