mirror of
https://github.com/storytold/lightcraft.git
synced 2026-11-21 20:38:15 +00:00
geocode: use craft-zip for GeoNames downloads (#4566)
* T006: read GeoNames ZIPs with craft-zip * geocode: take craft-zip at the workspace's craft-libs rev; build the fixture with its writer craft-zip at 23acc704, the rev every other craft-libs crate here uses, has the same Archive and Writer code as 5c496f6e (that rev only adds the seekable reader), so the workspace stays on one craft-libs rev. unzip::make now builds its one-file fixture with craft_zip::Writer instead of hand-written headers, so geocode no longer needs crc32fast or miniz_oxide. Co-Authored-By: Claude Opus 5.5 <[email protected]> * geocode: retain the selected ZIP entry index --------- Co-authored-by: FredApps <[email protected]> Co-authored-by: Claude Opus 5.5 <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 5.5
FredApps
parent
a3f65b67b8
commit
976f52ab69
Generated
+1
-1
@@ -4116,7 +4116,7 @@ dependencies = [
|
||||
name = "lightcraft-geocode"
|
||||
version = "0.21.1"
|
||||
dependencies = [
|
||||
"miniz_oxide 0.9.1",
|
||||
"craft-zip",
|
||||
"serde",
|
||||
]
|
||||
|
||||
|
||||
@@ -9,5 +9,5 @@ description = "LightCraft offline reverse geocoding: GPS position to sublocation
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
miniz_oxide = { workspace = true }
|
||||
craft-zip = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
|
||||
+149
-92
@@ -1,113 +1,63 @@
|
||||
//! Just enough ZIP to read one file out of a GeoNames dump (`cities15000.zip`): the central
|
||||
//! directory, stored or deflated entries, sizes capped.
|
||||
//!
|
||||
//! Sources: PKWARE APPNOTE (prose spec); own design.
|
||||
//! Read the city-list entry from a GeoNames ZIP within a caller-owned output limit.
|
||||
|
||||
use craft_zip::{Archive, Limits};
|
||||
|
||||
use crate::GeocodeError;
|
||||
|
||||
fn u16_at(b: &[u8], i: usize) -> Option<u16> {
|
||||
let s = b.get(i..i.checked_add(2)?)?;
|
||||
Some(u16::from_le_bytes([*s.first()?, *s.get(1)?]))
|
||||
}
|
||||
|
||||
fn u32_at(b: &[u8], i: usize) -> Option<u32> {
|
||||
let s = b.get(i..i.checked_add(4)?)?;
|
||||
Some(u32::from_le_bytes([*s.first()?, *s.get(1)?, *s.get(2)?, *s.get(3)?]))
|
||||
}
|
||||
|
||||
fn bad(what: &str) -> GeocodeError {
|
||||
GeocodeError::Invalid(format!("not a readable ZIP archive ({what})"))
|
||||
}
|
||||
|
||||
/// The contents of the entry whose name ends with `name`, at most `max` bytes.
|
||||
/// The contents of the first entry whose name ends with `name`, at most `max` bytes.
|
||||
pub fn extract(zip: &[u8], name: &str, max: usize) -> Result<Vec<u8>, GeocodeError> {
|
||||
// end of central directory: the last "PK\x05\x06" within the final 64 KiB + 22 bytes
|
||||
let from = zip.len().saturating_sub(65_557);
|
||||
let eocd = (from..zip.len().saturating_sub(21))
|
||||
.rev()
|
||||
.find(|&i| zip.get(i..i.saturating_add(4)) == Some(b"PK\x05\x06"))
|
||||
.ok_or_else(|| bad("no directory"))?;
|
||||
let field16 = |at: usize, what: &str| u16_at(zip, at).map(usize::from).ok_or_else(|| bad(what));
|
||||
let field32 = |at: usize, what: &str| u32_at(zip, at).and_then(|v| usize::try_from(v).ok()).ok_or_else(|| bad(what));
|
||||
let count = field16(eocd.saturating_add(10), "directory")?;
|
||||
let mut p = field32(eocd.saturating_add(16), "directory")?;
|
||||
for _ in 0..count {
|
||||
if zip.get(p..p.saturating_add(4)) != Some(b"PK\x01\x02") {
|
||||
return Err(bad("directory entry"));
|
||||
}
|
||||
let method = field16(p.saturating_add(10), "entry")?;
|
||||
let csize = field32(p.saturating_add(20), "entry")?;
|
||||
let usize_ = field32(p.saturating_add(24), "entry")?;
|
||||
let nlen = field16(p.saturating_add(28), "entry")?;
|
||||
let xlen = field16(p.saturating_add(30), "entry")?;
|
||||
let clen = field16(p.saturating_add(32), "entry")?;
|
||||
let local = field32(p.saturating_add(42), "entry")?;
|
||||
let name_at = p.saturating_add(46);
|
||||
let entry = zip.get(name_at..name_at.saturating_add(nlen)).ok_or_else(|| bad("entry name"))?;
|
||||
p = name_at.saturating_add(nlen).saturating_add(xlen).saturating_add(clen);
|
||||
if !String::from_utf8_lossy(entry).ends_with(name) {
|
||||
continue;
|
||||
}
|
||||
if usize_ > max {
|
||||
return Err(GeocodeError::Invalid(format!("{name} is larger than {max} bytes")));
|
||||
}
|
||||
if zip.get(local..local.saturating_add(4)) != Some(b"PK\x03\x04") {
|
||||
return Err(bad("local header"));
|
||||
}
|
||||
let lnlen = field16(local.saturating_add(26), "local header")?;
|
||||
let lxlen = field16(local.saturating_add(28), "local header")?;
|
||||
let start = local.saturating_add(30).saturating_add(lnlen).saturating_add(lxlen);
|
||||
let data = zip.get(start..start.saturating_add(csize)).ok_or_else(|| bad("truncated"))?;
|
||||
return match method {
|
||||
0 if data.len() <= max => Ok(data.to_vec()),
|
||||
0 => Err(GeocodeError::Invalid(format!("{name} is larger than {max} bytes"))),
|
||||
8 => {
|
||||
let decoded = miniz_oxide::inflate::decompress_to_vec_with_limit(data, max).map_err(|e| bad(&format!("deflate: {:?}", e.status)))?;
|
||||
if decoded.len() != usize_ {
|
||||
return Err(bad("uncompressed size"));
|
||||
}
|
||||
Ok(decoded)
|
||||
}
|
||||
m => Err(bad(&format!("compression method {m}"))),
|
||||
};
|
||||
let max_bytes = u64::try_from(max).map_err(|_| bad("output limit"))?;
|
||||
let mut archive =
|
||||
Archive::open(zip, Limits { max_entries: u64::from(u16::MAX), max_total_bytes: max_bytes }).map_err(|error| bad(&error.to_string()))?;
|
||||
// Exact duplicate names make the shared index's selected entry ambiguous.
|
||||
if archive.has_duplicate_names() {
|
||||
return Err(bad("duplicate entry name"));
|
||||
}
|
||||
Err(GeocodeError::Invalid(format!("{name} is not in the archive")))
|
||||
// Different raw names can decode to the same text. Keep the first matching index.
|
||||
let mut selected = None;
|
||||
for index in 0..archive.len() {
|
||||
let entry = archive.entry_name(index).map_err(|error| bad(&error.to_string()))?;
|
||||
if entry.is_some_and(|entry| entry.ends_with(name)) {
|
||||
selected = Some(index);
|
||||
break;
|
||||
}
|
||||
}
|
||||
let index = selected.ok_or_else(|| GeocodeError::Invalid(format!("{name} is not in the archive")))?;
|
||||
let (declared, _) = archive.entry_sizes(index).map_err(|error| bad(&error.to_string()))?.ok_or_else(|| bad("entry size"))?;
|
||||
let decoded = archive
|
||||
.read_index(index, max_bytes)
|
||||
.map_err(
|
||||
|error| {
|
||||
if error.is_limit() { GeocodeError::Invalid(format!("{name} is larger than {max} bytes")) } else { bad(&error.to_string()) }
|
||||
},
|
||||
)?
|
||||
.ok_or_else(|| bad("entry"))?;
|
||||
// A short read can remain within the output budget, so check the exact size too.
|
||||
if decoded.len() as u64 != declared {
|
||||
return Err(bad("uncompressed size"));
|
||||
}
|
||||
Ok(decoded)
|
||||
}
|
||||
|
||||
/// A one-file deflated ZIP (test fixtures: the engine's download test serves one).
|
||||
/// A one-file deflated ZIP (test fixtures: the engine's download test serves one). A writer error
|
||||
/// gives an empty archive, which [`extract`] refuses.
|
||||
#[doc(hidden)]
|
||||
pub fn make(name: &str, data: &[u8]) -> Vec<u8> {
|
||||
let c = miniz_oxide::deflate::compress_to_vec(data, 6);
|
||||
let mut z = Vec::new();
|
||||
z.extend_from_slice(b"PK\x03\x04");
|
||||
z.extend_from_slice(&[20, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0]);
|
||||
z.extend_from_slice(&(c.len() as u32).to_le_bytes());
|
||||
z.extend_from_slice(&(data.len() as u32).to_le_bytes());
|
||||
z.extend_from_slice(&(name.len() as u16).to_le_bytes());
|
||||
z.extend_from_slice(&[0, 0]);
|
||||
z.extend_from_slice(name.as_bytes());
|
||||
z.extend_from_slice(&c);
|
||||
let cd = z.len();
|
||||
z.extend_from_slice(b"PK\x01\x02");
|
||||
z.extend_from_slice(&[20, 0, 20, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0]);
|
||||
z.extend_from_slice(&(c.len() as u32).to_le_bytes());
|
||||
z.extend_from_slice(&(data.len() as u32).to_le_bytes());
|
||||
z.extend_from_slice(&(name.len() as u16).to_le_bytes());
|
||||
z.extend_from_slice(&[0; 12]);
|
||||
z.extend_from_slice(&0u32.to_le_bytes());
|
||||
z.extend_from_slice(name.as_bytes());
|
||||
let cdlen = z.len() - cd;
|
||||
z.extend_from_slice(b"PK\x05\x06");
|
||||
z.extend_from_slice(&[0, 0, 0, 0, 1, 0, 1, 0]);
|
||||
z.extend_from_slice(&(cdlen as u32).to_le_bytes());
|
||||
z.extend_from_slice(&(cd as u32).to_le_bytes());
|
||||
z.extend_from_slice(&[0, 0]);
|
||||
z
|
||||
let mut writer = craft_zip::Writer::new();
|
||||
if writer.add(name, data, craft_zip::Compression::Deflated).is_err() {
|
||||
return Vec::new();
|
||||
}
|
||||
writer.finish().unwrap_or_default()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use craft_zip::{Compression, Writer};
|
||||
|
||||
#[test]
|
||||
fn round_trip_and_junk() {
|
||||
@@ -137,4 +87,111 @@ mod tests {
|
||||
|
||||
assert!(extract(&z, "cities15000.txt", 100).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_stored_and_deflated_entries_at_the_exact_limit() {
|
||||
for compression in [Compression::Stored, Compression::Deflated] {
|
||||
let zip = archive(&[("folder/cities15000.txt", b"Paris\n")], compression);
|
||||
assert_eq!(extract(&zip, "cities15000.txt", 6).unwrap(), b"Paris\n");
|
||||
assert!(extract(&zip, "cities15000.txt", 5).is_err());
|
||||
assert_eq!(extract(&zip, "missing.txt", 6).unwrap_err().to_string(), "missing.txt is not in the archive");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_the_first_distinct_suffix_match_in_directory_order() {
|
||||
let zip = archive(&[("z/cities15000.txt", b"first"), ("a/cities15000.txt", b"second")], Compression::Deflated);
|
||||
assert_eq!(extract(&zip, "cities15000.txt", 100).unwrap(), b"first");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_first_suffix_match_with_colliding_decoded_names() {
|
||||
let mut zip = archive(&[("a/cities15000.txt", b"first"), ("é/cities15000.txt", b"second")], Compression::Stored);
|
||||
let first_directory = directory_entries(&zip)[0];
|
||||
// Keep the name length, replace 'a' with CP437 'é' and clear the UTF-8 flags.
|
||||
zip[30] = 0x82;
|
||||
zip[7] &= !0x08;
|
||||
zip[first_directory + 46] = 0x82;
|
||||
zip[first_directory + 9] &= !0x08;
|
||||
|
||||
assert_eq!(extract(&zip, "cities15000.txt", 100).unwrap(), b"first");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_bad_checksums_without_trying_a_later_suffix_match() {
|
||||
for compression in [Compression::Stored, Compression::Deflated] {
|
||||
let mut zip = archive(&[("a/cities15000.txt", b"first"), ("b/cities15000.txt", b"second")], compression);
|
||||
let directory = directory_entries(&zip);
|
||||
zip[14..18].copy_from_slice(&0u32.to_le_bytes());
|
||||
zip[directory[0] + 16..directory[0] + 20].copy_from_slice(&0u32.to_le_bytes());
|
||||
assert!(extract(&zip, "cities15000.txt", 100).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_duplicate_exact_member_names() {
|
||||
let mut zip = archive(&[("a/cities15000.txt", b"first"), ("b/cities15000.txt", b"second")], Compression::Stored);
|
||||
let names: Vec<_> =
|
||||
zip.windows(b"b/cities15000.txt".len()).enumerate().filter_map(|(at, bytes)| (bytes == b"b/cities15000.txt").then_some(at)).collect();
|
||||
assert_eq!(names.len(), 2);
|
||||
for at in names {
|
||||
zip[at] = b'a';
|
||||
}
|
||||
assert!(extract(&zip, "cities15000.txt", 100).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_shorter_and_longer_declared_sizes() {
|
||||
for compression in [Compression::Stored, Compression::Deflated] {
|
||||
for declared in [4u32, 6] {
|
||||
let mut zip = archive(&[("cities15000.txt", b"Paris")], compression);
|
||||
let directory = directory_entries(&zip)[0];
|
||||
zip[22..26].copy_from_slice(&declared.to_le_bytes());
|
||||
zip[directory + 24..directory + 28].copy_from_slice(&declared.to_le_bytes());
|
||||
assert!(extract(&zip, "cities15000.txt", 100).is_err(), "{compression:?}, declared {declared}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_truncation_and_unsupported_compression() {
|
||||
let original = archive(&[("cities15000.txt", b"Paris")], Compression::Stored);
|
||||
for cut in [0, 4, 29, original.len() - 1] {
|
||||
assert!(extract(&original[..cut], "cities15000.txt", 100).is_err());
|
||||
}
|
||||
let mut unsupported = original.clone();
|
||||
let directory = directory_entries(&unsupported)[0];
|
||||
unsupported[8..10].copy_from_slice(&99u16.to_le_bytes());
|
||||
unsupported[directory + 10..directory + 12].copy_from_slice(&99u16.to_le_bytes());
|
||||
assert!(extract(&unsupported, "cities15000.txt", 100).is_err());
|
||||
|
||||
let mut truncated_payload = original;
|
||||
truncated_payload[directory + 20..directory + 24].copy_from_slice(&u32::MAX.to_le_bytes());
|
||||
assert!(extract(&truncated_payload, "cities15000.txt", 100).is_err());
|
||||
}
|
||||
|
||||
fn archive(entries: &[(&str, &[u8])], compression: Compression) -> Vec<u8> {
|
||||
let mut writer = Writer::new();
|
||||
for (name, bytes) in entries {
|
||||
writer.add(name, bytes, compression).unwrap();
|
||||
}
|
||||
writer.finish().unwrap()
|
||||
}
|
||||
|
||||
fn directory_entries(zip: &[u8]) -> Vec<usize> {
|
||||
let end = zip.len() - 22;
|
||||
assert_eq!(&zip[end..end + 4], b"PK\x05\x06");
|
||||
let count = u16::from_le_bytes(zip[end + 10..end + 12].try_into().unwrap());
|
||||
let mut at = u32::from_le_bytes(zip[end + 16..end + 20].try_into().unwrap()) as usize;
|
||||
let mut entries = Vec::new();
|
||||
for _ in 0..count {
|
||||
assert_eq!(&zip[at..at + 4], b"PK\x01\x02");
|
||||
entries.push(at);
|
||||
let name = u16::from_le_bytes(zip[at + 28..at + 30].try_into().unwrap()) as usize;
|
||||
let extra = u16::from_le_bytes(zip[at + 30..at + 32].try_into().unwrap()) as usize;
|
||||
let comment = u16::from_le_bytes(zip[at + 32..at + 34].try_into().unwrap()) as usize;
|
||||
at += 46 + name + extra + comment;
|
||||
}
|
||||
entries
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user