geocode: use craft-zip for GeoNames downloads (#4566)

* T006: read GeoNames ZIPs with craft-zip

* geocode: take craft-zip at the workspace's craft-libs rev; build the fixture with its writer

craft-zip at 23acc704, the rev every other craft-libs crate here uses, has
the same Archive and Writer code as 5c496f6e (that rev only adds the
seekable reader), so the workspace stays on one craft-libs rev.

unzip::make now builds its one-file fixture with craft_zip::Writer instead
of hand-written headers, so geocode no longer needs crc32fast or
miniz_oxide.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

* geocode: retain the selected ZIP entry index

---------

Co-authored-by: FredApps <[email protected]>
Co-authored-by: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Terra854
2026-11-19 21:31:08 -05:00
committed by GitHub
co-authored by Claude Opus 5.5 FredApps
parent a3f65b67b8
commit 976f52ab69
3 changed files with 151 additions and 94 deletions
Generated
+1 -1
View File
@@ -4116,7 +4116,7 @@ dependencies = [
name = "lightcraft-geocode"
version = "0.21.1"
dependencies = [
"miniz_oxide 0.9.1",
"craft-zip",
"serde",
]
+1 -1
View File
@@ -9,5 +9,5 @@ description = "LightCraft offline reverse geocoding: GPS position to sublocation
workspace = true
[dependencies]
miniz_oxide = { workspace = true }
craft-zip = { workspace = true }
serde = { workspace = true }
+149 -92
View File
@@ -1,113 +1,63 @@
//! Just enough ZIP to read one file out of a GeoNames dump (`cities15000.zip`): the central
//! directory, stored or deflated entries, sizes capped.
//!
//! Sources: PKWARE APPNOTE (prose spec); own design.
//! Read the city-list entry from a GeoNames ZIP within a caller-owned output limit.
use craft_zip::{Archive, Limits};
use crate::GeocodeError;
fn u16_at(b: &[u8], i: usize) -> Option<u16> {
let s = b.get(i..i.checked_add(2)?)?;
Some(u16::from_le_bytes([*s.first()?, *s.get(1)?]))
}
fn u32_at(b: &[u8], i: usize) -> Option<u32> {
let s = b.get(i..i.checked_add(4)?)?;
Some(u32::from_le_bytes([*s.first()?, *s.get(1)?, *s.get(2)?, *s.get(3)?]))
}
fn bad(what: &str) -> GeocodeError {
GeocodeError::Invalid(format!("not a readable ZIP archive ({what})"))
}
/// The contents of the entry whose name ends with `name`, at most `max` bytes.
/// The contents of the first entry whose name ends with `name`, at most `max` bytes.
pub fn extract(zip: &[u8], name: &str, max: usize) -> Result<Vec<u8>, GeocodeError> {
// end of central directory: the last "PK\x05\x06" within the final 64 KiB + 22 bytes
let from = zip.len().saturating_sub(65_557);
let eocd = (from..zip.len().saturating_sub(21))
.rev()
.find(|&i| zip.get(i..i.saturating_add(4)) == Some(b"PK\x05\x06"))
.ok_or_else(|| bad("no directory"))?;
let field16 = |at: usize, what: &str| u16_at(zip, at).map(usize::from).ok_or_else(|| bad(what));
let field32 = |at: usize, what: &str| u32_at(zip, at).and_then(|v| usize::try_from(v).ok()).ok_or_else(|| bad(what));
let count = field16(eocd.saturating_add(10), "directory")?;
let mut p = field32(eocd.saturating_add(16), "directory")?;
for _ in 0..count {
if zip.get(p..p.saturating_add(4)) != Some(b"PK\x01\x02") {
return Err(bad("directory entry"));
}
let method = field16(p.saturating_add(10), "entry")?;
let csize = field32(p.saturating_add(20), "entry")?;
let usize_ = field32(p.saturating_add(24), "entry")?;
let nlen = field16(p.saturating_add(28), "entry")?;
let xlen = field16(p.saturating_add(30), "entry")?;
let clen = field16(p.saturating_add(32), "entry")?;
let local = field32(p.saturating_add(42), "entry")?;
let name_at = p.saturating_add(46);
let entry = zip.get(name_at..name_at.saturating_add(nlen)).ok_or_else(|| bad("entry name"))?;
p = name_at.saturating_add(nlen).saturating_add(xlen).saturating_add(clen);
if !String::from_utf8_lossy(entry).ends_with(name) {
continue;
}
if usize_ > max {
return Err(GeocodeError::Invalid(format!("{name} is larger than {max} bytes")));
}
if zip.get(local..local.saturating_add(4)) != Some(b"PK\x03\x04") {
return Err(bad("local header"));
}
let lnlen = field16(local.saturating_add(26), "local header")?;
let lxlen = field16(local.saturating_add(28), "local header")?;
let start = local.saturating_add(30).saturating_add(lnlen).saturating_add(lxlen);
let data = zip.get(start..start.saturating_add(csize)).ok_or_else(|| bad("truncated"))?;
return match method {
0 if data.len() <= max => Ok(data.to_vec()),
0 => Err(GeocodeError::Invalid(format!("{name} is larger than {max} bytes"))),
8 => {
let decoded = miniz_oxide::inflate::decompress_to_vec_with_limit(data, max).map_err(|e| bad(&format!("deflate: {:?}", e.status)))?;
if decoded.len() != usize_ {
return Err(bad("uncompressed size"));
}
Ok(decoded)
}
m => Err(bad(&format!("compression method {m}"))),
};
let max_bytes = u64::try_from(max).map_err(|_| bad("output limit"))?;
let mut archive =
Archive::open(zip, Limits { max_entries: u64::from(u16::MAX), max_total_bytes: max_bytes }).map_err(|error| bad(&error.to_string()))?;
// Exact duplicate names make the shared index's selected entry ambiguous.
if archive.has_duplicate_names() {
return Err(bad("duplicate entry name"));
}
Err(GeocodeError::Invalid(format!("{name} is not in the archive")))
// Different raw names can decode to the same text. Keep the first matching index.
let mut selected = None;
for index in 0..archive.len() {
let entry = archive.entry_name(index).map_err(|error| bad(&error.to_string()))?;
if entry.is_some_and(|entry| entry.ends_with(name)) {
selected = Some(index);
break;
}
}
let index = selected.ok_or_else(|| GeocodeError::Invalid(format!("{name} is not in the archive")))?;
let (declared, _) = archive.entry_sizes(index).map_err(|error| bad(&error.to_string()))?.ok_or_else(|| bad("entry size"))?;
let decoded = archive
.read_index(index, max_bytes)
.map_err(
|error| {
if error.is_limit() { GeocodeError::Invalid(format!("{name} is larger than {max} bytes")) } else { bad(&error.to_string()) }
},
)?
.ok_or_else(|| bad("entry"))?;
// A short read can remain within the output budget, so check the exact size too.
if decoded.len() as u64 != declared {
return Err(bad("uncompressed size"));
}
Ok(decoded)
}
/// A one-file deflated ZIP (test fixtures: the engine's download test serves one).
/// A one-file deflated ZIP (test fixtures: the engine's download test serves one). A writer error
/// gives an empty archive, which [`extract`] refuses.
#[doc(hidden)]
pub fn make(name: &str, data: &[u8]) -> Vec<u8> {
let c = miniz_oxide::deflate::compress_to_vec(data, 6);
let mut z = Vec::new();
z.extend_from_slice(b"PK\x03\x04");
z.extend_from_slice(&[20, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0]);
z.extend_from_slice(&(c.len() as u32).to_le_bytes());
z.extend_from_slice(&(data.len() as u32).to_le_bytes());
z.extend_from_slice(&(name.len() as u16).to_le_bytes());
z.extend_from_slice(&[0, 0]);
z.extend_from_slice(name.as_bytes());
z.extend_from_slice(&c);
let cd = z.len();
z.extend_from_slice(b"PK\x01\x02");
z.extend_from_slice(&[20, 0, 20, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0]);
z.extend_from_slice(&(c.len() as u32).to_le_bytes());
z.extend_from_slice(&(data.len() as u32).to_le_bytes());
z.extend_from_slice(&(name.len() as u16).to_le_bytes());
z.extend_from_slice(&[0; 12]);
z.extend_from_slice(&0u32.to_le_bytes());
z.extend_from_slice(name.as_bytes());
let cdlen = z.len() - cd;
z.extend_from_slice(b"PK\x05\x06");
z.extend_from_slice(&[0, 0, 0, 0, 1, 0, 1, 0]);
z.extend_from_slice(&(cdlen as u32).to_le_bytes());
z.extend_from_slice(&(cd as u32).to_le_bytes());
z.extend_from_slice(&[0, 0]);
z
let mut writer = craft_zip::Writer::new();
if writer.add(name, data, craft_zip::Compression::Deflated).is_err() {
return Vec::new();
}
writer.finish().unwrap_or_default()
}
#[cfg(test)]
mod tests {
use super::*;
use craft_zip::{Compression, Writer};
#[test]
fn round_trip_and_junk() {
@@ -137,4 +87,111 @@ mod tests {
assert!(extract(&z, "cities15000.txt", 100).is_err());
}
#[test]
fn reads_stored_and_deflated_entries_at_the_exact_limit() {
for compression in [Compression::Stored, Compression::Deflated] {
let zip = archive(&[("folder/cities15000.txt", b"Paris\n")], compression);
assert_eq!(extract(&zip, "cities15000.txt", 6).unwrap(), b"Paris\n");
assert!(extract(&zip, "cities15000.txt", 5).is_err());
assert_eq!(extract(&zip, "missing.txt", 6).unwrap_err().to_string(), "missing.txt is not in the archive");
}
}
#[test]
fn reads_the_first_distinct_suffix_match_in_directory_order() {
let zip = archive(&[("z/cities15000.txt", b"first"), ("a/cities15000.txt", b"second")], Compression::Deflated);
assert_eq!(extract(&zip, "cities15000.txt", 100).unwrap(), b"first");
}
#[test]
fn reads_first_suffix_match_with_colliding_decoded_names() {
let mut zip = archive(&[("a/cities15000.txt", b"first"), ("é/cities15000.txt", b"second")], Compression::Stored);
let first_directory = directory_entries(&zip)[0];
// Keep the name length, replace 'a' with CP437 'é' and clear the UTF-8 flags.
zip[30] = 0x82;
zip[7] &= !0x08;
zip[first_directory + 46] = 0x82;
zip[first_directory + 9] &= !0x08;
assert_eq!(extract(&zip, "cities15000.txt", 100).unwrap(), b"first");
}
#[test]
fn rejects_bad_checksums_without_trying_a_later_suffix_match() {
for compression in [Compression::Stored, Compression::Deflated] {
let mut zip = archive(&[("a/cities15000.txt", b"first"), ("b/cities15000.txt", b"second")], compression);
let directory = directory_entries(&zip);
zip[14..18].copy_from_slice(&0u32.to_le_bytes());
zip[directory[0] + 16..directory[0] + 20].copy_from_slice(&0u32.to_le_bytes());
assert!(extract(&zip, "cities15000.txt", 100).is_err());
}
}
#[test]
fn rejects_duplicate_exact_member_names() {
let mut zip = archive(&[("a/cities15000.txt", b"first"), ("b/cities15000.txt", b"second")], Compression::Stored);
let names: Vec<_> =
zip.windows(b"b/cities15000.txt".len()).enumerate().filter_map(|(at, bytes)| (bytes == b"b/cities15000.txt").then_some(at)).collect();
assert_eq!(names.len(), 2);
for at in names {
zip[at] = b'a';
}
assert!(extract(&zip, "cities15000.txt", 100).is_err());
}
#[test]
fn rejects_shorter_and_longer_declared_sizes() {
for compression in [Compression::Stored, Compression::Deflated] {
for declared in [4u32, 6] {
let mut zip = archive(&[("cities15000.txt", b"Paris")], compression);
let directory = directory_entries(&zip)[0];
zip[22..26].copy_from_slice(&declared.to_le_bytes());
zip[directory + 24..directory + 28].copy_from_slice(&declared.to_le_bytes());
assert!(extract(&zip, "cities15000.txt", 100).is_err(), "{compression:?}, declared {declared}");
}
}
}
#[test]
fn rejects_truncation_and_unsupported_compression() {
let original = archive(&[("cities15000.txt", b"Paris")], Compression::Stored);
for cut in [0, 4, 29, original.len() - 1] {
assert!(extract(&original[..cut], "cities15000.txt", 100).is_err());
}
let mut unsupported = original.clone();
let directory = directory_entries(&unsupported)[0];
unsupported[8..10].copy_from_slice(&99u16.to_le_bytes());
unsupported[directory + 10..directory + 12].copy_from_slice(&99u16.to_le_bytes());
assert!(extract(&unsupported, "cities15000.txt", 100).is_err());
let mut truncated_payload = original;
truncated_payload[directory + 20..directory + 24].copy_from_slice(&u32::MAX.to_le_bytes());
assert!(extract(&truncated_payload, "cities15000.txt", 100).is_err());
}
fn archive(entries: &[(&str, &[u8])], compression: Compression) -> Vec<u8> {
let mut writer = Writer::new();
for (name, bytes) in entries {
writer.add(name, bytes, compression).unwrap();
}
writer.finish().unwrap()
}
fn directory_entries(zip: &[u8]) -> Vec<usize> {
let end = zip.len() - 22;
assert_eq!(&zip[end..end + 4], b"PK\x05\x06");
let count = u16::from_le_bytes(zip[end + 10..end + 12].try_into().unwrap());
let mut at = u32::from_le_bytes(zip[end + 16..end + 20].try_into().unwrap()) as usize;
let mut entries = Vec::new();
for _ in 0..count {
assert_eq!(&zip[at..at + 4], b"PK\x01\x02");
entries.push(at);
let name = u16::from_le_bytes(zip[at + 28..at + 30].try_into().unwrap()) as usize;
let extra = u16::from_le_bytes(zip[at + 30..at + 32].try_into().unwrap()) as usize;
let comment = u16::from_le_bytes(zip[at + 32..at + 34].try_into().unwrap()) as usize;
at += 46 + name + extra + comment;
}
entries
}
}