mirror of
https://github.com/storytold/effectcraft.git
synced 2026-11-21 20:38:55 +00:00
media: preserve contiguous audio packet placement across rounding and gaps (#4008)
* media: preserve contiguous audio packet placement across rounding and gaps Port @IppSec's contiguous_starts packet-placement logic from storytold/filmcraft#359 into CraftMediaSource so fixed-frame and Opus packets stay contiguous across 1ms Matroska timestamp rounding and laced blocks, while honoring real silence gaps, overlaps, and delayed first packets on both sequential playback and cold/backward seeks. Refs: storytold/effectcraft#1546 Refs: storytold/filmcraft#359 Co-authored-by: ippsec <[email protected]> * media: fix opus_packet_samples helper and TrackDecoder borrow * media: use Footage and re-exported matroska mux types in audio gap tests --------- Co-authored-by: ippsec <[email protected]>
This commit is contained in:
co-authored by
ippsec
parent
7cbdf73b15
commit
aa623b7577
@@ -159,37 +159,158 @@ impl ContainerFile {
|
||||
}
|
||||
}
|
||||
|
||||
fn audio_seek_info(&self, track_idx: usize, start: i64, native_rate: u32) -> (usize, i64) {
|
||||
match self {
|
||||
ContainerFile::Isobmff(f) => {
|
||||
let track = &f.tracks[track_idx];
|
||||
let t_track = (i128::from(start.max(0)) * i128::from(track.timescale)) / i128::from(native_rate);
|
||||
let sample_idx = track.sample_at_presentation_time(i64::try_from(t_track).unwrap_or(0)).unwrap_or(0);
|
||||
// Back up a few packets for AAC/Opus overlap/preroll.
|
||||
let seek_idx = sample_idx.saturating_sub(2);
|
||||
let sample_pts = track.presentation_pts(seek_idx).unwrap_or(0);
|
||||
let decoded_pos = i64::try_from((i128::from(sample_pts) * i128::from(native_rate)) / i128::from(track.timescale.max(1))).unwrap_or(0);
|
||||
(seek_idx, decoded_pos)
|
||||
fn audio_seek_info(audio_starts: &[i64], seek_preroll_samples: i64, start: i64) -> (usize, i64) {
|
||||
let target = start.saturating_sub(seek_preroll_samples).max(0);
|
||||
let sample_idx = audio_starts.partition_point(|&s| s <= target).saturating_sub(1);
|
||||
let seek_idx = sample_idx.saturating_sub(2);
|
||||
let decoded_pos = audio_starts.get(seek_idx).copied().unwrap_or(0);
|
||||
(seek_idx, decoded_pos)
|
||||
}
|
||||
}
|
||||
|
||||
/// Where the container puts one audio packet, for [`contiguous_starts`].
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
struct PacketTime {
|
||||
/// Its timestamp, in output samples.
|
||||
stamp: i64,
|
||||
/// Whether `stamp` is the packet's own. A Matroska frame after the first in a laced block
|
||||
/// without a duration repeats the block's timestamp.
|
||||
stamped: bool,
|
||||
/// Samples it decodes to, when known.
|
||||
samples: Option<i64>,
|
||||
}
|
||||
|
||||
/// Packet start positions in output samples: each packet starts where the previous one ends,
|
||||
/// unless its own timestamp is more than `tolerance` samples from there (a real gap or overlap in
|
||||
/// the recording) or the previous packet's length is unknown; then it starts at its timestamp.
|
||||
///
|
||||
/// Container timestamps are coarser than a sample. Matroska stores milliseconds, and a remux to MP4
|
||||
/// (OBS's) turns that rounding into uneven sample durations: a 1024-sample AAC frame at 48 kHz
|
||||
/// lasts 21.33 ms, so frames are stamped at 0, 1008, 2064, 3072… Placing each decoded packet at its
|
||||
/// stamp overwrote the end of one packet and left a hole of silence after another, a click every
|
||||
/// few packets: crackling throughout playback and export (`storytold/effectcraft#1546`,
|
||||
/// `storytold/filmcraft#359`).
|
||||
fn contiguous_starts(packets: impl IntoIterator<Item = PacketTime>, tolerance: i64) -> Vec<i64> {
|
||||
let tolerance = tolerance.max(0).unsigned_abs();
|
||||
let mut next: Option<i64> = None;
|
||||
packets
|
||||
.into_iter()
|
||||
.map(|p| {
|
||||
let start = match next {
|
||||
Some(n) if !p.stamped || n.abs_diff(p.stamp) <= tolerance => n,
|
||||
_ => p.stamp,
|
||||
};
|
||||
next = p.samples.and_then(|k| start.checked_add(k));
|
||||
start
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn fixed_packet_samples(dec: &mut AudioTrackDecoder, codec: CodecId, lookahead: &[Vec<u8>]) -> Option<i64> {
|
||||
if !matches!(codec, CodecId::Aac | CodecId::HeAac | CodecId::Mp1 | CodecId::Mp2 | CodecId::Mp3 | CodecId::Ac3 | CodecId::Eac3) {
|
||||
return None;
|
||||
}
|
||||
let mut frame_len: Option<usize> = None;
|
||||
for pkt in lookahead {
|
||||
let Ok(block) = dec.decode(pkt) else {
|
||||
dec.reset();
|
||||
return None;
|
||||
};
|
||||
let n = block.channels.first().map_or(0, Vec::len);
|
||||
if n == 0 {
|
||||
dec.reset();
|
||||
return None;
|
||||
}
|
||||
match frame_len {
|
||||
None => frame_len = Some(n),
|
||||
Some(prev) if prev != n => {
|
||||
dec.reset();
|
||||
return None;
|
||||
}
|
||||
ContainerFile::Matroska(f) => {
|
||||
let track = &f.tracks[track_idx];
|
||||
// Indexed PTS already subtracts CodecDelay rounded to the container clock.
|
||||
// Translate between that clock and the warm path's whole-frame delay.
|
||||
let tick_ns = i128::from(f.info.timestamp_scale.max(1));
|
||||
let delay_ns = i128::from(track.codec_delay_ns);
|
||||
let indexed_delay_ns = ((delay_ns + tick_ns / 2) / tick_ns) * tick_ns;
|
||||
let delay_frames = audio_frames_from_ns(delay_ns, native_rate);
|
||||
let preroll_ns = i128::from(track.seek_pre_roll_ns.max(80_000_000));
|
||||
let target_ns = ((i128::from(start.max(0)) + delay_frames) * 1_000_000_000) / i128::from(native_rate);
|
||||
let seek_ns = (target_ns - preroll_ns - indexed_delay_ns).max(0);
|
||||
let t_track = track.ns_to_ticks(i64::try_from(seek_ns).unwrap_or(i64::MAX));
|
||||
let sample_idx = track.sample_at_pts(t_track).unwrap_or(0);
|
||||
let seek_idx = sample_idx.saturating_sub(2);
|
||||
let sample_pts = track.samples.get(seek_idx).map_or(0, |s| s.pts);
|
||||
let sample_ns = i128::from(track.ticks_to_ns(sample_pts)) + indexed_delay_ns;
|
||||
let decoded_pos = i64::try_from(audio_frames_from_ns(sample_ns, native_rate) - delay_frames).unwrap_or(0);
|
||||
(seek_idx, decoded_pos)
|
||||
Some(_) => {}
|
||||
}
|
||||
}
|
||||
dec.reset();
|
||||
frame_len.and_then(|n| i64::try_from(n).ok())
|
||||
}
|
||||
|
||||
/// Duration of an Opus packet in 48 kHz samples from its TOC (RFC 6716 §3.1); `None` if invalid.
|
||||
fn opus_packet_samples(pkt: &[u8]) -> Option<u32> {
|
||||
let &toc = pkt.first()?;
|
||||
let config = toc >> 3;
|
||||
let frame = match config {
|
||||
0..=11 => [480, 960, 1920, 2880][(config & 3) as usize],
|
||||
12..=15 => [480, 960][(config & 1) as usize],
|
||||
_ => [120, 240, 480, 960][(config & 3) as usize],
|
||||
};
|
||||
let frames = match toc & 3 {
|
||||
0 => 1,
|
||||
1 | 2 => 2,
|
||||
_ => (*pkt.get(1)? & 0x3F) as u32,
|
||||
};
|
||||
let n = frame * frames;
|
||||
(n > 0 && n <= 5760).then_some(n)
|
||||
}
|
||||
|
||||
fn compute_audio_starts(
|
||||
container: &ContainerFile,
|
||||
source: &dyn ByteSource,
|
||||
track_idx: usize,
|
||||
codec: CodecId,
|
||||
native_rate: u32,
|
||||
fixed_samples: Option<i64>,
|
||||
) -> Vec<i64> {
|
||||
let rate = i128::from(native_rate);
|
||||
match container {
|
||||
ContainerFile::Isobmff(f) => {
|
||||
let Some(track) = f.tracks.get(track_idx) else { return Vec::new() };
|
||||
let ts = i128::from(track.timescale.max(1));
|
||||
let stamps: Vec<i64> = (0..track.samples.len())
|
||||
.map(|i| {
|
||||
let pts = track.presentation_pts(i).unwrap_or(0);
|
||||
i64::try_from(i128::from(pts) * rate / ts).unwrap_or(0)
|
||||
})
|
||||
.collect();
|
||||
let unit = i64::try_from((rate + ts - 1) / ts).unwrap_or(i64::MAX);
|
||||
if codec == CodecId::Opus {
|
||||
let packets = stamps.into_iter().enumerate().map(|(i, stamp)| {
|
||||
let samples = f.read_sample(source, track_idx, i).ok().and_then(|p| opus_packet_samples(&p)).map(i64::from);
|
||||
PacketTime { stamp, stamped: true, samples }
|
||||
});
|
||||
return contiguous_starts(packets, unit.saturating_mul(2).saturating_add(1));
|
||||
}
|
||||
let Some(frame) = fixed_samples else {
|
||||
return stamps;
|
||||
};
|
||||
let tolerance = (frame / 2).max(unit.saturating_mul(2).saturating_add(1));
|
||||
let packets = stamps.into_iter().map(|stamp| PacketTime { stamp, stamped: true, samples: Some(frame) });
|
||||
contiguous_starts(packets, tolerance)
|
||||
}
|
||||
ContainerFile::Matroska(f) => {
|
||||
let Some(track) = f.tracks.get(track_idx) else { return Vec::new() };
|
||||
// Indexed PTS already subtracts CodecDelay rounded to the container clock.
|
||||
// Translate between that clock and the warm path's whole-frame delay.
|
||||
let tick_ns = i128::from(f.info.timestamp_scale.max(1));
|
||||
let delay_ns = i128::from(track.codec_delay_ns);
|
||||
let indexed_delay_ns = ((delay_ns + tick_ns / 2) / tick_ns) * tick_ns;
|
||||
let delay_frames = audio_frames_from_ns(delay_ns, native_rate);
|
||||
let stamp_for = |pts: i64| -> i64 {
|
||||
let ns = i128::from(track.ticks_to_ns(pts)) + indexed_delay_ns;
|
||||
i64::try_from(audio_frames_from_ns(ns, native_rate) - delay_frames).unwrap_or(0)
|
||||
};
|
||||
let tick_samples = i64::try_from(i128::from(track.ticks_to_ns(2)) * rate / 1_000_000_000).unwrap_or(i64::MAX).saturating_add(1);
|
||||
if codec == CodecId::Opus {
|
||||
let packets = track.samples.iter().enumerate().map(|(i, s)| {
|
||||
let samples = f.read_sample(source, track_idx, i).ok().and_then(|p| opus_packet_samples(&p)).map(i64::from);
|
||||
PacketTime { stamp: stamp_for(s.pts), stamped: s.lace == 0 || s.duration > 0, samples }
|
||||
});
|
||||
return contiguous_starts(packets, tick_samples);
|
||||
}
|
||||
let Some(frame) = fixed_samples else {
|
||||
return track.samples.iter().map(|s| stamp_for(s.pts)).collect();
|
||||
};
|
||||
let packets = track.samples.iter().map(|s| PacketTime { stamp: stamp_for(s.pts), stamped: s.lace == 0 || s.duration > 0, samples: Some(frame) });
|
||||
contiguous_starts(packets, (frame / 2).max(tick_samples))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -204,7 +325,7 @@ where
|
||||
{
|
||||
let mut audio_tracks = Vec::new();
|
||||
let mut audio_infos = Vec::new();
|
||||
for (idx, sample_count, params, codec_delay_ns) in candidates {
|
||||
for (idx, sample_count, params, _codec_delay_ns) in candidates {
|
||||
if audio_tracks.len() >= MAX_AUDIO_STREAMS {
|
||||
break;
|
||||
}
|
||||
@@ -215,10 +336,20 @@ where
|
||||
lookahead.push(pkt);
|
||||
}
|
||||
let refs: Vec<&[u8]> = lookahead.iter().map(Vec::as_slice).collect();
|
||||
let dec = AudioTrackDecoder::new(¶ms, &refs).ok()?;
|
||||
let mut dec = AudioTrackDecoder::new(¶ms, &refs).ok()?;
|
||||
let native_rate = dec.sample_rate().max(1);
|
||||
let channels = u32::from(dec.layout().channels()).max(1);
|
||||
let delay_samples = i64::try_from(audio_frames_from_ns(i128::from(codec_delay_ns), native_rate)).unwrap_or(0);
|
||||
let codec = dec.codec();
|
||||
let fixed_samples = fixed_packet_samples(&mut dec, codec, &lookahead);
|
||||
let audio_starts = compute_audio_starts(container, source, idx, codec, native_rate, fixed_samples);
|
||||
let seek_preroll_samples = match container {
|
||||
ContainerFile::Matroska(f) if codec == CodecId::Opus => {
|
||||
let preroll_ns = i128::from(f.tracks[idx].seek_pre_roll_ns.max(80_000_000));
|
||||
i64::try_from(preroll_ns * i128::from(native_rate) / 1_000_000_000).unwrap_or(0)
|
||||
}
|
||||
_ => 0,
|
||||
};
|
||||
let initial_pos = audio_starts.first().copied().unwrap_or(0);
|
||||
audio_infos.push(AudioStreamInfo {
|
||||
sample_rate: native_rate,
|
||||
channels,
|
||||
@@ -228,7 +359,9 @@ where
|
||||
audio_tracks.push(AudioTrackEntry {
|
||||
track_idx: idx,
|
||||
native_rate,
|
||||
state: Mutex::new(AudioTrackState { decoder: dec, next_sample: 0, decoded_pos: -delay_samples, buf: Vec::new() }),
|
||||
audio_starts,
|
||||
seek_preroll_samples,
|
||||
state: Mutex::new(AudioTrackState { decoder: dec, next_sample: 0, decoded_pos: initial_pos, buf: Vec::new() }),
|
||||
});
|
||||
}
|
||||
Some((audio_tracks, audio_infos))
|
||||
@@ -509,6 +642,8 @@ struct AudioTrackState {
|
||||
struct AudioTrackEntry {
|
||||
track_idx: usize,
|
||||
native_rate: u32,
|
||||
audio_starts: Vec<i64>,
|
||||
seek_preroll_samples: i64,
|
||||
state: Mutex<AudioTrackState>,
|
||||
}
|
||||
|
||||
@@ -622,7 +757,18 @@ impl MediaSource for CraftMediaSource {
|
||||
};
|
||||
|
||||
let mut state = entry.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let decoded = decode_audio_range(&self.file, &*self.source, entry.track_idx, native_rate, channels_count, &mut state, native_start, native_frames)?;
|
||||
let decoded = decode_audio_range(
|
||||
&self.file,
|
||||
&*self.source,
|
||||
entry.track_idx,
|
||||
native_rate,
|
||||
channels_count,
|
||||
&entry.audio_starts,
|
||||
entry.seek_preroll_samples,
|
||||
&mut state,
|
||||
native_start,
|
||||
native_frames,
|
||||
)?;
|
||||
|
||||
if sample_rate == native_rate {
|
||||
return Ok(AudioBuffer { sample_rate, channels: decoded });
|
||||
@@ -655,6 +801,8 @@ fn decode_audio_range(
|
||||
track_idx: usize,
|
||||
native_rate: u32,
|
||||
channels_count: usize,
|
||||
audio_starts: &[i64],
|
||||
seek_preroll_samples: i64,
|
||||
state: &mut AudioTrackState,
|
||||
start: i64,
|
||||
frames: usize,
|
||||
@@ -668,7 +816,7 @@ fn decode_audio_range(
|
||||
let buf_len = state.buf.first().map_or(0, Vec::len) as i64;
|
||||
let buf_end = state.decoded_pos.saturating_add(buf_len);
|
||||
if start < state.decoded_pos || start > buf_end + i64::from(native_rate) * 2 {
|
||||
let (seek_idx, decoded_pos) = file.audio_seek_info(track_idx, start, native_rate);
|
||||
let (seek_idx, decoded_pos) = ContainerFile::audio_seek_info(audio_starts, seek_preroll_samples, start);
|
||||
state.decoder.reset();
|
||||
state.next_sample = seek_idx;
|
||||
state.decoded_pos = decoded_pos;
|
||||
@@ -687,9 +835,38 @@ fn decode_audio_range(
|
||||
return Err(MediaError::Cancelled);
|
||||
}
|
||||
let idx = state.next_sample;
|
||||
let pk_start = audio_starts.get(idx).copied().unwrap_or_else(|| state.decoded_pos + state.buf.first().map_or(0, Vec::len) as i64);
|
||||
if pk_start >= end {
|
||||
break;
|
||||
}
|
||||
let pkt = file.read_sample(source, track_idx, idx)?;
|
||||
state.next_sample = idx + 1;
|
||||
let block = state.decoder.decode(&pkt).map_err(|e| MediaError::Decode(e.to_string()))?;
|
||||
let buf_len = state.buf.first().map_or(0, Vec::len) as i64;
|
||||
if buf_len == 0 {
|
||||
state.decoded_pos = pk_start;
|
||||
} else {
|
||||
let buf_end = state.decoded_pos.saturating_add(buf_len);
|
||||
if pk_start > buf_end {
|
||||
let gap = usize::try_from(pk_start - buf_end).unwrap_or(0);
|
||||
for ch in &mut state.buf {
|
||||
ch.resize(ch.len() + gap, 0.0);
|
||||
}
|
||||
} else if pk_start < buf_end {
|
||||
if pk_start <= state.decoded_pos {
|
||||
for ch in &mut state.buf {
|
||||
ch.clear();
|
||||
}
|
||||
state.decoded_pos = pk_start;
|
||||
} else {
|
||||
let overlap = usize::try_from(buf_end - pk_start).unwrap_or(0);
|
||||
for ch in &mut state.buf {
|
||||
let keep = ch.len().saturating_sub(overlap);
|
||||
ch.truncate(keep);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (c, ch) in state.buf.iter_mut().enumerate() {
|
||||
if let Some(src_ch) = block.channels.get(c) {
|
||||
ch.extend_from_slice(src_ch);
|
||||
@@ -1318,4 +1495,61 @@ mod tests {
|
||||
assert_eq!(out.map(|f| f.par), Some((3, 4)));
|
||||
}
|
||||
}
|
||||
|
||||
/// 1024-sample frames at 48 kHz stamped to the nearest millisecond, then back in samples.
|
||||
fn ms_stamps(count: i64) -> Vec<i64> {
|
||||
(0..count).map(|k| (k * 1024 * 1000 + 24_000) / 48_000 * 48).collect()
|
||||
}
|
||||
|
||||
fn timed(stamps: &[i64], samples: Option<i64>) -> Vec<PacketTime> {
|
||||
stamps.iter().map(|&stamp| PacketTime { stamp, stamped: true, samples }).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rounded_stamps_give_back_to_back_packets() {
|
||||
let stamps = ms_stamps(200);
|
||||
assert_eq!(&stamps[..4], [0, 1008, 2064, 3072]);
|
||||
let starts = contiguous_starts(timed(&stamps, Some(1024)), 512);
|
||||
assert_eq!(starts, (0..200).map(|k| k * 1024).collect::<Vec<_>>());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn real_gaps_and_overlaps_resynchronise() {
|
||||
// A 100 ms gap after packet 3, then a packet stamped 600 samples early.
|
||||
let mut stamps: Vec<i64> = (0..8).map(|k| k * 1024).collect();
|
||||
for s in &mut stamps[4..] {
|
||||
*s += 4800;
|
||||
}
|
||||
stamps[7] -= 600;
|
||||
let starts = contiguous_starts(timed(&stamps, Some(1024)), 512);
|
||||
assert_eq!(starts, [0, 1024, 2048, 3072, 8896, 9920, 10944, 11368]);
|
||||
// Within the tolerance: runs on.
|
||||
stamps[7] += 200;
|
||||
assert_eq!(contiguous_starts(timed(&stamps, Some(1024)), 512)[7], 11968);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unstamped_packets_follow_on_and_unknown_lengths_resynchronise() {
|
||||
let packets = [0, 1024, 1024, 1024, 4096].map(|stamp| PacketTime { stamp, stamped: true, samples: Some(1024) });
|
||||
let mut laced = packets;
|
||||
laced[2].stamped = false;
|
||||
laced[3].stamped = false;
|
||||
assert_eq!(contiguous_starts(laced, 512), [0, 1024, 2048, 3072, 4096]);
|
||||
let mut unknown = timed(&[0, 1024, 2100, 3124], Some(1024));
|
||||
unknown[1].samples = None;
|
||||
assert_eq!(contiguous_starts(unknown, 512), [0, 1024, 2100, 3124]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hostile_values_stay_finite() {
|
||||
let packets = [
|
||||
PacketTime { stamp: i64::MAX, stamped: true, samples: Some(i64::MAX) },
|
||||
PacketTime { stamp: i64::MIN, stamped: false, samples: Some(-5) },
|
||||
PacketTime { stamp: 0, stamped: true, samples: Some(i64::MIN) },
|
||||
PacketTime { stamp: 7, stamped: true, samples: None },
|
||||
];
|
||||
assert_eq!(contiguous_starts(packets, i64::MIN).len(), 4);
|
||||
assert_eq!(contiguous_starts(packets, i64::MAX).len(), 4);
|
||||
assert!(contiguous_starts([], 512).is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1727,3 +1727,165 @@ fn audio_only_isobmff_and_matroska_route_through_craft_audio() {
|
||||
assert!((rms_seek - 0.0625).abs() < 0.02, "{file} seek rms {rms_seek}");
|
||||
}
|
||||
}
|
||||
|
||||
fn read_audio_sequential(pool: &MediaPool, f: &Footage, total_frames: usize, chunk_frames: usize, rate: u32) -> Vec<f32> {
|
||||
let mut out = Vec::with_capacity(total_frames * 2);
|
||||
let mut pos = 0usize;
|
||||
while pos < total_frames {
|
||||
let len = chunk_frames.min(total_frames - pos);
|
||||
let t = Tick::from_units(pos as i64, i64::from(rate));
|
||||
out.extend_from_slice(&pool.audio_samples(f, t, len, rate));
|
||||
pos += len;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn slice_rms(samples: &[f32]) -> f32 {
|
||||
if samples.is_empty() {
|
||||
return 0.0;
|
||||
}
|
||||
(samples.iter().map(|v| v * v).sum::<f32>() / samples.len() as f32).sqrt()
|
||||
}
|
||||
|
||||
fn assert_samples_close(a: &[f32], b: &[f32], tol: f32, ctx: &str) {
|
||||
assert_eq!(a.len(), b.len(), "{ctx}: length mismatch");
|
||||
let max_diff = a.iter().zip(b).map(|(x, y)| (x - y).abs()).fold(0.0f32, f32::max);
|
||||
assert!(max_diff <= tol, "{ctx}: max_diff {max_diff} > {tol}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mkv_and_mp4_aac_play_continuously_without_crackle_and_seek_matches_sequential() {
|
||||
let Some(mp4_path) =
|
||||
try_ffmpeg_fixture("aac_1546_pair.mp4", &["-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000:duration=2", "-c:a", "aac", "-b:a", "128k"])
|
||||
else {
|
||||
return;
|
||||
};
|
||||
let mp4_str = mp4_path.to_string_lossy().into_owned();
|
||||
let Some(mkv_path) = try_ffmpeg_fixture("aac_1546_pair.mkv", &["-i", &mp4_str, "-c:a", "copy"]) else {
|
||||
return;
|
||||
};
|
||||
|
||||
let mp4 = probe(&mp4_path).expect("probe mp4");
|
||||
let mkv = probe(&mkv_path).expect("probe mkv");
|
||||
let rate = 48_000u32;
|
||||
let total_frames = 80_000usize;
|
||||
|
||||
let mp4_seq = read_audio_sequential(&MediaPool::new(), &mp4, total_frames, 4096, rate);
|
||||
let mkv_seq = read_audio_sequential(&MediaPool::new(), &mkv, total_frames, 4096, rate);
|
||||
|
||||
// After the initial priming window (0.1s), MKV and MP4 must match without crackle.
|
||||
let skip = 4800 * 2;
|
||||
assert_samples_close(&mp4_seq[skip..], &mkv_seq[skip..], 0.01, "mkv vs mp4 sequential");
|
||||
|
||||
// Cold seeks on fresh MediaPools at t=0.5s and t=1.0s must match sequential playback.
|
||||
for seek_frame in [24_000usize, 48_000usize] {
|
||||
let t = Tick::from_units(seek_frame as i64, i64::from(rate));
|
||||
let cold = MediaPool::new().audio_samples(&mkv, t, 4800, rate);
|
||||
let start = seek_frame * 2;
|
||||
assert_samples_close(&cold, &mkv_seq[start..start + 4800 * 2], 1e-4, "mkv cold seek vs sequential");
|
||||
}
|
||||
|
||||
// Backward seek after reading near the end must also match sequential playback.
|
||||
let back_pool = MediaPool::new();
|
||||
let _ = back_pool.audio_samples(&mkv, Tick::from_units(60_000, i64::from(rate)), 4800, rate);
|
||||
let backward = back_pool.audio_samples(&mkv, Tick::from_units(24_000, i64::from(rate)), 4800, rate);
|
||||
assert_samples_close(&backward, &mkv_seq[24_000 * 2..24_000 * 2 + 4800 * 2], 1e-4, "mkv backward seek vs sequential");
|
||||
}
|
||||
|
||||
fn build_gapped_aac_mkv(out_path: &Path, gap_at_pkt: usize, gap_ns: i64, start_offset_ns: i64) -> Option<()> {
|
||||
let mp4_path =
|
||||
try_ffmpeg_fixture("aac_1546_pair.mp4", &["-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000:duration=2", "-c:a", "aac", "-b:a", "128k"])?;
|
||||
let bytes: std::sync::Arc<[u8]> = std::sync::Arc::from(std::fs::read(&mp4_path).ok()?);
|
||||
let mp4 = craft_media::isobmff::open(bytes.clone()).ok()?;
|
||||
let track_idx = mp4.tracks.iter().position(|t| t.kind == craft_media::isobmff::TrackKind::Audio)?;
|
||||
let track = &mp4.tracks[track_idx];
|
||||
let asc = match &track.entries.first()?.codec {
|
||||
craft_media::isobmff::CodecConfig::Aac(a) => a.asc.clone(),
|
||||
_ => return None,
|
||||
};
|
||||
|
||||
let mut spec = craft_media::matroska::TrackSpec::new(craft_media::matroska::TrackKind::Audio, "A_AAC");
|
||||
spec.codec_private = asc;
|
||||
spec.audio = Some((48_000.0, u64::from(track.audio().map_or(2, |a| a.channels)), None));
|
||||
let mut writer = craft_media::matroska::MkvWriter::new(std::io::Cursor::new(Vec::new()), vec![spec], craft_media::matroska::MuxOptions::default()).ok()?;
|
||||
|
||||
let pkt_count = track.samples.len().min(60);
|
||||
for i in 0..pkt_count {
|
||||
let pkt = mp4.read_sample(&*bytes, track_idx, i).ok()?;
|
||||
let base_ns = (i as i64 * 1024 * 1_000_000_000) / 48_000;
|
||||
let extra = if i >= gap_at_pkt { gap_ns } else { 0 };
|
||||
writer.write_frame(0, start_offset_ns + base_ns + extra, true, &pkt, None).ok()?;
|
||||
}
|
||||
let mkv_bytes = writer.finish().ok()?.into_inner();
|
||||
std::fs::write(out_path, mkv_bytes).ok()?;
|
||||
Some(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audio_real_silence_gap_honored_on_sequential_and_seek() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let mkv_path = dir.path().join("real_gap.mkv");
|
||||
// 24 packets (~0.512s) at 0, then a 500 ms silence gap before packet 24.
|
||||
if build_gapped_aac_mkv(&mkv_path, 24, 500_000_000, 0).is_none() {
|
||||
return;
|
||||
}
|
||||
|
||||
let f = probe(&mkv_path).expect("probe gapped mkv");
|
||||
let rate = 48_000u32;
|
||||
let total_frames = 72_000usize;
|
||||
let seq = read_audio_sequential(&MediaPool::new(), &f, total_frames, 4096, rate);
|
||||
|
||||
// Pre-gap tone (0.1s..0.4s), silence gap (0.55s..0.95s), post-gap tone (1.1s..1.4s).
|
||||
assert!(slice_rms(&seq[4800 * 2..19_200 * 2]) > 0.04, "pre-gap tone expected");
|
||||
assert!(slice_rms(&seq[26_400 * 2..45_600 * 2]) < 1e-4, "silence gap expected");
|
||||
assert!(slice_rms(&seq[52_800 * 2..67_200 * 2]) > 0.04, "post-gap tone expected");
|
||||
|
||||
for seek_frame in [9_600usize, 30_000usize, 54_000usize] {
|
||||
let t = Tick::from_units(seek_frame as i64, i64::from(rate));
|
||||
let cold = MediaPool::new().audio_samples(&f, t, 4800, rate);
|
||||
let start = seek_frame * 2;
|
||||
assert_samples_close(&cold, &seq[start..start + 4800 * 2], 1e-4, "gap cold seek vs sequential");
|
||||
}
|
||||
|
||||
let back_pool = MediaPool::new();
|
||||
let _ = back_pool.audio_samples(&f, Tick::from_units(54_000, i64::from(rate)), 4800, rate);
|
||||
for seek_frame in [30_000usize, 9_600usize] {
|
||||
let backward = back_pool.audio_samples(&f, Tick::from_units(seek_frame as i64, i64::from(rate)), 4800, rate);
|
||||
let start = seek_frame * 2;
|
||||
assert_samples_close(&backward, &seq[start..start + 4800 * 2], 1e-4, "gap backward seek vs sequential");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audio_delayed_first_packet_honored_on_sequential_and_seek() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let mkv_path = dir.path().join("delayed_first.mkv");
|
||||
// First packet delayed by 500 ms.
|
||||
if build_gapped_aac_mkv(&mkv_path, usize::MAX, 0, 500_000_000).is_none() {
|
||||
return;
|
||||
}
|
||||
|
||||
let f = probe(&mkv_path).expect("probe delayed mkv");
|
||||
let rate = 48_000u32;
|
||||
let total_frames = 57_600usize;
|
||||
let seq = read_audio_sequential(&MediaPool::new(), &f, total_frames, 4096, rate);
|
||||
|
||||
// Leading silence (0..0.45s) and delayed tone (0.6s..1.0s).
|
||||
assert!(slice_rms(&seq[..21_600 * 2]) < 1e-4, "leading silence expected");
|
||||
assert!(slice_rms(&seq[28_800 * 2..48_000 * 2]) > 0.04, "delayed tone expected");
|
||||
|
||||
for seek_frame in [4_800usize, 30_000usize] {
|
||||
let t = Tick::from_units(seek_frame as i64, i64::from(rate));
|
||||
let cold = MediaPool::new().audio_samples(&f, t, 4800, rate);
|
||||
let start = seek_frame * 2;
|
||||
assert_samples_close(&cold, &seq[start..start + 4800 * 2], 1e-4, "delayed cold seek vs sequential");
|
||||
}
|
||||
|
||||
let back_pool = MediaPool::new();
|
||||
let _ = back_pool.audio_samples(&f, Tick::from_units(36_000, i64::from(rate)), 4800, rate);
|
||||
for seek_frame in [30_000usize, 4_800usize] {
|
||||
let backward = back_pool.audio_samples(&f, Tick::from_units(seek_frame as i64, i64::from(rate)), 4800, rate);
|
||||
let start = seek_frame * 2;
|
||||
assert_samples_close(&backward, &seq[start..start + 4800 * 2], 1e-4, "delayed backward seek vs sequential");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user