mirror of
https://github.com/storytold/effectcraft.git
synced 2026-11-21 20:38:55 +00:00
fix(media): silence reads past the MOV audio edit end (#4024)
* Stop presenting MOV audio at the edit end * Credit Terra854 in EffectCraft About
This commit is contained in:
@@ -143,3 +143,6 @@ sync_github_name = true
|
||||
real_name = "Nicola Falcier"
|
||||
display_name = "Nicola Falcier"
|
||||
sync_github_name = false
|
||||
|
||||
[people.Terra854]
|
||||
display_name = "Terra854"
|
||||
|
||||
@@ -293,19 +293,23 @@ impl ContainerFile {
|
||||
(seek_idx, decoded_pos)
|
||||
}
|
||||
|
||||
fn audio_silent_prefix(&self, track_idx: usize, start: i64, frames: usize, sample_rate: u32) -> usize {
|
||||
let Self::Isobmff(file) = self else { return 0 };
|
||||
let Some((leading_duration, media_time)) = file.tracks.get(track_idx).and_then(|track| unit_rate_media_edit(&track.edits)) else {
|
||||
return 0;
|
||||
fn audio_edit_range(&self, track_idx: usize, start: i64, frames: usize, sample_rate: u32) -> std::ops::Range<usize> {
|
||||
let Self::Isobmff(file) = self else { return 0..frames };
|
||||
let Some((leading_duration, media)) = file.tracks.get(track_idx).and_then(|track| unit_rate_media_edit(&track.edits)) else {
|
||||
return 0..frames;
|
||||
};
|
||||
if leading_duration == 0 || media_time == 0 || file.timescale == 0 {
|
||||
return 0;
|
||||
if file.timescale == 0 {
|
||||
return 0..frames;
|
||||
}
|
||||
|
||||
// The net packet offset includes the trim and can put decoder preroll inside
|
||||
// the empty edit. Use the empty edit's duration on the output clock instead.
|
||||
let first_presented = (u128::from(leading_duration) * u128::from(sample_rate)).div_ceil(u128::from(file.timescale)) as i128;
|
||||
usize::try_from((first_presented - i128::from(start)).max(0)).unwrap_or(usize::MAX).min(frames)
|
||||
// Packet positions include preroll. The edit's movie-clock boundaries
|
||||
// determine which samples are presented, with the end excluded.
|
||||
let offset = |ticks: u128| {
|
||||
let frame = (ticks * u128::from(sample_rate)).div_ceil(u128::from(file.timescale)) as i128;
|
||||
usize::try_from((frame - i128::from(start)).max(0)).unwrap_or(usize::MAX).min(frames)
|
||||
};
|
||||
let leading = u128::from(leading_duration);
|
||||
offset(leading)..offset(leading + u128::from(media.segment_duration))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -690,14 +694,14 @@ fn clean_aperture_region(aperture: isobmff::CleanAperture, (width, height): (u32
|
||||
}
|
||||
|
||||
/// A single unit-rate media span, optionally preceded by one empty span.
|
||||
fn unit_rate_media_edit(edits: &[isobmff::Edit]) -> Option<(u64, i64)> {
|
||||
fn unit_rate_media_edit(edits: &[isobmff::Edit]) -> Option<(u64, &isobmff::Edit)> {
|
||||
const UNIT_RATE: i32 = 0x0001_0000;
|
||||
let (leading_duration, media) = match edits {
|
||||
[media] => (0, media),
|
||||
[empty, media] if empty.media_time == -1 && empty.segment_duration > 0 && empty.media_rate == UNIT_RATE => (empty.segment_duration, media),
|
||||
_ => return None,
|
||||
};
|
||||
(media.media_time >= 0 && media.segment_duration != 0 && media.media_rate == UNIT_RATE).then_some((leading_duration, media.media_time))
|
||||
(media.media_time >= 0 && media.segment_duration != 0 && media.media_rate == UNIT_RATE).then_some((leading_duration, media))
|
||||
}
|
||||
|
||||
/// A scalar EDL origin cannot describe dwell, disjoint edits or respeeding.
|
||||
@@ -705,7 +709,7 @@ fn continuous_edit_mapping(track: &isobmff::Track, movie_timescale: u32) -> Opti
|
||||
if track.edits.is_empty() {
|
||||
return (track.edit_offset == 0).then_some((0, 0));
|
||||
}
|
||||
let (leading_duration, media_time) = unit_rate_media_edit(&track.edits)?;
|
||||
let (leading_duration, media) = unit_rate_media_edit(&track.edits)?;
|
||||
if movie_timescale == 0 || track.timescale == 0 {
|
||||
return None;
|
||||
}
|
||||
@@ -716,8 +720,8 @@ fn continuous_edit_mapping(track: &isobmff::Track, movie_timescale: u32) -> Opti
|
||||
if lead % movie_clock != 0 {
|
||||
return None;
|
||||
}
|
||||
let offset = i64::try_from(lead / movie_clock - i128::from(media_time)).ok()?;
|
||||
(offset == track.edit_offset).then_some((media_time, offset))
|
||||
let offset = i64::try_from(lead / movie_clock - i128::from(media.media_time)).ok()?;
|
||||
(offset == track.edit_offset).then_some((media.media_time, offset))
|
||||
}
|
||||
|
||||
/// The EDL origin is the timecode at presentation time zero, not the raw
|
||||
@@ -1446,15 +1450,20 @@ impl MediaSource for CraftMediaSource {
|
||||
return Err(MediaError::Decode("invalid audio sample rate 0".into()));
|
||||
}
|
||||
let channels_count = self.info.audio_streams.get(stream).map_or(2, |s| s.channels as usize);
|
||||
let silent_frames = self.file.audio_silent_prefix(entry.track_idx, start, frames, sample_rate);
|
||||
let audible_frames = frames - silent_frames;
|
||||
let edit_range = self.file.audio_edit_range(entry.track_idx, start, frames, sample_rate);
|
||||
let audible_frames = edit_range.len();
|
||||
if audible_frames == 0 {
|
||||
return Ok(AudioBuffer::new(sample_rate, channels_count as u16, frames));
|
||||
}
|
||||
|
||||
let audible_start = i64::try_from(i128::from(start) + silent_frames as i128).unwrap_or(i64::MAX);
|
||||
let audible_start = i64::try_from(i128::from(start) + edit_range.start as i128).unwrap_or(i64::MAX);
|
||||
let native_rate = entry.native_rate;
|
||||
let (native_start, native_frames) = native_audio_window(audible_start, audible_frames, native_rate, sample_rate);
|
||||
let (native_start, mut native_frames) = native_audio_window(audible_start, audible_frames, native_rate, sample_rate);
|
||||
if native_rate != sample_rate {
|
||||
// The resampler asks for lookahead. Do not interpolate with samples
|
||||
// that are stored in the track but lie beyond the edit.
|
||||
native_frames = self.file.audio_edit_range(entry.track_idx, native_start, native_frames, native_rate).end;
|
||||
}
|
||||
|
||||
let decoded = match &entry.backend {
|
||||
AudioBackend::Decoder { seek_preroll_samples, state: state_mutex } => {
|
||||
@@ -1503,10 +1512,10 @@ impl MediaSource for CraftMediaSource {
|
||||
|
||||
let mut channels =
|
||||
if sample_rate == native_rate { decoded } else { resample_audio(&decoded, channels_count, native_rate, sample_rate, audible_frames) };
|
||||
if silent_frames > 0 {
|
||||
if audible_frames < frames {
|
||||
for channel in &mut channels {
|
||||
channel.resize(frames, 0.0);
|
||||
channel.rotate_right(silent_frames);
|
||||
channel.rotate_right(edit_range.start);
|
||||
}
|
||||
}
|
||||
Ok(AudioBuffer { sample_rate, channels })
|
||||
@@ -2550,29 +2559,37 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// Original PCM generated in memory to exercise an empty edit followed by a trim.
|
||||
/// Original PCM generated in memory at the MOV writer's default movie timescale.
|
||||
fn edited_pcm_mov(rate: u32, samples: &[i16], edits: Vec<isobmff::Edit>) -> Vec<u8> {
|
||||
use isobmff::{Brand, Mp4Writer, PcmConfig, SampleEntry, TrackConfig, WriteSample, WriterOptions};
|
||||
|
||||
let mut writer = Mp4Writer::new(std::io::Cursor::new(Vec::new()), WriterOptions::new(Brand::Mov)).unwrap();
|
||||
let pcm = PcmConfig { bits: 16, float: false, big_endian: false, signed: true, channels: 1, sample_rate: f64::from(rate) };
|
||||
let mut config = TrackConfig::new(SampleEntry::pcm(pcm), rate);
|
||||
config.edits = edits;
|
||||
let track = writer.add_track(config).unwrap();
|
||||
for chunk in samples.chunks(256) {
|
||||
let data: Vec<u8> = chunk.iter().flat_map(|sample| sample.to_le_bytes()).collect();
|
||||
writer.write_sample(track, WriteSample { data: &data, duration: chunk.len() as u32, composition_offset: 0, is_sync: true }).unwrap();
|
||||
}
|
||||
writer.finish().unwrap().into_inner()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mov_pcm_trim_after_empty_edit_preserves_leading_silence() {
|
||||
use isobmff::{Brand, Edit, Mp4Writer, PcmConfig, SampleEntry, TrackConfig, WriteSample, WriterOptions};
|
||||
|
||||
const RATE: u32 = 48_000;
|
||||
const DELAY_MS: u64 = 100;
|
||||
const DELAY: i64 = 4_800;
|
||||
let samples: Vec<i16> = (0..12_000).map(|i| 1_000 + (i % 257) as i16 * 32).collect();
|
||||
for trim in [2_400i64, 4_800, 9_600] {
|
||||
let mut writer = Mp4Writer::new(std::io::Cursor::new(Vec::new()), WriterOptions::new(Brand::Mov)).unwrap();
|
||||
let pcm = PcmConfig { bits: 16, float: false, big_endian: false, signed: true, channels: 1, sample_rate: f64::from(RATE) };
|
||||
let mut config = TrackConfig::new(SampleEntry::pcm(pcm), RATE);
|
||||
config.edits = vec![
|
||||
Edit { segment_duration: DELAY_MS, media_time: -1, media_rate: 0x0001_0000 },
|
||||
Edit { segment_duration: 40, media_time: trim, media_rate: 0x0001_0000 },
|
||||
];
|
||||
let track = writer.add_track(config).unwrap();
|
||||
for chunk in samples.chunks(256) {
|
||||
let data: Vec<u8> = chunk.iter().flat_map(|sample| sample.to_le_bytes()).collect();
|
||||
writer.write_sample(track, WriteSample { data: &data, duration: chunk.len() as u32, composition_offset: 0, is_sync: true }).unwrap();
|
||||
}
|
||||
let bytes = writer.finish().unwrap().into_inner();
|
||||
let bytes = edited_pcm_mov(
|
||||
RATE,
|
||||
&samples,
|
||||
vec![
|
||||
isobmff::Edit { segment_duration: DELAY_MS, media_time: -1, media_rate: 0x0001_0000 },
|
||||
isobmff::Edit { segment_duration: 40, media_time: trim, media_rate: 0x0001_0000 },
|
||||
],
|
||||
);
|
||||
let file = isobmff::open(bytes.as_slice()).unwrap();
|
||||
assert_eq!(file.timescale, 1_000);
|
||||
assert_eq!(file.tracks[0].edit_offset, DELAY - trim);
|
||||
@@ -2589,6 +2606,38 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mov_pcm_audio_is_silent_after_the_edit_span() {
|
||||
const RATE: u32 = 48_000;
|
||||
const EDIT_MS: u64 = 41;
|
||||
const TRIM: i64 = 9_600;
|
||||
const EDIT_FRAMES: usize = 1_968;
|
||||
let samples: Vec<i16> = (0..12_000).map(|i| 1_000 + (i % 257) as i16 * 32).collect();
|
||||
let last = f32::from(samples[TRIM as usize + EDIT_FRAMES - 1]) / 32_768.0;
|
||||
let previous = f32::from(samples[TRIM as usize + EDIT_FRAMES - 2]) / 32_768.0;
|
||||
for delay_ms in [0, 100] {
|
||||
let mut edits = Vec::new();
|
||||
if delay_ms > 0 {
|
||||
edits.push(isobmff::Edit { segment_duration: delay_ms, media_time: -1, media_rate: 0x0001_0000 });
|
||||
}
|
||||
edits.push(isobmff::Edit { segment_duration: EDIT_MS, media_time: TRIM, media_rate: 0x0001_0000 });
|
||||
let bytes = edited_pcm_mov(RATE, &samples, edits);
|
||||
let source = open_bytes("edited-pcm.mov", bytes.into()).expect("shared MOV route").unwrap();
|
||||
let read = |start, frames, rate| source.audio(start, frames, rate).expect("direct shared PCM read").channels.remove(0);
|
||||
|
||||
let end = ((delay_ms + EDIT_MS) * 48) as i64;
|
||||
assert_eq!(read(end - 2, 4, RATE), [previous, last, 0.0, 0.0], "crossing the boundary, delay {delay_ms}");
|
||||
assert_eq!(read(end, 2, RATE), [0.0, 0.0], "read after the end");
|
||||
assert_eq!(read(end - 2, 2, RATE), [previous, last], "backward seek");
|
||||
assert_eq!(read(end * 2 - 4, 6, RATE * 2), [previous, (previous + last) / 2.0, last, last, 0.0, 0.0]);
|
||||
|
||||
let first_silent = ((delay_ms + EDIT_MS) * 44_100).div_ceil(1_000) as i64;
|
||||
let boundary = read(first_silent - 1, 2, 44_100);
|
||||
assert!(boundary[0] > 0.0, "sample before the fractional boundary");
|
||||
assert_eq!(boundary[1], 0.0, "sample after the fractional boundary");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn vui_overrides_container_color_and_falls_back_when_unspecified() {
|
||||
let container = color_from_isobmff(&isobmff::ColorInfo::Nclx { primaries: 9, transfer: 16, matrix: 9, full_range: false });
|
||||
|
||||
Reference in New Issue
Block a user