From 6e3459a8ebc88cce8362fece73ed44f657e2fb7f Mon Sep 17 00:00:00 2001 From: Terra854 Date: Sun, 22 Nov 2026 01:53:47 +0800 Subject: [PATCH] media: use the shared reader for uncompressed RGB AVI (#4050) --- crates/media/src/craft_source.rs | 59 +++++++++-- crates/media/src/craft_source/raw_avi.rs | 96 ++++++++++++++++++ .../media/src/craft_source/raw_avi_tests.rs | 99 +++++++++++++++++++ 3 files changed, 246 insertions(+), 8 deletions(-) create mode 100644 crates/media/src/craft_source/raw_avi.rs create mode 100644 crates/media/src/craft_source/raw_avi_tests.rs diff --git a/crates/media/src/craft_source.rs b/crates/media/src/craft_source.rs index 3512107f1..48a6b679e 100644 --- a/crates/media/src/craft_source.rs +++ b/crates/media/src/craft_source.rs @@ -25,9 +25,9 @@ //! - **AVI / MPEG-TS / MPEG-PS**: //! - Shared (`craft-media::avi`, `craft-media::mpegts` + `craft-video` + `craft-audio`): //! `H264`, `Hevc`, `Mpeg2Video`, `Av1`, `Vp9`, `Vp8`, `ProRes`, `Dnxhd`; audio `Aac`, -//! `Ac3`/`Eac3`, `Mp3`/`Mp2`/`Mp1`, `Pcm`. +//! `Ac3`/`Eac3`, `Mp3`/`Mp2`/`Mp1`, `Pcm`. AVI also accepts complete 24/32-bit `BI_RGB` streams. //! - Fallback (`filmcraft-codecs`): unsupported codecs (`Mjpeg`, `Mpeg4Video`, `Ffv1`, `Theora`, -//! `Apv`, raw RGB AVI) or unsupported audio. +//! `Apv`, other raw AVI layouts) or unsupported audio. //! - **Other containers & standalone audio**: //! - Shared (`craft_audio::Decoder` with `decode-native`): standalone `WAV`, `AIFF`, `MP3`/`MP2`/`MP1`, //! `AAC` (ADTS), `AC-3`, and `Ogg` (`Opus`). @@ -53,6 +53,9 @@ use craft_time::{FrameRate, Tick}; use craft_video::frame::{ColorFamily, ColorTags, Plane, Samples, Subsampling, VideoFrame as DecodedVideoFrame}; use craft_video::{TrackDecoder, TrackDecoderOptions, VideoError, prores}; +mod raw_avi; +use raw_avi::RawAvi; + /// Maximum worker threads per `TrackDecoder` instance. const MAX_WORKERS: usize = 8; @@ -955,7 +958,7 @@ fn open_isobmff(name: &str, source: SharedBytes) -> Option> Err(e) => return Some(Err(MediaError::Decode(e.to_string()))), }; let v_info = VideoStreamInfo { width, height, frame_rate, par, codec: codec_name, pixel_format, color, has_alpha, bitrate, hdr: None }; - let v_state = Mutex::new(VideoState { decoder: video_decoder, next_decode_idx: 0, gop_start_idx: 0, flushed: false, decoded: Vec::new() }); + let v_state = VideoState::packet(video_decoder); (Some(v_info), Some(v_state), FramePresentation { container_color, par, rotation, aperture }, duration) } else { let duration = audio_candidates @@ -1064,7 +1067,7 @@ fn open_matroska(name: &str, source: SharedBytes) -> Option bitrate: None, hdr: None, }; - let v_state = Mutex::new(VideoState { decoder: video_decoder, next_decode_idx: 0, gop_start_idx: 0, flushed: false, decoded: Vec::new() }); + let v_state = VideoState::packet(video_decoder); (Some(v_info), Some(v_state), container_color, par, rotation, duration) } else { let duration = audio_candidates.iter().map(|&(idx, _, _, _)| matroska_duration(&file, &file.tracks[idx])).max().unwrap_or(Tick::ZERO); @@ -1299,7 +1302,7 @@ fn open_parsed_mxf(name: &str, source: SharedBytes, file: mxf::MxfFile) -> Optio bitrate: None, hdr: None, }; - let v_state = Mutex::new(VideoState { decoder: video_decoder, next_decode_idx: 0, gop_start_idx: 0, flushed: false, decoded: Vec::new() }); + let v_state = VideoState::packet(video_decoder); (Some(v_info), Some(v_state), FramePresentation { container_color, par, rotation: 0, aperture: None }, duration) } else { let duration = audio_tracks @@ -1371,6 +1374,25 @@ fn open_avi(name: &str, source: SharedBytes) -> Option> { return None; } + if let Some(idx) = video_idx + && let Some(raw) = RawAvi::new(&file.streams[idx]) + { + // Shared AVI audio positions are relative to the first chunk. Keep the + // legacy path for a track with a separate stream start. + if audio_candidates.iter().any(|&(audio, _, _, _)| file.streams[audio].start != 0) { + return None; + } + let duration = audio_candidates + .iter() + .map(|&(audio, _, _, _)| { + let s = &file.streams[audio]; + Tick::from_units(i64::from(s.length).saturating_mul(i64::from(s.scale.max(1))), i64::from(s.rate.max(1))) + }) + .fold(raw.duration(file.streams[idx].chunks.len()), Tick::max); + let video = ContainerVideoPart { track_idx: idx, info: raw.info(&file.streams[idx]), state: Mutex::new(VideoState::RawAvi(raw)) }; + return finish_container_source(name, "avi", source, ContainerFile::Avi(Box::new(file)), Some(video), duration, audio_candidates); + } + let (video, duration) = if let Some(v_idx) = video_idx { let video_stream = &file.streams[v_idx]; let video_params = video_stream.codec_params(); @@ -1395,7 +1417,7 @@ fn open_avi(name: &str, source: SharedBytes) -> Option> { let alpha = video_stream.chunks.first().and_then(|chunk| prores_has_alpha(chunk.size, || file.read_chunk(&*source, v_idx, 0).ok())); info.has_alpha = alpha.unwrap_or(false); } - let state = Mutex::new(VideoState { decoder: video_decoder, next_decode_idx: 0, gop_start_idx: 0, flushed: false, decoded: Vec::new() }); + let state = VideoState::packet(video_decoder); (Some(ContainerVideoPart { track_idx: v_idx, info, state }), duration) } else { let duration = audio_candidates @@ -1516,7 +1538,7 @@ fn open_mpeg(name: &str, fmt: mpegts::Format, source: SharedBytes) -> Option, } -struct VideoState { +enum VideoState { + Packet(PacketVideoState), + RawAvi(RawAvi), +} + +impl VideoState { + fn packet(decoder: TrackDecoder) -> Mutex { + Mutex::new(Self::Packet(PacketVideoState { decoder, next_decode_idx: 0, gop_start_idx: 0, flushed: false, decoded: Vec::new() })) + } +} + +struct PacketVideoState { decoder: TrackDecoder, next_decode_idx: usize, gop_start_idx: usize, @@ -1709,6 +1742,13 @@ impl MediaSource for CraftMediaSource { let want_draft = req.scale < 0.99; let mut state = video_mutex.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let state = match &mut *state { + VideoState::Packet(state) => state, + VideoState::RawAvi(raw) => { + let ContainerFile::Avi(file) = &self.file else { return Err(MediaError::NoStream("AVI video")) }; + return raw.frame(file, &*self.source, video_idx, target_idx); + } + }; if let Some(hit) = state.decoded.iter().find(|c| c.pts == target_pts && c.draft == want_draft) { return Ok(hit.frame.clone()); } @@ -2626,6 +2666,9 @@ fn estimate_matroska_frame_rate(track: &matroska::Track) -> FrameRate { FrameRate::FPS_30 } +#[cfg(all(test, any(unix, windows)))] +mod raw_avi_tests; + #[cfg(test)] mod tests { use super::*; diff --git a/crates/media/src/craft_source/raw_avi.rs b/crates/media/src/craft_source/raw_avi.rs new file mode 100644 index 000000000..76dd8f517 --- /dev/null +++ b/crates/media/src/craft_source/raw_avi.rs @@ -0,0 +1,96 @@ +use std::sync::Arc; + +use craft_color::ColorInfo; +use craft_frame::VideoFrame; +use craft_media::{ByteSource, avi}; +use craft_media_source::{MediaError, Result, VideoStreamInfo}; +use craft_time::{FrameRate, Tick}; +use craft_video::raw_video::{self, RawFormat, RawPixels}; + +pub(super) struct RawAvi { + format: RawFormat, + width: u32, + height: u32, + bottom_up: bool, + payload_bytes: usize, + rate: FrameRate, + cached: Option<(usize, Arc)>, +} + +impl RawAvi { + pub(super) fn new(stream: &avi::Stream) -> Option { + let bitmap = stream.video.as_ref()?; + let format = raw_video::RawFormat::from_avi(bitmap.compression, bitmap.bit_count)?; + if !matches!(format, RawFormat::Bgr24 | RawFormat::Bgrx32) || stream.start != 0 { + return None; + } + let width = u32::try_from(bitmap.width).ok()?; + let height = bitmap.height.unsigned_abs(); + if !(1..=16_384).contains(&width) || !(1..=16_384).contains(&height) { + return None; + } + let stride = (u64::from(width) * u64::from(bitmap.bit_count / 8)).next_multiple_of(4); + let payload_bytes = stride * u64::from(height); + let decoded_bytes = u64::from(width) * u64::from(height) * 4; + // A request can hold its input and new picture while the old picture is cached. + if payload_bytes + 2 * decoded_bytes > craft_video::budget::DECODE_MEMORY_BUDGET { + return None; + } + // Empty AVI slots and partial chunks need the legacy reader's recovery policy. + if stream.chunks.iter().any(|chunk| u64::from(chunk.size) < payload_bytes) { + return None; + } + Some(Self { + format, + width, + height, + bottom_up: bitmap.height > 0, + payload_bytes: usize::try_from(payload_bytes).ok()?, + rate: FrameRate::new(i64::from(stream.rate), i64::from(stream.scale)), + cached: None, + }) + } + + pub(super) fn duration(&self, slots: usize) -> Tick { + self.rate.tick_of(i64::try_from(slots).unwrap_or(i64::MAX)) + } + + pub(super) fn info(&self, stream: &avi::Stream) -> VideoStreamInfo { + let bytes: u128 = stream.chunks.iter().map(|chunk| u128::from(chunk.size)).sum(); + let bytes_per_slot = bytes as f64 / stream.chunks.len() as f64; + VideoStreamInfo { + width: self.width, + height: self.height, + frame_rate: self.rate, + par: (1, 1), + codec: self.format.label().to_string(), + pixel_format: String::new(), + color: ColorInfo::SRGB_FULL, + has_alpha: false, + bitrate: Some((bytes_per_slot * 8.0 * self.rate.as_f64()) as u64), + hdr: None, + } + } + + pub(super) fn frame(&mut self, file: &avi::AviFile, source: &dyn ByteSource, stream: usize, index: usize) -> Result> { + if let Some((cached_index, frame)) = &self.cached + && *cached_index == index + { + return Ok(frame.clone()); + } + if filmcraft_media::cancel::cancelled() { + return Err(MediaError::Cancelled); + } + let packet = file.read_chunk_range(source, stream, index, 0, self.payload_bytes).map_err(|e| MediaError::Io(e.to_string()))?; + let pixels = raw_video::decode(self.format, self.width, self.height, self.bottom_up, &packet) + .map_err(|e| MediaError::Decode(format!("uncompressed AVI picture: {e:?}")))?; + let RawPixels::Rgba(pixels) = pixels else { + return Err(MediaError::Unsupported("unexpected uncompressed AVI pixel layout".into())); + }; + let mut frame = VideoFrame::rgba8(self.width, self.height, pixels).map_err(|e| MediaError::Decode(e.to_string()))?; + frame.pts = self.rate.tick_of(i64::try_from(index).unwrap_or(i64::MAX)); + let frame = Arc::new(frame); + self.cached = Some((index, frame.clone())); + Ok(frame) + } +} diff --git a/crates/media/src/craft_source/raw_avi_tests.rs b/crates/media/src/craft_source/raw_avi_tests.rs new file mode 100644 index 000000000..96ce62454 --- /dev/null +++ b/crates/media/src/craft_source/raw_avi_tests.rs @@ -0,0 +1,99 @@ +use std::io::Cursor; +use std::sync::Arc; + +use craft_color::ColorInfo; +use craft_frame::PixelData; +use craft_media::avi; +use craft_media_source::FrameRequest; +use craft_time::Tick; + +use crate::stream::{self, MediaSource}; + +const WIDTH: u32 = 3; +const HEIGHT: u32 = 2; +const FPS: u32 = 25; +const AUDIO_RATE: u32 = 8_000; +const AUDIO_PER_FRAME: usize = (AUDIO_RATE / FPS) as usize; + +fn rgba(frame: u8, x: u8, y: u8) -> [u8; 4] { + [16 + 30 * frame, 21 + 71 * y, 5 + 17 * x, 255] +} + +fn bitmap(frame: u8, depth: u16, bottom_up: bool) -> Vec { + let stride = (WIDTH as usize * usize::from(depth / 8)).next_multiple_of(4); + let mut bytes = Vec::new(); + for row in 0..HEIGHT as u8 { + let y = if bottom_up { HEIGHT as u8 - 1 - row } else { row }; + for x in 0..WIDTH as u8 { + let [r, g, b, _] = rgba(frame, x, y); + bytes.extend([b, g, r]); + if depth == 32 { + bytes.push(43); + } + } + bytes.resize((usize::from(row) + 1) * stride, 111); + } + bytes +} + +// Original in-memory fixture. The shared AVI muxer writes deterministic test pixels and PCM. +fn fixture(depth: u16, bottom_up: bool, sparse: bool) -> Result, avi::Error> { + let mut video = avi::VideoSpec::new([0; 4], WIDTH, HEIGHT, FPS, 1); + if !bottom_up { + video.format.height = -(HEIGHT as i32); + } + video.format.bit_count = depth; + video.format.size_image = bitmap(0, depth, bottom_up).len() as u32; + let audio = avi::AudioSpec::pcm(AUDIO_RATE, 1, 16); + let mut writer = + avi::AviWriter::new(Cursor::new(Vec::new()), vec![avi::StreamSpec::Video(video), avi::StreamSpec::Audio(audio)], avi::MuxOptions::default())?; + for i in 0..3u8 { + let picture = if sparse && i == 1 { Vec::new() } else { bitmap(i, depth, bottom_up) }; + let sound = (1000 * (i16::from(i) + 1)).to_le_bytes().repeat(AUDIO_PER_FRAME); + writer.write_chunk(0, &picture, true)?; + writer.write_chunk(1, &sound, true)?; + } + Ok(writer.finish()?.into_inner()) +} + +#[test] +fn raw_rgb_avi_preserves_pixels_and_pcm_across_indexed_seeks() -> Result<(), Box> { + let dir = tempfile::tempdir()?; + for (name, depth, bottom_up) in [("bottom-up.avi", 24, true), ("top-down.avi", 32, false)] { + let bytes: Arc<[u8]> = fixture(depth, bottom_up, false)?.into(); + let path = dir.path().join(name); + std::fs::write(&path, &bytes)?; + let disk = stream::open_source(&path).ok_or("unrecognized AVI")?.0?; + let memory = stream::open_bytes(name, bytes)?; + for opened in [disk, memory] { + let MediaSource::Craft(source) = opened else { panic!("{name}: expected shared raw AVI route") }; + let info = source.info(); + let video = info.video.as_ref().ok_or("missing video")?; + assert_eq!((video.width, video.height, video.has_alpha), (WIDTH, HEIGHT, false)); + assert_eq!(video.codec, format!("Uncompressed RGB {depth}-bit")); + assert_eq!(info.duration, Tick::from_units(3, i64::from(FPS))); + for i in [2u8, 0, 1] { + let time = Tick::from_units(2 * i64::from(i) + 1, 2 * i64::from(FPS)); + let frame = source.video_frame(FrameRequest { time, scale: 1.0 })?; + assert_eq!(frame.pts, Tick::from_units(i64::from(i), i64::from(FPS))); + assert_eq!((frame.par, frame.color), ((1, 1), ColorInfo::SRGB_FULL)); + let PixelData::Rgba8(pixels) = &frame.data else { panic!("expected 8-bit RGBA") }; + let expected: Vec = (0..HEIGHT as u8).flat_map(|y| (0..WIDTH as u8).flat_map(move |x| rgba(i, x, y))).collect(); + assert_eq!(pixels.as_slice(), expected); + } + let audio = source.audio(AUDIO_PER_FRAME as i64 - 2, 4, AUDIO_RATE)?; + assert_eq!(audio.channels, [vec![1000.0 / 32768.0, 1000.0 / 32768.0, 2000.0 / 32768.0, 2000.0 / 32768.0]]); + } + } + Ok(()) +} + +#[test] +fn raw_rgb_avi_with_dropped_slot_keeps_legacy_playback() -> Result<(), Box> { + let opened = stream::open_bytes("sparse.avi", fixture(24, true, true)?.into())?; + let MediaSource::Film(source) = opened else { panic!("expected legacy AVI slot handling") }; + let time = filmcraft_time::Tick::from_units(1, i64::from(FPS)); + let frame = source.video_frame(filmcraft_media::FrameRequest { time, scale: 1.0 })?; + assert_eq!((frame.width, frame.height), (WIDTH, HEIGHT)); + Ok(()) +}