media: use the shared reader for uncompressed RGB AVI (#4050)

This commit is contained in:
Terra854
2026-11-21 12:53:47 -05:00
committed by GitHub
parent fcf35f3f9f
commit 6e3459a8eb
3 changed files with 246 additions and 8 deletions
+51 -8
View File
@@ -25,9 +25,9 @@
//! - **AVI / MPEG-TS / MPEG-PS**:
//! - Shared (`craft-media::avi`, `craft-media::mpegts` + `craft-video` + `craft-audio`):
//! `H264`, `Hevc`, `Mpeg2Video`, `Av1`, `Vp9`, `Vp8`, `ProRes`, `Dnxhd`; audio `Aac`,
//! `Ac3`/`Eac3`, `Mp3`/`Mp2`/`Mp1`, `Pcm`.
//! `Ac3`/`Eac3`, `Mp3`/`Mp2`/`Mp1`, `Pcm`. AVI also accepts complete 24/32-bit `BI_RGB` streams.
//! - Fallback (`filmcraft-codecs`): unsupported codecs (`Mjpeg`, `Mpeg4Video`, `Ffv1`, `Theora`,
//! `Apv`, raw RGB AVI) or unsupported audio.
//! `Apv`, other raw AVI layouts) or unsupported audio.
//! - **Other containers & standalone audio**:
//! - Shared (`craft_audio::Decoder` with `decode-native`): standalone `WAV`, `AIFF`, `MP3`/`MP2`/`MP1`,
//! `AAC` (ADTS), `AC-3`, and `Ogg` (`Opus`).
@@ -53,6 +53,9 @@ use craft_time::{FrameRate, Tick};
use craft_video::frame::{ColorFamily, ColorTags, Plane, Samples, Subsampling, VideoFrame as DecodedVideoFrame};
use craft_video::{TrackDecoder, TrackDecoderOptions, VideoError, prores};
mod raw_avi;
use raw_avi::RawAvi;
/// Maximum worker threads per `TrackDecoder` instance.
const MAX_WORKERS: usize = 8;
@@ -955,7 +958,7 @@ fn open_isobmff(name: &str, source: SharedBytes) -> Option<Result<SharedSource>>
Err(e) => return Some(Err(MediaError::Decode(e.to_string()))),
};
let v_info = VideoStreamInfo { width, height, frame_rate, par, codec: codec_name, pixel_format, color, has_alpha, bitrate, hdr: None };
let v_state = Mutex::new(VideoState { decoder: video_decoder, next_decode_idx: 0, gop_start_idx: 0, flushed: false, decoded: Vec::new() });
let v_state = VideoState::packet(video_decoder);
(Some(v_info), Some(v_state), FramePresentation { container_color, par, rotation, aperture }, duration)
} else {
let duration = audio_candidates
@@ -1064,7 +1067,7 @@ fn open_matroska(name: &str, source: SharedBytes) -> Option<Result<SharedSource>
bitrate: None,
hdr: None,
};
let v_state = Mutex::new(VideoState { decoder: video_decoder, next_decode_idx: 0, gop_start_idx: 0, flushed: false, decoded: Vec::new() });
let v_state = VideoState::packet(video_decoder);
(Some(v_info), Some(v_state), container_color, par, rotation, duration)
} else {
let duration = audio_candidates.iter().map(|&(idx, _, _, _)| matroska_duration(&file, &file.tracks[idx])).max().unwrap_or(Tick::ZERO);
@@ -1299,7 +1302,7 @@ fn open_parsed_mxf(name: &str, source: SharedBytes, file: mxf::MxfFile) -> Optio
bitrate: None,
hdr: None,
};
let v_state = Mutex::new(VideoState { decoder: video_decoder, next_decode_idx: 0, gop_start_idx: 0, flushed: false, decoded: Vec::new() });
let v_state = VideoState::packet(video_decoder);
(Some(v_info), Some(v_state), FramePresentation { container_color, par, rotation: 0, aperture: None }, duration)
} else {
let duration = audio_tracks
@@ -1371,6 +1374,25 @@ fn open_avi(name: &str, source: SharedBytes) -> Option<Result<SharedSource>> {
return None;
}
if let Some(idx) = video_idx
&& let Some(raw) = RawAvi::new(&file.streams[idx])
{
// Shared AVI audio positions are relative to the first chunk. Keep the
// legacy path for a track with a separate stream start.
if audio_candidates.iter().any(|&(audio, _, _, _)| file.streams[audio].start != 0) {
return None;
}
let duration = audio_candidates
.iter()
.map(|&(audio, _, _, _)| {
let s = &file.streams[audio];
Tick::from_units(i64::from(s.length).saturating_mul(i64::from(s.scale.max(1))), i64::from(s.rate.max(1)))
})
.fold(raw.duration(file.streams[idx].chunks.len()), Tick::max);
let video = ContainerVideoPart { track_idx: idx, info: raw.info(&file.streams[idx]), state: Mutex::new(VideoState::RawAvi(raw)) };
return finish_container_source(name, "avi", source, ContainerFile::Avi(Box::new(file)), Some(video), duration, audio_candidates);
}
let (video, duration) = if let Some(v_idx) = video_idx {
let video_stream = &file.streams[v_idx];
let video_params = video_stream.codec_params();
@@ -1395,7 +1417,7 @@ fn open_avi(name: &str, source: SharedBytes) -> Option<Result<SharedSource>> {
let alpha = video_stream.chunks.first().and_then(|chunk| prores_has_alpha(chunk.size, || file.read_chunk(&*source, v_idx, 0).ok()));
info.has_alpha = alpha.unwrap_or(false);
}
let state = Mutex::new(VideoState { decoder: video_decoder, next_decode_idx: 0, gop_start_idx: 0, flushed: false, decoded: Vec::new() });
let state = VideoState::packet(video_decoder);
(Some(ContainerVideoPart { track_idx: v_idx, info, state }), duration)
} else {
let duration = audio_candidates
@@ -1516,7 +1538,7 @@ fn open_mpeg(name: &str, fmt: mpegts::Format, source: SharedBytes) -> Option<Res
indexed_span()
};
let info = default_video_info(decoded_dims.0, decoded_dims.1, frame_rate, &video_params);
let state = Mutex::new(VideoState { decoder: video_decoder, next_decode_idx: 0, gop_start_idx: 0, flushed: false, decoded: Vec::new() });
let state = VideoState::packet(video_decoder);
(Some(ContainerVideoPart { track_idx: v_idx, info, state }), duration)
} else {
let duration = audio_candidates
@@ -1535,7 +1557,18 @@ struct CachedFrame {
frame: Arc<VideoFrame>,
}
struct VideoState {
enum VideoState {
Packet(PacketVideoState),
RawAvi(RawAvi),
}
impl VideoState {
fn packet(decoder: TrackDecoder) -> Mutex<Self> {
Mutex::new(Self::Packet(PacketVideoState { decoder, next_decode_idx: 0, gop_start_idx: 0, flushed: false, decoded: Vec::new() }))
}
}
struct PacketVideoState {
decoder: TrackDecoder,
next_decode_idx: usize,
gop_start_idx: usize,
@@ -1709,6 +1742,13 @@ impl MediaSource for CraftMediaSource {
let want_draft = req.scale < 0.99;
let mut state = video_mutex.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
let state = match &mut *state {
VideoState::Packet(state) => state,
VideoState::RawAvi(raw) => {
let ContainerFile::Avi(file) = &self.file else { return Err(MediaError::NoStream("AVI video")) };
return raw.frame(file, &*self.source, video_idx, target_idx);
}
};
if let Some(hit) = state.decoded.iter().find(|c| c.pts == target_pts && c.draft == want_draft) {
return Ok(hit.frame.clone());
}
@@ -2626,6 +2666,9 @@ fn estimate_matroska_frame_rate(track: &matroska::Track) -> FrameRate {
FrameRate::FPS_30
}
#[cfg(all(test, any(unix, windows)))]
mod raw_avi_tests;
#[cfg(test)]
mod tests {
use super::*;
+96
View File
@@ -0,0 +1,96 @@
use std::sync::Arc;
use craft_color::ColorInfo;
use craft_frame::VideoFrame;
use craft_media::{ByteSource, avi};
use craft_media_source::{MediaError, Result, VideoStreamInfo};
use craft_time::{FrameRate, Tick};
use craft_video::raw_video::{self, RawFormat, RawPixels};
pub(super) struct RawAvi {
format: RawFormat,
width: u32,
height: u32,
bottom_up: bool,
payload_bytes: usize,
rate: FrameRate,
cached: Option<(usize, Arc<VideoFrame>)>,
}
impl RawAvi {
pub(super) fn new(stream: &avi::Stream) -> Option<Self> {
let bitmap = stream.video.as_ref()?;
let format = raw_video::RawFormat::from_avi(bitmap.compression, bitmap.bit_count)?;
if !matches!(format, RawFormat::Bgr24 | RawFormat::Bgrx32) || stream.start != 0 {
return None;
}
let width = u32::try_from(bitmap.width).ok()?;
let height = bitmap.height.unsigned_abs();
if !(1..=16_384).contains(&width) || !(1..=16_384).contains(&height) {
return None;
}
let stride = (u64::from(width) * u64::from(bitmap.bit_count / 8)).next_multiple_of(4);
let payload_bytes = stride * u64::from(height);
let decoded_bytes = u64::from(width) * u64::from(height) * 4;
// A request can hold its input and new picture while the old picture is cached.
if payload_bytes + 2 * decoded_bytes > craft_video::budget::DECODE_MEMORY_BUDGET {
return None;
}
// Empty AVI slots and partial chunks need the legacy reader's recovery policy.
if stream.chunks.iter().any(|chunk| u64::from(chunk.size) < payload_bytes) {
return None;
}
Some(Self {
format,
width,
height,
bottom_up: bitmap.height > 0,
payload_bytes: usize::try_from(payload_bytes).ok()?,
rate: FrameRate::new(i64::from(stream.rate), i64::from(stream.scale)),
cached: None,
})
}
pub(super) fn duration(&self, slots: usize) -> Tick {
self.rate.tick_of(i64::try_from(slots).unwrap_or(i64::MAX))
}
pub(super) fn info(&self, stream: &avi::Stream) -> VideoStreamInfo {
let bytes: u128 = stream.chunks.iter().map(|chunk| u128::from(chunk.size)).sum();
let bytes_per_slot = bytes as f64 / stream.chunks.len() as f64;
VideoStreamInfo {
width: self.width,
height: self.height,
frame_rate: self.rate,
par: (1, 1),
codec: self.format.label().to_string(),
pixel_format: String::new(),
color: ColorInfo::SRGB_FULL,
has_alpha: false,
bitrate: Some((bytes_per_slot * 8.0 * self.rate.as_f64()) as u64),
hdr: None,
}
}
pub(super) fn frame(&mut self, file: &avi::AviFile, source: &dyn ByteSource, stream: usize, index: usize) -> Result<Arc<VideoFrame>> {
if let Some((cached_index, frame)) = &self.cached
&& *cached_index == index
{
return Ok(frame.clone());
}
if filmcraft_media::cancel::cancelled() {
return Err(MediaError::Cancelled);
}
let packet = file.read_chunk_range(source, stream, index, 0, self.payload_bytes).map_err(|e| MediaError::Io(e.to_string()))?;
let pixels = raw_video::decode(self.format, self.width, self.height, self.bottom_up, &packet)
.map_err(|e| MediaError::Decode(format!("uncompressed AVI picture: {e:?}")))?;
let RawPixels::Rgba(pixels) = pixels else {
return Err(MediaError::Unsupported("unexpected uncompressed AVI pixel layout".into()));
};
let mut frame = VideoFrame::rgba8(self.width, self.height, pixels).map_err(|e| MediaError::Decode(e.to_string()))?;
frame.pts = self.rate.tick_of(i64::try_from(index).unwrap_or(i64::MAX));
let frame = Arc::new(frame);
self.cached = Some((index, frame.clone()));
Ok(frame)
}
}
@@ -0,0 +1,99 @@
use std::io::Cursor;
use std::sync::Arc;
use craft_color::ColorInfo;
use craft_frame::PixelData;
use craft_media::avi;
use craft_media_source::FrameRequest;
use craft_time::Tick;
use crate::stream::{self, MediaSource};
const WIDTH: u32 = 3;
const HEIGHT: u32 = 2;
const FPS: u32 = 25;
const AUDIO_RATE: u32 = 8_000;
const AUDIO_PER_FRAME: usize = (AUDIO_RATE / FPS) as usize;
fn rgba(frame: u8, x: u8, y: u8) -> [u8; 4] {
[16 + 30 * frame, 21 + 71 * y, 5 + 17 * x, 255]
}
fn bitmap(frame: u8, depth: u16, bottom_up: bool) -> Vec<u8> {
let stride = (WIDTH as usize * usize::from(depth / 8)).next_multiple_of(4);
let mut bytes = Vec::new();
for row in 0..HEIGHT as u8 {
let y = if bottom_up { HEIGHT as u8 - 1 - row } else { row };
for x in 0..WIDTH as u8 {
let [r, g, b, _] = rgba(frame, x, y);
bytes.extend([b, g, r]);
if depth == 32 {
bytes.push(43);
}
}
bytes.resize((usize::from(row) + 1) * stride, 111);
}
bytes
}
// Original in-memory fixture. The shared AVI muxer writes deterministic test pixels and PCM.
fn fixture(depth: u16, bottom_up: bool, sparse: bool) -> Result<Vec<u8>, avi::Error> {
let mut video = avi::VideoSpec::new([0; 4], WIDTH, HEIGHT, FPS, 1);
if !bottom_up {
video.format.height = -(HEIGHT as i32);
}
video.format.bit_count = depth;
video.format.size_image = bitmap(0, depth, bottom_up).len() as u32;
let audio = avi::AudioSpec::pcm(AUDIO_RATE, 1, 16);
let mut writer =
avi::AviWriter::new(Cursor::new(Vec::new()), vec![avi::StreamSpec::Video(video), avi::StreamSpec::Audio(audio)], avi::MuxOptions::default())?;
for i in 0..3u8 {
let picture = if sparse && i == 1 { Vec::new() } else { bitmap(i, depth, bottom_up) };
let sound = (1000 * (i16::from(i) + 1)).to_le_bytes().repeat(AUDIO_PER_FRAME);
writer.write_chunk(0, &picture, true)?;
writer.write_chunk(1, &sound, true)?;
}
Ok(writer.finish()?.into_inner())
}
#[test]
fn raw_rgb_avi_preserves_pixels_and_pcm_across_indexed_seeks() -> Result<(), Box<dyn std::error::Error>> {
let dir = tempfile::tempdir()?;
for (name, depth, bottom_up) in [("bottom-up.avi", 24, true), ("top-down.avi", 32, false)] {
let bytes: Arc<[u8]> = fixture(depth, bottom_up, false)?.into();
let path = dir.path().join(name);
std::fs::write(&path, &bytes)?;
let disk = stream::open_source(&path).ok_or("unrecognized AVI")?.0?;
let memory = stream::open_bytes(name, bytes)?;
for opened in [disk, memory] {
let MediaSource::Craft(source) = opened else { panic!("{name}: expected shared raw AVI route") };
let info = source.info();
let video = info.video.as_ref().ok_or("missing video")?;
assert_eq!((video.width, video.height, video.has_alpha), (WIDTH, HEIGHT, false));
assert_eq!(video.codec, format!("Uncompressed RGB {depth}-bit"));
assert_eq!(info.duration, Tick::from_units(3, i64::from(FPS)));
for i in [2u8, 0, 1] {
let time = Tick::from_units(2 * i64::from(i) + 1, 2 * i64::from(FPS));
let frame = source.video_frame(FrameRequest { time, scale: 1.0 })?;
assert_eq!(frame.pts, Tick::from_units(i64::from(i), i64::from(FPS)));
assert_eq!((frame.par, frame.color), ((1, 1), ColorInfo::SRGB_FULL));
let PixelData::Rgba8(pixels) = &frame.data else { panic!("expected 8-bit RGBA") };
let expected: Vec<u8> = (0..HEIGHT as u8).flat_map(|y| (0..WIDTH as u8).flat_map(move |x| rgba(i, x, y))).collect();
assert_eq!(pixels.as_slice(), expected);
}
let audio = source.audio(AUDIO_PER_FRAME as i64 - 2, 4, AUDIO_RATE)?;
assert_eq!(audio.channels, [vec![1000.0 / 32768.0, 1000.0 / 32768.0, 2000.0 / 32768.0, 2000.0 / 32768.0]]);
}
}
Ok(())
}
#[test]
fn raw_rgb_avi_with_dropped_slot_keeps_legacy_playback() -> Result<(), Box<dyn std::error::Error>> {
let opened = stream::open_bytes("sparse.avi", fixture(24, true, true)?.into())?;
let MediaSource::Film(source) = opened else { panic!("expected legacy AVI slot handling") };
let time = filmcraft_time::Tick::from_units(1, i64::from(FPS));
let frame = source.video_frame(filmcraft_media::FrameRequest { time, scale: 1.0 })?;
assert_eq!((frame.width, frame.height), (WIDTH, HEIGHT));
Ok(())
}