app: add support for multiple pipelines, and introduce a YUV shader. Migrate video AV1 to YUV 420 format.

bin/app/Cargo.lock

@@ -880,6 +880,7 @@ version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4"
 dependencies = [
+ "sha2",
  "tinyvec",
 ]
 

bin/app/src/app/schema/mod.rs

@@ -82,7 +82,7 @@ mod desktop_paths {
     use std::path::PathBuf;
 
     //pub const VID_PATH: &str = "assets/forest_1920x1080.ivf";
-    pub const VID_PATH: &str = "assets/forest2/forest_1920x1080.ivf.{frame}";
+    pub const VID_PATH: &str = "assets/forest_1920x1080.ivf";
     pub const VID_ASPECT_RATIO: f32 = 16. / 9.;
 
     pub fn get_chatdb_path() -> PathBuf {

bin/app/src/app/schema/test.rs

@@ -44,7 +44,7 @@ mod ui_consts {
 mod ui_consts {
     //pub const CHATDB_PATH: &str = "chatdb";
     //pub const KING_PATH: &str = "assets/king.png";
-    pub const VID_PATH: &str = "assets/forest2/forest_1920x1080.ivf.{frame}";
+    pub const VID_PATH: &str = "assets/forest_1920x1080.ivf";
 }
 
 use ui_consts::*;

bin/app/src/gfx/mod.rs

@@ -18,7 +18,7 @@
 
 use darkfi_serial::{
     async_trait, AsyncEncodable, AsyncWrite, Decodable, Encodable, FutAsyncWriteExt,
-    SerialDecodable, SerialEncodable,
+    SerialDecodable, SerialEncodable, VarInt,
 };
 #[cfg(target_os = "android")]
 use miniquad::native::egl;
@@ -355,7 +355,7 @@ impl RenderApi {
 pub struct DrawMesh {
     pub vertex_buffer: ManagedBufferPtr,
     pub index_buffer: ManagedBufferPtr,
-    pub texture: Option<ManagedTexturePtr>,
+    pub textures: Option<Vec<ManagedTexturePtr>>,
     pub num_elements: i32,
 }
 
@@ -369,15 +369,23 @@ impl DrawMesh {
         let vertex_buffer_id = self.vertex_buffer.id;
         let index_buffer_id = self.index_buffer.id;
         let _buffers_keep_alive = [self.vertex_buffer, self.index_buffer];
-        let texture = match self.texture {
-            Some(gfx_texture) => Some(Self::get_texture(textures, gfx_texture, debug_str)),
+
+        let textures = match self.textures {
+            Some(gfx_textures) => {
+                let mut compiled = Vec::with_capacity(gfx_textures.len());
+                for gfx_texture in gfx_textures {
+                    compiled.push(Self::get_texture(textures, gfx_texture, debug_str));
+                }
+                Some(compiled)
+            }
             None => None,
         };
+
         GfxDrawMesh {
             vertex_buffer: Self::get_buffer(buffers, vertex_buffer_id, debug_str),
             index_buffer: Self::get_buffer(buffers, index_buffer_id, debug_str),
             _buffers_keep_alive,
-            texture,
+            textures,
             num_elements: self.num_elements,
         }
     }
@@ -419,12 +427,15 @@ impl Encodable for DrawMesh {
         len += self.index_buffer.epoch.encode(s)?;
         len += self.index_buffer.tag.encode(s)?;
         len += self.index_buffer.buftype.encode(s)?;
-        match &self.texture {
-            Some(t) => {
+        match &self.textures {
+            Some(texs) => {
                 len += 1u8.encode(s)?;
-                len += t.id.encode(s)?;
-                len += t.epoch.encode(s)?;
-                len += t.tag.encode(s)?;
+                len += VarInt(texs.len() as u64).encode(s)?;
+                for t in texs {
+                    len += t.id.encode(s)?;
+                    len += t.epoch.encode(s)?;
+                    len += t.tag.encode(s)?;
+                }
             }
             None => {
                 len += 0u8.encode(s)?;
@@ -435,6 +446,12 @@ impl Encodable for DrawMesh {
     }
 }
 
+#[derive(Clone, Copy, Debug, SerialEncodable)]
+pub enum GraphicPipeline {
+    RGB,
+    YUV,
+}
+
 #[async_trait]
 impl AsyncEncodable for DrawMesh {
     async fn encode_async<W: AsyncWrite + Unpin + Send>(
@@ -454,6 +471,7 @@ pub enum DrawInstruction {
     Draw(DrawMesh),
     Animation(AnimId),
     EnableDebug,
+    SetPipeline(GraphicPipeline),
 }
 
 impl DrawInstruction {
@@ -473,6 +491,7 @@ impl DrawInstruction {
             }
             Self::Animation(anim) => GfxDrawInstruction::Animation(anim),
             Self::EnableDebug => GfxDrawInstruction::EnableDebug,
+            Self::SetPipeline(pipeline) => GfxDrawInstruction::SetPipeline(pipeline),
         }
     }
 }
@@ -520,7 +539,7 @@ struct GfxDrawMesh {
     index_buffer: miniquad::BufferId,
     /// Keeps the buffers alive for the duration of this draw call
     _buffers_keep_alive: [ManagedBufferPtr; 2],
-    texture: Option<(ManagedTexturePtr, miniquad::TextureId)>,
+    textures: Option<Vec<(ManagedTexturePtr, miniquad::TextureId)>>,
     num_elements: i32,
 }
 
@@ -533,6 +552,7 @@ enum GfxDrawInstruction {
     Draw(GfxDrawMesh),
     Animation(AnimId),
     EnableDebug,
+    SetPipeline(GraphicPipeline),
 }
 
 #[derive(Clone, Debug)]
@@ -548,10 +568,12 @@ struct RenderContext<'a> {
     draw_calls: &'a HashMap<DcId, GfxDrawCall>,
     uniforms_data: [u8; 128],
     white_texture: miniquad::TextureId,
+    loaded_pipelines: &'a [Pipeline; 2],
 
     scale: f32,
     view: Rectangle,
     cursor: Point,
+    gfx_pipeline: GraphicPipeline,
 
     anims: &'a mut HashMap<AnimId, GfxSeqAnim>,
 }
@@ -614,6 +636,7 @@ impl<'a> RenderContext<'a> {
         let old_scale = self.scale;
         let old_view = self.view;
         let old_cursor = self.cursor;
+        let old_pipeline = self.gfx_pipeline;
 
         for (idx, instr) in draw_call.instrs.iter().enumerate() {
             if DEBUG_TRAX {
@@ -674,14 +697,14 @@ impl<'a> RenderContext<'a> {
                     if is_debug {
                         debug!(target: "gfx", "{ws}draw({mesh:?})");
                     }
-                    let texture = match mesh.texture {
-                        Some((_, texture)) => texture,
-                        None => self.white_texture,
+                    let images = match &mesh.textures {
+                        Some(texs) => texs.iter().map(|(_, tex_id)| *tex_id).collect(),
+                        None => vec![self.white_texture],
                     };
                     let bindings = Bindings {
                         vertex_buffers: vec![mesh.vertex_buffer],
                         index_buffer: mesh.index_buffer,
-                        images: vec![texture],
+                        images,
                     };
                     self.ctx.apply_bindings(&bindings);
                     self.ctx.draw(0, mesh.num_elements, 1);
@@ -700,6 +723,15 @@ impl<'a> RenderContext<'a> {
                     is_debug = true;
                     debug!(target: "gfx", "Frame start");
                 }
+                GfxDrawInstruction::SetPipeline(pipeline) => {
+                    self.gfx_pipeline = *pipeline;
+                    let pipeline_idx = *pipeline as usize;
+                    assert!(pipeline_idx < self.loaded_pipelines.len());
+                    self.ctx.apply_pipeline(&self.loaded_pipelines[pipeline_idx]);
+                    if is_debug {
+                        debug!(target: "gfx", "{ws}set_pipeline({pipeline:?})");
+                    }
+                }
             }
         }
 
@@ -727,6 +759,10 @@ impl<'a> RenderContext<'a> {
         self.apply_view();
 
         self.cursor = old_cursor;
+        self.gfx_pipeline = old_pipeline;
+        let pipeline_idx = self.gfx_pipeline as usize;
+        assert!(pipeline_idx < self.loaded_pipelines.len());
+        self.ctx.apply_pipeline(&self.loaded_pipelines[pipeline_idx]);
         self.apply_model();
     }
 }
@@ -904,7 +940,7 @@ struct Stage {
     ctx: Box<dyn RenderingBackend>,
     #[cfg(target_os = "android")]
     libegl: egl::LibEgl,
-    pipeline: Pipeline,
+    loaded_pipelines: [Pipeline; 2],
     white_texture: miniquad::TextureId,
     draw_calls: HashMap<DcId, GfxDrawCall>,
     pending_batches: HashMap<BatchGuardId, Vec<GraphicsMethod>>,
@@ -966,42 +1002,8 @@ impl Stage {
 
         let anims: HashMap<AnimId, GfxSeqAnim> = HashMap::new();
 
-        let mut shader_meta: ShaderMeta = shader::meta();
-        shader_meta.uniforms.uniforms.push(UniformDesc::new("Projection", UniformType::Mat4));
-        shader_meta.uniforms.uniforms.push(UniformDesc::new("Model", UniformType::Mat4));
-
-        let shader = ctx
-            .new_shader(
-                match ctx.info().backend {
-                    Backend::OpenGl => ShaderSource::Glsl {
-                        vertex: shader::GL_VERTEX,
-                        fragment: shader::GL_FRAGMENT,
-                    },
-                    Backend::Metal => ShaderSource::Msl { program: shader::METAL },
-                },
-                shader_meta,
-            )
-            .unwrap();
-
-        let params = PipelineParams {
-            color_blend: Some(BlendState::new(
-                Equation::Add,
-                BlendFactor::Value(BlendValue::SourceAlpha),
-                BlendFactor::OneMinusValue(BlendValue::SourceAlpha),
-            )),
-            ..Default::default()
-        };
-
-        let pipeline = ctx.new_pipeline(
-            &[BufferLayout::default()],
-            &[
-                VertexAttribute::new("in_pos", VertexFormat::Float2),
-                VertexAttribute::new("in_color", VertexFormat::Float4),
-                VertexAttribute::new("in_uv", VertexFormat::Float2),
-            ],
-            shader,
-            params,
-        );
+        let rgb_pipeline = shader::create_rgb_pipeline(&mut ctx);
+        let yuv_pipeline = shader::create_yuv_pipeline(&mut ctx);
 
         #[cfg(target_os = "android")]
         let libegl = egl::LibEgl::try_load().expect("Cant load LibEGL");
@@ -1010,7 +1012,7 @@ impl Stage {
             ctx,
             #[cfg(target_os = "android")]
             libegl,
-            pipeline,
+            loaded_pipelines: [rgb_pipeline, yuv_pipeline],
             white_texture,
             draw_calls: HashMap::from([(
                 0,
@@ -1790,7 +1792,9 @@ impl EventHandler for Stage {
 
     fn draw(&mut self) {
         self.ctx.begin_default_pass(PassAction::clear_color(0., 0., 0., 1.));
-        self.ctx.apply_pipeline(&self.pipeline);
+
+        // Apply default RGB pipeline
+        self.ctx.apply_pipeline(&self.loaded_pipelines[GraphicPipeline::RGB as usize]);
 
         // This will make the top left (0, 0) and the bottom right (1, 1)
         // Default is (-1, 1) -> (1, -1)
@@ -1816,9 +1820,11 @@ impl EventHandler for Stage {
             draw_calls: &self.draw_calls,
             uniforms_data,
             white_texture: self.white_texture,
+            loaded_pipelines: &self.loaded_pipelines,
             scale: 1.,
             view: Rectangle::from([0., 0., screen_w, screen_h]),
             cursor: Point::from([0., 0.]),
+            gfx_pipeline: GraphicPipeline::RGB,
             anims: &mut self.anims,
         };
         render_ctx.draw();

bin/app/src/gfx/shader.rs

@@ -35,7 +35,7 @@ void main() {
     uv = in_uv;
 }"#;
 
-pub const GL_FRAGMENT: &str = r#"#version 100
+pub const GL_FRAGMENT_RGB: &str = r#"#version 100
 varying lowp vec4 color;
 varying lowp vec2 uv;
 
@@ -45,7 +45,28 @@ void main() {
     gl_FragColor = color * texture2D(tex, uv);
 }"#;
 
-pub const METAL: &str = r#"
+pub const GL_FRAGMENT_YUV: &str = r#"#version 100
+varying lowp vec4 color;
+varying lowp vec2 uv;
+
+uniform sampler2D tex_y;
+uniform sampler2D tex_u;
+uniform sampler2D tex_v;
+
+void main() {
+    lowp float y = texture2D(tex_y, uv).r;
+    lowp float u = texture2D(tex_u, uv).r - 0.5;
+    lowp float v = texture2D(tex_v, uv).r - 0.5;
+
+    // BT.601 YUV to RGB conversion
+    lowp float r = y + 1.402 * v;
+    lowp float g = y - 0.344 * u - 0.714 * v;
+    lowp float b = y + 1.772 * u;
+
+    gl_FragColor = color * vec4(r, g, b, 1.0);
+}"#;
+
+pub const METAL_RGB: &str = r#"
 #include <metal_stdlib>
 
 using namespace metal;
@@ -88,9 +109,127 @@ fragment float4 fragmentShader(RasterizerData in [[stage_in]], texture2d<float>
 
 "#;
 
-pub fn meta() -> ShaderMeta {
+pub const METAL_YUV: &str = r#"
+#include <metal_stdlib>
+
+using namespace metal;
+
+struct Uniforms
+{
+    float4x4 Projection;
+    float4x4 Model;
+};
+
+struct Vertex
+{
+    float2 in_pos   [[attribute(0)]];
+    float4 in_color [[attribute(1)]];
+    float2 in_uv    [[attribute(2)]];
+};
+
+struct RasterizerData
+{
+    float4 position [[position]];
+    float4 color [[user(locn0)]];
+    float2 uv [[user(locn1)]];
+};
+
+vertex RasterizerData vertexShader(Vertex v [[stage_in]])
+{
+    RasterizerData out;
+
+    out.position = uniforms.Model * uniforms.Projection * float4(v.in_pos.xy, 0.0, 1.0);
+    out.color = v.in_color;
+    out.uv = v.texcoord;
+
+    return out
+}
+
+fragment float4 fragmentShader(RasterizerData in [[stage_in]],
+                                texture2d<float> tex_y [[texture(0)]],
+                                texture2d<float> tex_u [[texture(1)]],
+                                texture2d<float> tex_v [[texture(2)]],
+                                sampler tex_y_smplr [[sampler(0)]],
+                                sampler tex_u_smplr [[sampler(1)]],
+                                sampler tex_v_smplr [[sampler(2)]])
+{
+    float y = tex_y.sample(tex_y_smplr, in.uv).r;
+    float u = tex_u.sample(tex_u_smplr, in.uv).r - 0.5;
+    float v = tex_v.sample(tex_v_smplr, in.uv).r - 0.5;
+
+    // BT.601 YUV to RGB conversion
+    float r = y + 1.402 * v;
+    float g = y - 0.344 * u - 0.714 * v;
+    float b = y + 1.772 * u;
+
+    return in.color * float4(r, g, b, 1.0);
+}
+
+"#;
+
+pub fn meta_rgb() -> ShaderMeta {
     ShaderMeta {
         images: vec!["tex".to_string()],
         uniforms: UniformBlockLayout { uniforms: vec![] },
     }
 }
+
+pub fn meta_yuv() -> ShaderMeta {
+    ShaderMeta {
+        images: vec!["tex_y".to_string(), "tex_u".to_string(), "tex_v".to_string()],
+        uniforms: UniformBlockLayout { uniforms: vec![] },
+    }
+}
+
+pub fn create_rgb_pipeline(ctx: &mut Box<dyn RenderingBackend>) -> Pipeline {
+    let shader_meta = meta_rgb();
+
+    let shader_source = match ctx.info().backend {
+        Backend::OpenGl => ShaderSource::Glsl { vertex: GL_VERTEX, fragment: GL_FRAGMENT_RGB },
+        Backend::Metal => ShaderSource::Msl { program: METAL_RGB },
+    };
+
+    create_pipeline_with_meta(ctx, shader_source, shader_meta)
+}
+
+pub fn create_yuv_pipeline(ctx: &mut Box<dyn RenderingBackend>) -> Pipeline {
+    let shader_meta = meta_yuv();
+
+    let shader_source = match ctx.info().backend {
+        Backend::OpenGl => ShaderSource::Glsl { vertex: GL_VERTEX, fragment: GL_FRAGMENT_YUV },
+        Backend::Metal => ShaderSource::Msl { program: METAL_YUV },
+    };
+
+    create_pipeline_with_meta(ctx, shader_source, shader_meta)
+}
+
+fn create_pipeline_with_meta(
+    ctx: &mut Box<dyn RenderingBackend>,
+    shader_source: ShaderSource,
+    mut shader_meta: ShaderMeta,
+) -> Pipeline {
+    shader_meta.uniforms.uniforms.push(UniformDesc::new("Projection", UniformType::Mat4));
+    shader_meta.uniforms.uniforms.push(UniformDesc::new("Model", UniformType::Mat4));
+
+    let shader = ctx.new_shader(shader_source, shader_meta).unwrap();
+
+    let params = PipelineParams {
+        color_blend: Some(BlendState::new(
+            Equation::Add,
+            BlendFactor::Value(BlendValue::SourceAlpha),
+            BlendFactor::OneMinusValue(BlendValue::SourceAlpha),
+        )),
+        ..Default::default()
+    };
+
+    ctx.new_pipeline(
+        &[BufferLayout::default()],
+        &[
+            VertexAttribute::new("in_pos", VertexFormat::Float2),
+            VertexAttribute::new("in_color", VertexFormat::Float4),
+            VertexAttribute::new("in_uv", VertexFormat::Float2),
+        ],
+        shader,
+        params,
+    )
+}

bin/app/src/mesh.rs

@@ -49,21 +49,22 @@ pub struct MeshInfo {
 }
 
 impl MeshInfo {
-    /// Convenience method
-    pub fn draw_with_texture(self, texture: ManagedTexturePtr) -> DrawMesh {
+    /// Convenience method for textured mesh
+    pub fn draw_with_textures(self, textures: Vec<ManagedTexturePtr>) -> DrawMesh {
         DrawMesh {
             vertex_buffer: self.vertex_buffer,
             index_buffer: self.index_buffer,
-            texture: Some(texture),
+            textures: Some(textures),
             num_elements: self.num_elements,
         }
     }
+
     /// Convenience method
     pub fn draw_untextured(self) -> DrawMesh {
         DrawMesh {
             vertex_buffer: self.vertex_buffer,
             index_buffer: self.index_buffer,
-            texture: None,
+            textures: None,
             num_elements: self.num_elements,
         }
     }

bin/app/src/text2/render.rs

@@ -133,7 +133,7 @@ fn render_glyph_run(
         );
     }
 
-    mesh.alloc(render_api).draw_with_texture(atlas.texture)
+    mesh.alloc(render_api).draw_with_textures(vec![atlas.texture])
 }
 
 fn render_underline(

bin/app/src/ui/chatview/page.rs

@@ -206,7 +206,7 @@ impl PrivMessage {
         }
 
         let mesh = mesh.alloc(render_api);
-        let mesh = mesh.draw_with_texture(self.atlas.texture.clone());
+        let mesh = mesh.draw_with_textures(vec![self.atlas.texture.clone()]);
         self.mesh_cache = Some(mesh.clone());
 
         mesh
@@ -479,7 +479,7 @@ impl DateMessage {
         }
 
         let mesh = mesh.alloc(render_api);
-        let mesh = mesh.draw_with_texture(self.atlas.texture.clone());
+        let mesh = mesh.draw_with_textures(vec![self.atlas.texture.clone()]);
         self.mesh_cache = Some(mesh.clone());
 
         mesh
@@ -671,7 +671,7 @@ impl FileMessage {
             let mut mesh_img = MeshBuilder::new(gfxtag!("file_img"));
             mesh_img.draw_box(&mesh_rect, COLOR_WHITE, &uv_rect);
             let mesh_img = mesh_img.alloc(render_api);
-            let mesh_img = mesh_img.draw_with_texture(texture);
+            let mesh_img = mesh_img.draw_with_textures(vec![texture]);
             return vec![mesh_img, mesh_gradient];
         }
         drop(imgbuf_);
@@ -718,7 +718,7 @@ impl FileMessage {
         mesh.draw_box_shadow(&mesh_rect, glow_color, Self::GLOW_SIZE);
 
         let mesh = mesh.alloc(render_api);
-        let mesh = mesh.draw_with_texture(self.atlas.texture.clone());
+        let mesh = mesh.draw_with_textures(vec![self.atlas.texture.clone()]);
 
         vec![mesh]
     }

bin/app/src/ui/emoji_picker/emoji.rs

@@ -108,7 +108,7 @@ impl EmojiMeshes {
         let uv = atlas.fetch_uv(glyph.glyph_id).expect("missing glyph UV rect");
         let mut mesh = MeshBuilder::new(gfxtag!("emoji_mesh"));
         mesh.draw_box(&Rectangle::new(x, y, w, h), COLOR_WHITE, &uv);
-        mesh.alloc(&self.render_api).draw_with_texture(atlas.texture)
+        mesh.alloc(&self.render_api).draw_with_textures(vec![atlas.texture])
     }
 
     pub fn get_list<'a>(&'a self) -> &'a Vec<String> {

bin/app/src/ui/image.rs

@@ -157,7 +157,7 @@ impl Image {
         let mesh = DrawMesh {
             vertex_buffer: mesh.vertex_buffer,
             index_buffer: mesh.index_buffer,
-            texture: Some(texture),
+            textures: Some(vec![texture]),
             num_elements: mesh.num_elements,
         };
 

bin/app/src/ui/vector_art/mod.rs

@@ -109,7 +109,7 @@ impl VectorArt {
         //debug!(target: "ui::vector_art", "vec_draw_instrs {verts:?} | {indices:?} | {num_elements}");
         let vertex_buffer = self.render_api.new_vertex_buffer(verts, gfxtag!("vectorart"));
         let index_buffer = self.render_api.new_index_buffer(indices, gfxtag!("vectorart"));
-        let mesh = DrawMesh { vertex_buffer, index_buffer, texture: None, num_elements };
+        let mesh = DrawMesh { vertex_buffer, index_buffer, textures: None, num_elements };
 
         vec![DrawInstruction::Move(rect.pos()), DrawInstruction::Draw(mesh)]
     }

bin/app/src/ui/vid/mod.rs

@@ -24,8 +24,8 @@ use tracing::instrument;
 
 use crate::{
     gfx::{
-        anim::Frame, gfxtag, DrawCall, DrawInstruction, DrawMesh, ManagedSeqAnimPtr,
-        ManagedTexturePtr, Rectangle, RenderApi,
+        anim::Frame, gfxtag, DrawCall, DrawInstruction, DrawMesh, GraphicPipeline,
+        ManagedSeqAnimPtr, ManagedTexturePtr, Rectangle, RenderApi,
     },
     mesh::{MeshBuilder, MeshInfo, COLOR_WHITE},
     prop::{BatchGuardPtr, PropertyAtomicGuard, PropertyRect, PropertyStr, PropertyUint32, Role},
@@ -38,19 +38,26 @@ use super::{DrawUpdate, OnModify, UIObject};
 mod ivf;
 mod threads;
 
-use threads::{spawn_decoder_thread, spawn_loader_demuxer_thread};
+use threads::spawn_decoder_thread;
 
 macro_rules! t { ($($arg:tt)*) => { trace!(target: "ui:video", $($arg)*); } }
 
 pub type VideoPtr = Arc<Video>;
 
+#[derive(Clone)]
+pub struct YuvTextures {
+    y: ManagedTexturePtr,
+    u: ManagedTexturePtr,
+    v: ManagedTexturePtr,
+}
+
 #[derive(Clone)]
 pub struct Av1VideoData {
-    textures: Vec<Option<ManagedTexturePtr>>,
+    textures: Vec<Option<YuvTextures>>,
     anim: ManagedSeqAnimPtr,
 
-    textures_pub: async_broadcast::Sender<(usize, ManagedTexturePtr)>,
-    textures_sub: async_broadcast::Receiver<(usize, ManagedTexturePtr)>,
+    textures_pub: async_broadcast::Sender<(usize, YuvTextures)>,
+    textures_sub: async_broadcast::Receiver<(usize, YuvTextures)>,
 }
 
 impl Av1VideoData {
@@ -126,23 +133,11 @@ impl Video {
     fn load_video(&self) {
         let path = self.path.get();
 
-        // Thread 1 -> thread 2 channel: raw AV1 encoded frames
-        let (frame_tx, frame_rx) = mpsc::channel();
-
-        // Thread 1 (loader + demuxer):
-        // loads chunks, demuxes IVF -> AV1 frames, initializes vid_data
-        let loader_handle = spawn_loader_demuxer_thread(
-            path,
-            frame_tx,
-            self.vid_data.clone(),
-            self.render_api.clone(),
-        );
-        // Thread 2 (decoder):
-        // blocks on frame_rx, decodes AV1 -> RGB, creates textures directly
+        // Decoder thread:
+        // loads path, decodes AV1 -> RGB, creates textures directly
         let decoder_handle =
-            spawn_decoder_thread(frame_rx, self.vid_data.clone(), self.render_api.clone());
+            spawn_decoder_thread(path, self.vid_data.clone(), self.render_api.clone());
 
-        *self._load_handle.lock() = Some(loader_handle);
         *self._decoder_handle.lock() = Some(decoder_handle);
     }
 
@@ -202,24 +197,24 @@ impl Video {
         let mut loaded_n_frames = 0;
         let total_frames = vid_data.textures.len();
 
-        for (texture_idx, (mut texture, mut tsub)) in
+        for (tex_idx, (mut tex, mut tsub)) in
             vid_data.textures.into_iter().zip(tsubs.into_iter()).enumerate()
         {
             let vertex_buffer = mesh.vertex_buffer.clone();
             let index_buffer = mesh.index_buffer.clone();
 
-            let Some(texture) = texture.take() else {
+            let Some(tex) = tex.take() else {
                 let anim = vid_data.anim.clone();
                 let task = self.ex.spawn(async move {
-                    while let Ok((frame_idx, texture)) = tsub.recv().await {
-                        if frame_idx != texture_idx {
+                    while let Ok((frame_idx, tex)) = tsub.recv().await {
+                        if frame_idx != tex_idx {
                             continue
                         }
 
                         let mesh = DrawMesh {
                             vertex_buffer,
                             index_buffer,
-                            texture: Some(texture),
+                            textures: Some(vec![tex.y, tex.u, tex.v]),
                             num_elements: mesh.num_elements,
                         };
                         let dc = DrawCall {
@@ -240,7 +235,7 @@ impl Video {
             let mesh = DrawMesh {
                 vertex_buffer,
                 index_buffer,
-                texture: Some(texture),
+                textures: Some(vec![tex.y, tex.u, tex.v]),
                 num_elements: mesh.num_elements,
             };
             let dc = DrawCall {
@@ -249,7 +244,7 @@ impl Video {
                 z_index: 0,
                 debug_str: "video",
             };
-            vid_data.anim.update(texture_idx, Frame::new(40, dc));
+            vid_data.anim.update(tex_idx, Frame::new(40, dc));
             loaded_n_frames += 1;
         }
 
@@ -261,6 +256,7 @@ impl Video {
                 self.dc_key,
                 DrawCall::new(
                     vec![
+                        DrawInstruction::SetPipeline(GraphicPipeline::YUV),
                         DrawInstruction::Move(rect.pos()),
                         DrawInstruction::Animation(vid_data.anim.id),
                     ],

bin/app/src/ui/vid/threads.rs

@@ -32,97 +32,15 @@ use crate::{
     util::spawn_thread,
 };
 
-use super::{ivf::IvfStreamingDemuxer, Av1VideoData};
+use super::{ivf::IvfStreamingDemuxer, Av1VideoData, YuvTextures};
 
 macro_rules! d { ($($arg:tt)*) => { debug!(target: "ui:video", $($arg)*); } }
-macro_rules! w { ($($arg:tt)*) => { warn!(target: "ui:video", $($arg)*); } }
-
-/// Spawn the loader-demuxer thread (Thread 1 of 2)
-///
-/// This thread loads video file chunks sequentially and demuxes AV1 frames.
-///
-/// # Thread Coordination
-/// - Loads chunks from disk using format `path.{000, 001, 002, ...}` where `{frame}` is replaced
-/// - Demuxes IVF container to extract raw AV1 frames
-/// - Initializes vid_data with header info from first chunk
-/// - Sends frames to decoder thread via `frame_tx` channel
-/// - Signals completion by dropping `frame_tx`
-///
-/// # Arguments
-/// * `path` - Base path with `{frame}` placeholder, e.g. `"assets/video.ivf.{frame}"`
-/// * `frame_tx` - Channel sender for raw AV1 frames to decoder thread
-/// * `vid_data` - Shared video data storage to initialize
-/// * `render_api` - Render API for creating animation
-///
-/// # Returns
-/// JoinHandle for the spawned thread
-pub fn spawn_loader_demuxer_thread(
-    path: String,
-    frame_tx: Sender<Vec<u8>>,
-    vid_data: Arc<SyncMutex<Option<Av1VideoData>>>,
-    render_api: RenderApi,
-) -> std::thread::JoinHandle<()> {
-    spawn_thread("video-loader-demuxer", move || {
-        let mut chunk_idx: usize = 0;
-        let mut demuxer: Option<IvfStreamingDemuxer> = None;
-
-        loop {
-            // Replace {frame} placeholder with zero-padded chunk number
-            let chunk_path = path.replace("{frame}", &format!("{chunk_idx:03}"));
-            d!("Loading video chunk: {chunk_path}");
-
-            // Load chunk asynchronously via miniquad callback
-            let data = Arc::new(SyncMutex::new(None));
-            let data2 = data.clone();
-            miniquad::fs::load_file(&chunk_path, {
-                let chunk_path = chunk_path.clone();
-                move |res| match res {
-                    Ok(chunk) => *data2.lock() = Some(chunk),
-                    Err(err) => {
-                        error!("Failed to load chunk {chunk_path}: {err}");
-                    }
-                }
-            });
-            let data = std::mem::take(&mut *data.lock());
-
-            // Empty data means file not found - end of chunk sequence
-            let Some(data) = data else {
-                // Close channel to signal decoder thread
-                drop(frame_tx);
-                d!("Video demuxer finished");
-                return
-            };
-
-            if let Some(demuxer) = demuxer.as_mut() {
-                demuxer.feed_data(data);
-            } else {
-                // First chunk: initialize demuxer from IVF header
-                let dem = IvfStreamingDemuxer::from_first_chunk(data).unwrap();
-                let num_frames = dem.header.num_frames as usize;
-                demuxer = Some(dem);
-
-                // Initialize vid_data with header info
-                *vid_data.lock() = Some(Av1VideoData::new(num_frames, &render_api));
-            }
-
-            let demuxer = demuxer.as_mut().unwrap();
-            // Extract all complete frames from this chunk
-            while let Some(frame) = demuxer.try_read_frame() {
-                frame_tx.send(frame).unwrap();
-                d!("Sent video chunk {chunk_idx}");
-            }
-
-            chunk_idx += 1;
-        }
-    })
-}
 
 /// Spawn the decoder thread (Thread 2 of 2)
 ///
 /// This thread decodes AV1 frames and creates GPU textures directly.
 ///
 /// # Thread Coordination
-/// - Receives raw AV1 frames from loader-demuxer thread via `frame_rx` channel
 /// - Uses optimistic decoding strategy:
 ///   1. Drain all pending frames with `try_recv()`
 ///   2. When queue is empty, block on `recv()` for next frame
@@ -132,7 +50,6 @@ pub fn spawn_loader_demuxer_thread(
 /// - On channel close, flushes decoder
 ///
 /// # Arguments
-/// * `frame_rx` - Channel receiver for raw AV1 frames from loader-demuxer thread
 /// * `vid_data` - Shared video data storage to update with textures
 /// * `render_api` - Render API for creating textures
 /// * `dc_key` - Draw call key for triggering updates
@@ -140,7 +57,7 @@ pub fn spawn_loader_demuxer_thread(
 /// # Returns
 /// JoinHandle for the spawned thread
 pub fn spawn_decoder_thread(
-    frame_rx: Receiver<Vec<u8>>,
+    path: String,
     vid_data: Arc<SyncMutex<Option<Av1VideoData>>>,
     render_api: RenderApi,
 ) -> std::thread::JoinHandle<()> {
@@ -154,10 +71,26 @@ pub fn spawn_decoder_thread(
         let mut decoder = Rav1dDecoder::with_settings(&settings).unwrap();
         //let mut decoder = Rav1dDecoder::new().unwrap();
 
+        let data = Arc::new(SyncMutex::new(None));
+        let data2 = data.clone();
+        miniquad::fs::load_file(&path, {
+            move |res| match res {
+                Ok(chunk) => *data2.lock() = Some(chunk),
+                Err(err) => {
+                    error!("Failed to load chunk: {err}");
+                }
+            }
+        });
+        let data = std::mem::take(&mut *data.lock()).unwrap();
+
+        let mut demuxer = IvfStreamingDemuxer::from_first_chunk(data).unwrap();
+        let num_frames = demuxer.header.num_frames as usize;
+
+        *vid_data.lock() = Some(Av1VideoData::new(num_frames, &render_api));
+
         let mut frame_idx = 0;
         loop {
-            // Blocking receive - returns Err when channel closes
-            let Ok(av1_frame) = frame_rx.recv() else {
+            let Some(av1_frame) = demuxer.try_read_frame() else {
                 // Channel closed - drain decoder (like dav1dplay)
                 while let Ok(pic) = decoder.get_picture() {
                     process(&mut frame_idx, &pic, &vid_data, &render_api);
@@ -193,7 +126,6 @@ pub fn spawn_decoder_thread(
                 }
             }
         }
-        d!("Video decode finished, total frames: {frame_idx}");
     })
 }
 
@@ -204,46 +136,59 @@ fn process(
     render_api: &RenderApi,
 ) {
     // rav1d stores data as planar GBR (Y=G, U=B, V=R)
-    let g_plane = pic.plane(PlanarImageComponent::Y);
-    let b_plane = pic.plane(PlanarImageComponent::U);
-    let r_plane = pic.plane(PlanarImageComponent::V);
+    let y_plane = pic.plane(PlanarImageComponent::Y);
+    let u_plane = pic.plane(PlanarImageComponent::U);
+    let v_plane = pic.plane(PlanarImageComponent::V);
 
-    let g_stride = pic.stride(PlanarImageComponent::Y) as usize;
-    let b_stride = pic.stride(PlanarImageComponent::U) as usize;
-    let r_stride = pic.stride(PlanarImageComponent::V) as usize;
+    let y_stride = pic.stride(PlanarImageComponent::Y) as usize;
+    let u_stride = pic.stride(PlanarImageComponent::U) as usize;
+    let v_stride = pic.stride(PlanarImageComponent::V) as usize;
 
     let width = pic.width() as usize;
     let height = pic.height() as usize;
 
-    let mut buf = Vec::with_capacity(width * height * 3);
-    // Pack planar RGB into RGB format
-    for y in 0..height {
-        for x in 0..width {
-            let g_idx = y * g_stride + x;
-            let b_idx = y * b_stride + x;
-            let r_idx = y * r_stride + x;
-
-            buf.push(r_plane[r_idx]);
-            buf.push(g_plane[g_idx]);
-            buf.push(b_plane[b_idx]);
-        }
-    }
+    // Y plane is full resolution
+    let y_data = y_plane[..(y_stride * height)].to_vec();
+
+    // U and V planes are half resolution (4:2:0 subsampling)
+    let uv_width = width / 2;
+    let uv_height = height / 2;
+    let u_data = u_plane[..(u_stride * uv_height)].to_vec();
+    let v_data = v_plane[..(v_stride * uv_height)].to_vec();
 
-    // Create texture with RGB data
-    let tex = render_api.new_texture(
+    // Create 3 separate textures with Alpha format (1 byte per pixel)
+    let tex_y = render_api.new_texture(
         width as u16,
         height as u16,
-        buf,
-        TextureFormat::RGB8,
-        gfxtag!("video"),
+        y_data,
+        TextureFormat::Alpha,
+        gfxtag!("video_y"),
     );
 
+    let tex_u = render_api.new_texture(
+        uv_width as u16,
+        uv_height as u16,
+        u_data,
+        TextureFormat::Alpha,
+        gfxtag!("video_u"),
+    );
+
+    let tex_v = render_api.new_texture(
+        uv_width as u16,
+        uv_height as u16,
+        v_data,
+        TextureFormat::Alpha,
+        gfxtag!("video_v"),
+    );
+
+    let yuv_texs = YuvTextures { y: tex_y, u: tex_u, v: tex_v };
+
     {
         // Store in vid_data
         let mut vd_guard = vid_data.lock();
         let vd = vd_guard.as_mut().unwrap();
-        vd.textures[*frame_idx] = Some(tex.clone());
-        let _ = vd.textures_pub.try_broadcast((*frame_idx, tex));
+        vd.textures[*frame_idx] = Some(yuv_texs.clone());
+        let _ = vd.textures_pub.try_broadcast((*frame_idx, yuv_texs));
     }
     //d!("Loaded video frame {frame_idx}");
     *frame_idx += 1;