app: animated vids working for desktop but slow init time (need to stream)

bin/app/src/app/mod.rs

@@ -134,7 +134,7 @@ impl App {
         self.sg_root.link(window.clone());
         self.sg_root.link(setting_root.clone());
 
-        schema::make(&self, window.clone(), &i18n_fish).await;
+        schema::test::make(&self, window.clone(), &i18n_fish).await;
 
         //settings::make(&self, window, self.ex.clone()).await;
 

bin/app/src/app/node.rs

@@ -151,6 +151,39 @@ pub fn create_image(name: &str) -> SceneNode {
     node
 }
 
+pub fn create_video(name: &str) -> SceneNode {
+    t!("create_video({name})");
+    let mut node = SceneNode::new(name, SceneNodeType::Image);
+
+    let mut prop = Property::new("rect", PropertyType::Float32, PropertySubType::Pixel);
+    prop.set_array_len(4);
+    prop.allow_exprs();
+    node.add_property(prop).unwrap();
+
+    let mut prop = Property::new("uv", PropertyType::Float32, PropertySubType::Pixel);
+    prop.set_array_len(4);
+    prop.allow_exprs();
+    prop.set_range_f32(0., 1.);
+    prop.set_defaults_f32(vec![0., 0., 1., 1.]).unwrap();
+    node.add_property(prop).unwrap();
+
+    let prop = Property::new("z_index", PropertyType::Uint32, PropertySubType::Null);
+    node.add_property(prop).unwrap();
+
+    let prop = Property::new("priority", PropertyType::Uint32, PropertySubType::Null);
+    node.add_property(prop).unwrap();
+
+    let mut prop = Property::new("path", PropertyType::Str, PropertySubType::Null);
+    prop.set_ui_text("Path", "Path format string using {frame} in the name");
+    node.add_property(prop).unwrap();
+
+    let mut prop = Property::new("length", PropertyType::Uint32, PropertySubType::Null);
+    prop.set_ui_text("Frame Length", "Total frames to load (last frame + 1)");
+    node.add_property(prop).unwrap();
+
+    node
+}
+
 pub fn create_text(name: &str) -> SceneNode {
     t!("create_text({name})");
     let mut node = SceneNode::new(name, SceneNodeType::Text);

bin/app/src/app/schema/mod.rs

@@ -37,7 +37,7 @@ use crate::{
 mod chat;
 mod menu;
 //mod settings;
-//pub mod test;
+pub mod test;
 
 const COLOR_SCHEME: ColorScheme = ColorScheme::DarkMode;
 //const COLOR_SCHEME: ColorScheme = ColorScheme::PaperLight;

bin/app/src/app/schema/test.rs

@@ -18,14 +18,14 @@
 
 use crate::{
     app::{
-        node::{create_chatedit, create_layer, create_text, create_vector_art},
+        node::{create_chatedit, create_layer, create_text, create_vector_art, create_video},
         App,
     },
     expr,
     mesh::COLOR_PURPLE,
     prop::{PropertyAtomicGuard, PropertyFloat32, Role},
     scene::SceneNodePtr,
-    ui::{ChatEdit, Layer, Text, VectorArt, VectorShape},
+    ui::{ChatEdit, Layer, Text, VectorArt, VectorShape, Video},
     util::i18n::I18nBabelFish,
 };
 
@@ -248,6 +248,19 @@ pub async fn make(app: &App, window: SceneNodePtr, i18n_fish: &I18nBabelFish) {
     layer_node.link(node);
     */
 
+    // Create KING GNU!
+    let node = create_video("king");
+    let prop = node.get_property("rect").unwrap();
+    prop.set_f32(atom, Role::App, 0, 80.).unwrap();
+    prop.set_f32(atom, Role::App, 1, 400.).unwrap();
+    prop.set_f32(atom, Role::App, 2, 600.).unwrap();
+    prop.set_f32(atom, Role::App, 3, 600.).unwrap();
+    node.set_property_str(atom, Role::App, "path", "assets/forest/forest_{frame}.jpg").unwrap();
+    node.set_property_u32(atom, Role::App, "z_index", 1).unwrap();
+    node.set_property_u32(atom, Role::App, "length", 357).unwrap();
+    let node = node.setup(|me| Video::new(me, app.render_api.clone())).await;
+    layer_node.link(node);
+
     // Create some text
     let node = create_text("label");
     let prop = node.get_property("rect").unwrap();
@@ -462,6 +475,7 @@ pub async fn make(app: &App, window: SceneNodePtr, i18n_fish: &I18nBabelFish) {
     layer_node.link(node);
     */
 
+    /*
     // Text edit
     let node = create_chatedit("editz");
     node.set_property_bool(atom, Role::App, "is_active", true).unwrap();
@@ -532,4 +546,5 @@ pub async fn make(app: &App, window: SceneNodePtr, i18n_fish: &I18nBabelFish) {
         node.call_method("focus", vec![]).await.unwrap();
     });
     app.tasks.lock().unwrap().push(focus_task);
+    */
 }

bin/app/src/gfx/anim.rs

@@ -16,34 +16,102 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-use std::cell::RefCell;
+use async_trait::async_trait;
+use darkfi_serial::{AsyncEncodable, AsyncWrite, Encodable, FutAsyncWriteExt, SerialEncodable};
+use std::{
+    cell::RefCell,
+    collections::HashMap,
+    io::Write,
+    sync::{
+        atomic::{AtomicU32, Ordering},
+        Arc,
+    },
+};
 
-use super::DrawCall;
+use super::{DrawCall, GfxBufferId, GfxDrawCall, GfxTextureId};
 
-pub(super) trait AbstractAnimation: std::fmt::Debug {
-    fn tick(&self) -> DrawCall;
+// This can be in instruction but also implement encodable
+// maybe just remove trax?
+
+#[derive(Debug, Clone, SerialEncodable)]
+pub struct GfxSequenceAnimation {
+    oneshot: bool,
+    frames: Vec<GfxSequenceAnimationFrame>,
 }
 
-#[derive(Debug)]
-struct SequenceAnimation {
+impl GfxSequenceAnimation {
+    pub fn new(oneshot: bool, frames: Vec<GfxSequenceAnimationFrame>) -> Self {
+        Self { oneshot, frames }
+    }
+
+    pub(super) fn compile(
+        self: Self,
+        textures: &HashMap<GfxTextureId, miniquad::TextureId>,
+        buffers: &HashMap<GfxBufferId, miniquad::BufferId>,
+    ) -> SequenceAnimation {
+        let mut frames = Vec::with_capacity(self.frames.len());
+        for gfxframe in self.frames {
+            let duration = std::time::Duration::from_millis(gfxframe.duration as u64);
+            let dc = gfxframe.dc.compile(textures, buffers, 0).unwrap();
+            frames.push(SequenceAnimationFrame { duration, dc });
+        }
+        SequenceAnimation::new(self.oneshot, frames)
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct GfxSequenceAnimationFrame {
+    /// Duration of this frame in ms
+    duration: u32,
+    dc: GfxDrawCall,
+}
+
+impl GfxSequenceAnimationFrame {
+    pub fn new(duration: u32, dc: GfxDrawCall) -> Self {
+        Self { duration, dc }
+    }
+}
+
+/// We have to implement this manually due to macro autism.
+/// Since it contains GfxDrawCall that contains Instruction that can contain this.
+impl Encodable for GfxSequenceAnimationFrame {
+    fn encode<S: Write>(&self, s: &mut S) -> std::result::Result<usize, std::io::Error> {
+        let mut len = 0;
+        len += self.duration.encode(s)?;
+        len += self.dc.encode(s)?;
+        Ok(len)
+    }
+}
+#[async_trait]
+impl AsyncEncodable for GfxSequenceAnimationFrame {
+    async fn encode_async<W: AsyncWrite + Unpin + Send>(
+        &self,
+        w: &mut W,
+    ) -> std::io::Result<usize> {
+        let mut len = 0;
+        len += self.duration.encode_async(w).await?;
+        len += self.dc.encode_async(w).await?;
+        Ok(len)
+    }
+}
+
+#[derive(Debug, Clone)]
+pub(super) struct SequenceAnimation {
     oneshot: bool,
     frames: Vec<SequenceAnimationFrame>,
     state: RefCell<SequenceAnimationState>,
 }
-#[derive(Debug)]
-struct SequenceAnimationFrame {
-    dc: DrawCall,
-    duration: std::time::Duration,
-}
-#[derive(Debug)]
-struct SequenceAnimationState {
-    /// Timer between frames
-    timer: Option<std::time::Instant>,
-    current_idx: usize,
-}
 
-impl AbstractAnimation for SequenceAnimation {
-    fn tick(&self) -> DrawCall {
+impl SequenceAnimation {
+    fn new(oneshot: bool, frames: Vec<SequenceAnimationFrame>) -> Self {
+        Self {
+            oneshot,
+            frames,
+            state: RefCell::new(SequenceAnimationState { timer: None, current_idx: 0 }),
+        }
+    }
+
+    pub fn tick(&self) -> DrawCall {
         let mut state = self.state.borrow_mut();
 
         let elapsed = state.timer.get_or_insert_with(|| std::time::Instant::now()).elapsed();
@@ -55,3 +123,16 @@ impl AbstractAnimation for SequenceAnimation {
         self.frames[state.current_idx].dc.clone()
     }
 }
+
+#[derive(Debug, Clone)]
+struct SequenceAnimationFrame {
+    duration: std::time::Duration,
+    dc: DrawCall,
+}
+
+#[derive(Debug, Clone)]
+struct SequenceAnimationState {
+    /// Timer between frames
+    timer: Option<std::time::Instant>,
+    current_idx: usize,
+}

bin/app/src/gfx/mod.rs

@@ -41,8 +41,8 @@ use std::{
     },
 };
 
-mod anim;
-use anim::AbstractAnimation;
+pub mod anim;
+use anim::{GfxSequenceAnimation, SequenceAnimation};
 mod favico;
 mod linalg;
 pub use linalg::{Dimension, Point, Rectangle};
@@ -399,6 +399,7 @@ pub enum GfxDrawInstruction {
     SetPos(Point),
     ApplyView(Rectangle),
     Draw(GfxDrawMesh),
+    Animation(GfxSequenceAnimation),
     EnableDebug,
 }
 
@@ -415,6 +416,7 @@ impl GfxDrawInstruction {
             Self::SetPos(pos) => DrawInstruction::SetPos(pos),
             Self::ApplyView(view) => DrawInstruction::ApplyView(view),
             Self::Draw(mesh) => DrawInstruction::Draw(mesh.compile(textures, buffers, debug_str)?),
+            Self::Animation(anim) => DrawInstruction::Animation(anim.compile(textures, buffers)),
             Self::EnableDebug => DrawInstruction::EnableDebug,
         };
         Some(instr)
@@ -438,9 +440,7 @@ impl GfxDrawCall {
     ) -> Self {
         Self { instrs, dcs, z_index, debug_str }
     }
-}
 
-impl GfxDrawCall {
     fn compile(
         self,
         textures: &HashMap<GfxTextureId, miniquad::TextureId>,
@@ -477,7 +477,7 @@ enum DrawInstruction {
     SetPos(Point),
     ApplyView(Rectangle),
     Draw(DrawMesh),
-    Animation { anim: Arc<dyn AbstractAnimation> },
+    Animation(SequenceAnimation),
     EnableDebug,
 }
 
@@ -630,7 +630,7 @@ impl<'a> RenderContext<'a> {
                     self.ctx.apply_bindings(&bindings);
                     self.ctx.draw(0, mesh.num_elements, 1);
                 }
-                DrawInstruction::Animation { anim } => {
+                DrawInstruction::Animation(anim) => {
                     let dc = anim.tick();
                     self.draw_call(&dc, indent + 1, is_debug);
                 }

bin/app/src/ui/video.rs

@@ -24,6 +24,7 @@ use std::{io::Cursor, sync::Arc};
 
 use crate::{
     gfx::{
+        anim::{GfxSequenceAnimation, GfxSequenceAnimationFrame},
         gfxtag, GfxDrawCall, GfxDrawInstruction, GfxDrawMesh, ManagedTexturePtr, Rectangle,
         RenderApi,
     },
@@ -45,7 +46,7 @@ pub struct Video {
     render_api: RenderApi,
     tasks: SyncMutex<Vec<smol::Task<()>>>,
 
-    texture: SyncMutex<Option<ManagedTexturePtr>>,
+    textures: SyncMutex<Vec<ManagedTexturePtr>>,
     dc_key: u64,
 
     rect: PropertyRect,
@@ -53,6 +54,7 @@ pub struct Video {
     z_index: PropertyUint32,
     priority: PropertyUint32,
     path: PropertyStr,
+    len: PropertyUint32,
 
     parent_rect: SyncMutex<Option<Rectangle>>,
 }
@@ -67,13 +69,14 @@ impl Video {
         let z_index = PropertyUint32::wrap(node_ref, Role::Internal, "z_index", 0).unwrap();
         let priority = PropertyUint32::wrap(node_ref, Role::Internal, "priority", 0).unwrap();
         let path = PropertyStr::wrap(node_ref, Role::Internal, "path", 0).unwrap();
+        let len = PropertyUint32::wrap(node_ref, Role::Internal, "length", 0).unwrap();
 
         let self_ = Arc::new(Self {
             node,
             render_api,
             tasks: SyncMutex::new(vec![]),
 
-            texture: SyncMutex::new(None),
+            textures: SyncMutex::new(vec![]),
             dc_key: OsRng.gen(),
 
             rect,
@@ -81,6 +84,7 @@ impl Video {
             z_index,
             priority,
             path,
+            len,
 
             parent_rect: SyncMutex::new(None),
         });
@@ -89,15 +93,25 @@ impl Video {
     }
 
     async fn reload(self: Arc<Self>, batch: BatchGuardPtr) {
-        let texture = self.load_texture();
-        *self.texture.lock() = Some(texture);
-
+        self.load_textures();
         self.clone().redraw(batch).await;
     }
 
-    fn load_texture(&self) -> ManagedTexturePtr {
-        let path = self.path.get();
+    fn load_textures(&self) {
+        let len = self.len.get();
+        let mut textures = Vec::with_capacity(len as usize);
+        let path_fmt = self.path.get();
+        for i in 0..len {
+            t!("i = {i}");
+            let path = path_fmt.replace("{frame}", &format!("{i:#03}"));
+            let texture = self.load_texture(path);
+            textures.push(texture);
+        }
+
+        *self.textures.lock() = textures;
+    }
 
+    fn load_texture(&self, path: String) -> ManagedTexturePtr {
         // TODO we should NOT use panic here
         let data = Arc::new(SyncMutex::new(vec![]));
         let data2 = data.clone();
@@ -157,24 +171,36 @@ impl Video {
         self.uv.eval(atom, &rect).ok()?;
 
         let mesh = self.regen_mesh();
-        let texture = self.texture.lock().clone().expect("Node missing texture_id!");
-
-        let mesh = GfxDrawMesh {
-            vertex_buffer: mesh.vertex_buffer,
-            index_buffer: mesh.index_buffer,
-            texture: Some(texture),
-            num_elements: mesh.num_elements,
-        };
+        let textures = self.textures.lock().clone();
+        assert!(!textures.is_empty());
+
+        let mut frames = Vec::with_capacity(textures.len());
+        for texture in textures {
+            let mesh = GfxDrawMesh {
+                vertex_buffer: mesh.vertex_buffer.clone(),
+                index_buffer: mesh.index_buffer.clone(),
+                texture: Some(texture),
+                num_elements: mesh.num_elements,
+            };
+            let dc = GfxDrawCall {
+                instrs: vec![GfxDrawInstruction::Draw(mesh)],
+                dcs: vec![],
+                z_index: 0,
+                debug_str: "video",
+            };
+            frames.push(GfxSequenceAnimationFrame::new(40, dc));
+        }
+        let anim = GfxSequenceAnimation::new(false, frames);
 
         Some(DrawUpdate {
             key: self.dc_key,
             draw_calls: vec![(
                 self.dc_key,
                 GfxDrawCall::new(
-                    vec![GfxDrawInstruction::Move(rect.pos()), GfxDrawInstruction::Draw(mesh)],
+                    vec![GfxDrawInstruction::Move(rect.pos()), GfxDrawInstruction::Animation(anim)],
                     vec![],
                     self.z_index.get(),
-                    "img",
+                    "vid",
                 ),
             )],
         })
@@ -188,7 +214,7 @@ impl UIObject for Video {
     }
 
     fn init(&self) {
-        *self.texture.lock() = Some(self.load_texture());
+        self.load_textures();
     }
 
     async fn start(self: Arc<Self>, ex: ExecutorPtr) {
@@ -206,7 +232,7 @@ impl UIObject for Video {
     fn stop(&self) {
         self.tasks.lock().clear();
         *self.parent_rect.lock() = None;
-        *self.texture.lock() = None;
+        self.textures.lock().clear();
     }
 
     async fn draw(
@@ -231,4 +257,3 @@ impl Drop for Video {
         );
     }
 }
-