app/vid: decode video using native hardware on android

bin/app/Cargo.toml

@@ -100,8 +100,7 @@ rav1d = { git = "https://github.com/narodnik/rav1d", branch = "add-rust-api", fe
 tracing-android = "0.2.0"
 # Required by Arti: tor-dirmgr
 tor-dirmgr = { version="0.37.0", features=["static"] }
-# Disable asm for Android due to PIC relocation issues
-rav1d = { git = "https://github.com/narodnik/rav1d", branch = "add-rust-api", default-features = false, features = ["bitdepth_8"] }
+# Android uses MediaCodec for hardware-accelerated H.264 decoding instead of rav1d
 
 [target.'cfg(target_os = "windows")'.dependencies]
 # Used by tor-dirmgr

bin/app/java/MainActivity.java

@@ -14,6 +14,7 @@ import java.util.HashMap;
 
 import autosuggest.InvisibleInputView;
 import autosuggest.CustomInputConnection;
+import videodecode.VideoDecoder;
 
 //% END
 
@@ -190,6 +191,12 @@ public boolean isImeVisible() {
     return insets.isVisible(Type.ime());
 }
 
+public VideoDecoder createVideoDecoder() {
+    VideoDecoder decoder = new VideoDecoder();
+    decoder.setContext(this);
+    return decoder;
+}
+
 //% END
 
 //% MAIN_ACTIVITY_ON_CREATE

bin/app/quad.toml

@@ -4,5 +4,6 @@ java_files = [
     "java/autosuggest/InvisibleInputView.java",
     #"java/autosuggest/InvisibleInputManager.java",
     "java/ForegroundService.java",
+    "java/videodecode/VideoDecoder.java"
 ]
 

bin/app/src/android.rs → bin/app/src/android/mod.rs

@@ -22,6 +22,8 @@ use std::{collections::HashMap, path::PathBuf, sync::LazyLock};
 
 use crate::AndroidSuggestEvent;
 
+pub mod vid;
+
 macro_rules! call_mainactivity_int_method {
     ($method:expr, $sig:expr $(, $args:expr)*) => {{
         unsafe {

bin/app/src/android/vid.rs

@@ -0,0 +1,213 @@
+/* This file is part of DarkFi (https://dark.fi)
+ *
+ * Copyright (C) 2020-2025 Dyne.org foundation
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+//! Android video decoder JNI functions for MediaCodec integration
+
+use miniquad::native::android::{self, ndk_sys, ndk_utils};
+use parking_lot::Mutex as SyncMutex;
+use std::{
+    collections::HashMap,
+    sync::{mpsc, LazyLock},
+};
+
+pub struct DecodedFrame {
+    pub width: usize,
+    pub height: usize,
+    pub y_data: Vec<u8>,
+    pub u_data: Vec<u8>,
+    pub v_data: Vec<u8>,
+}
+
+struct VideoDecoderGlobals {
+    senders: HashMap<usize, mpsc::Sender<DecodedFrame>>,
+    next_id: usize,
+}
+
+unsafe impl Send for VideoDecoderGlobals {}
+unsafe impl Sync for VideoDecoderGlobals {}
+
+static VIDEO_DECODER_GLOBALS: LazyLock<SyncMutex<VideoDecoderGlobals>> =
+    LazyLock::new(|| SyncMutex::new(VideoDecoderGlobals { senders: HashMap::new(), next_id: 0 }));
+
+fn send(id: usize, frame: DecodedFrame) {
+    let globals = &VIDEO_DECODER_GLOBALS.lock();
+    if let Some(sender) = globals.senders.get(&id) {
+        let _ = sender.send(frame);
+    }
+}
+
+pub fn register(sender: mpsc::Sender<DecodedFrame>) -> usize {
+    let mut globals = VIDEO_DECODER_GLOBALS.lock();
+    let id = globals.next_id;
+    globals.next_id += 1;
+    globals.senders.insert(id, sender);
+    id
+}
+
+pub fn unregister(id: usize) {
+    VIDEO_DECODER_GLOBALS.lock().senders.remove(&id);
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn Java_videodecode_VideoDecoder_onFrameDecoded(
+    env: *mut ndk_sys::JNIEnv,
+    _: ndk_sys::jobject,
+    decoder_id: ndk_sys::jint,
+    y_data: ndk_sys::jbyteArray,
+    u_data: ndk_sys::jbyteArray,
+    v_data: ndk_sys::jbyteArray,
+    width: ndk_sys::jint,
+    height: ndk_sys::jint,
+) {
+    use std::slice;
+
+    let get_array_length = (**env).GetArrayLength.unwrap();
+    let y_len = get_array_length(env, y_data) as usize;
+    let u_len = get_array_length(env, u_data) as usize;
+    let v_len = get_array_length(env, v_data) as usize;
+
+    let get_byte_array_elements = (**env).GetByteArrayElements.unwrap();
+    let y_ptr = get_byte_array_elements(env, y_data, std::ptr::null_mut()) as *const u8;
+    let u_ptr = get_byte_array_elements(env, u_data, std::ptr::null_mut()) as *const u8;
+    let v_ptr = get_byte_array_elements(env, v_data, std::ptr::null_mut()) as *const u8;
+
+    let y_vec = slice::from_raw_parts(y_ptr, y_len).to_vec();
+    let u_vec = slice::from_raw_parts(u_ptr, u_len).to_vec();
+    let v_vec = slice::from_raw_parts(v_ptr, v_len).to_vec();
+
+    let release_byte_array_elements = (**env).ReleaseByteArrayElements.unwrap();
+    release_byte_array_elements(env, y_data, y_ptr as *mut i8, 0);
+    release_byte_array_elements(env, u_data, u_ptr as *mut i8, 0);
+    release_byte_array_elements(env, v_data, v_ptr as *mut i8, 0);
+
+    let frame = DecodedFrame {
+        width: width as usize,
+        height: height as usize,
+        y_data: y_vec,
+        u_data: u_vec,
+        v_data: v_vec,
+    };
+
+    send(decoder_id as usize, frame);
+}
+
+pub struct VideoDecoderHandle {
+    pub obj: ndk_sys::jobject,
+}
+
+impl Drop for VideoDecoderHandle {
+    fn drop(&mut self) {
+        unsafe {
+            let env = android::attach_jni_env();
+            let delete_local_ref = (**env).DeleteLocalRef.unwrap();
+            delete_local_ref(env, self.obj);
+        }
+    }
+}
+
+pub fn videodecoder_init(path: &str) -> Option<VideoDecoderHandle> {
+    unsafe {
+        let env = android::attach_jni_env();
+
+        // Call MainActivity.createVideoDecoder() helper method
+        let activity_class = (**env).GetObjectClass.unwrap()(env, android::ACTIVITY);
+        let create_method = (**env).GetMethodID.unwrap()(
+            env,
+            activity_class,
+            b"createVideoDecoder\0".as_ptr() as _,
+            b"()Lvideodecode/VideoDecoder;\0".as_ptr() as _,
+        );
+
+        let decoder_obj = (**env).CallObjectMethod.unwrap()(env, android::ACTIVITY, create_method);
+
+        let delete_local_ref = (**env).DeleteLocalRef.unwrap();
+        delete_local_ref(env, activity_class);
+
+        if decoder_obj.is_null() {
+            error!(target: "android::vid", "Failed to create VideoDecoder object");
+            return None;
+        }
+
+        let cpath = std::ffi::CString::new(path).unwrap();
+        let jpath = (**env).NewStringUTF.unwrap()(env, cpath.as_ptr());
+
+        // Get VideoDecoder class
+        let decoder_class = (**env).GetObjectClass.unwrap()(env, decoder_obj);
+
+        let init_video = (**env).GetMethodID.unwrap()(
+            env,
+            decoder_class,
+            b"init\0".as_ptr() as _,
+            b"(Ljava/lang/String;)Z\0".as_ptr() as _,
+        );
+
+        let result = (**env).CallBooleanMethod.unwrap()(env, decoder_obj, init_video, jpath);
+
+        delete_local_ref(env, jpath);
+        delete_local_ref(env, decoder_class);
+
+        if result == 0 {
+            error!(target: "android::vid", "VideoDecoder.init() failed");
+            return None;
+        }
+
+        Some(VideoDecoderHandle { obj: decoder_obj })
+    }
+}
+
+pub fn videodecoder_set_id(decoder_obj: ndk_sys::jobject, id: usize) {
+    unsafe {
+        let env = android::attach_jni_env();
+
+        let class_ptr = (**env).GetObjectClass.unwrap()(env, decoder_obj);
+
+        let method_id = (**env).GetMethodID.unwrap()(
+            env,
+            class_ptr,
+            b"setDecoderId\0".as_ptr() as _,
+            b"(I)V\0".as_ptr() as _,
+        );
+
+        (**env).CallVoidMethod.unwrap()(env, decoder_obj, method_id, id as i32);
+
+        let delete_local_ref = (**env).DeleteLocalRef.unwrap();
+        delete_local_ref(env, class_ptr);
+    }
+}
+
+pub fn videodecoder_decode_all(decoder_obj: ndk_sys::jobject) -> i32 {
+    unsafe {
+        let env = android::attach_jni_env();
+
+        let class_ptr = (**env).GetObjectClass.unwrap()(env, decoder_obj);
+
+        let method_id = (**env).GetMethodID.unwrap()(
+            env,
+            class_ptr,
+            b"decodeAll\0".as_ptr() as _,
+            b"()I\0".as_ptr() as _,
+        );
+
+        let result = (**env).CallIntMethod.unwrap()(env, decoder_obj, method_id);
+
+        let delete_local_ref = (**env).DeleteLocalRef.unwrap();
+        delete_local_ref(env, class_ptr);
+
+        result
+    }
+}

bin/app/src/app/node.rs

@@ -175,6 +175,9 @@ pub fn create_video(name: &str) -> SceneNode {
     node.add_property(prop).unwrap();
 
     let mut prop = Property::new("path", PropertyType::Str, PropertySubType::Null);
+    #[cfg(target_os = "android")]
+    prop.set_ui_text("Path", "Path to .mp4 video file (H.264 format)");
+    #[cfg(not(target_os = "android"))]
     prop.set_ui_text("Path", "Path to .ivf video file (AV1 format)");
     node.add_property(prop).unwrap();
 

bin/app/src/app/schema/mod.rs

@@ -56,7 +56,7 @@ mod ui_consts {
     use crate::android::{get_appdata_path, get_external_storage_path};
     use std::path::PathBuf;
 
-    pub const VID_PATH: &str = "forest_720x1280.ivf";
+    pub const VID_PATH: &str = "forest_720x1280.mp4";
     pub const VID_ASPECT_RATIO: f32 = 9. / 16.;
     pub use super::android_ui_consts::*;
 

bin/app/src/app/schema/test.rs

@@ -37,7 +37,7 @@ const LIGHTMODE: bool = false;
 mod ui_consts {
     //pub const CHATDB_PATH: &str = "/data/data/darkfi.app/chatdb/";
     //pub const KING_PATH: &str = "king.png";
-    pub const VID_PATH: &str = "forest_720x1280.ivf";
+    pub const VID_PATH: &str = "forest_720x1280.mp4";
 }
 
 #[cfg(not(target_os = "android"))]

bin/app/src/ui/vid/decode/android.rs

@@ -0,0 +1,128 @@
+/* This file is part of DarkFi (https://dark.fi)
+ *
+ * Copyright (C) 2020-2025 Dyne.org foundation
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+//! Android-specific video decoding using MediaCodec
+
+use miniquad::TextureFormat;
+use parking_lot::Mutex as SyncMutex;
+use std::{
+    sync::{mpsc, Arc},
+    thread,
+};
+
+use crate::{
+    android::vid::{self, DecodedFrame},
+    gfx::{gfxtag, RenderApi},
+    ui::vid::{Av1VideoData, YuvTextures},
+    util::spawn_thread,
+};
+
+macro_rules! d { ($($arg:tt)*) => { debug!(target: "ui:video::decode", $($arg)*); } }
+
+pub fn spawn_decoder_thread(
+    path: String,
+    vid_data: Arc<SyncMutex<Option<Av1VideoData>>>,
+    render_api: RenderApi,
+) -> thread::JoinHandle<()> {
+    *vid_data.lock() = Some(Av1VideoData::new(150, &render_api));
+
+    spawn_thread("video-decoder-android", move || {
+        let now = std::time::Instant::now();
+        d!("Decoding MP4 video file: {path}");
+
+        let (frame_tx, frame_rx) = mpsc::channel::<DecodedFrame>();
+
+        let decoder_id = vid::register(frame_tx);
+
+        let Some(decoder_handle) = vid::videodecoder_init(&path) else {
+            error!(target: "ui:video::decode", "Failed to initialize MediaCodec decoder for: {path}");
+            return;
+        };
+
+        vid::videodecoder_set_id(decoder_handle.obj, decoder_id);
+
+        let decoded_count = vid::videodecoder_decode_all(decoder_handle.obj);
+
+        drop(decoder_handle);
+
+        let mut frame_idx = 0;
+        while let Ok(frame) = frame_rx.recv() {
+            process_frame(frame_idx, frame, &vid_data, &render_api);
+            frame_idx += 1;
+
+            if (frame_idx % 10) == 0 {
+                let pct_loaded = 100. * frame_idx as f32 / 150.0;
+                d!("Decoded video {pct_loaded:.2}%%");
+            }
+        }
+
+        d!("Finished decoding video: {path} in {:?}", now.elapsed());
+
+        vid::unregister(decoder_id);
+
+        let vd_guard = vid_data.lock();
+        let vd = vd_guard.as_ref().unwrap();
+        for (i, tex) in vd.textures.iter().enumerate() {
+            if tex.is_none() {
+                panic!("Frame idx {i} / 150 is none for video: {path}");
+            }
+        }
+    })
+}
+
+fn process_frame(
+    frame_idx: usize,
+    frame: DecodedFrame,
+    vid_data: &SyncMutex<Option<Av1VideoData>>,
+    render_api: &RenderApi,
+) {
+    let uv_width = frame.width / 2;
+    let uv_height = frame.height / 2;
+
+    let tex_y = render_api.new_texture(
+        frame.width as u16,
+        frame.height as u16,
+        frame.y_data,
+        TextureFormat::Alpha,
+        gfxtag!("video_y"),
+    );
+
+    let tex_u = render_api.new_texture(
+        uv_width as u16,
+        uv_height as u16,
+        frame.u_data,
+        TextureFormat::Alpha,
+        gfxtag!("video_u"),
+    );
+
+    let tex_v = render_api.new_texture(
+        uv_width as u16,
+        uv_height as u16,
+        frame.v_data,
+        TextureFormat::Alpha,
+        gfxtag!("video_v"),
+    );
+
+    let yuv_texs = YuvTextures { y: tex_y, u: tex_u, v: tex_v };
+
+    let mut vd_guard = vid_data.lock();
+    let vd = vd_guard.as_mut().unwrap();
+    vd.textures[frame_idx] = Some(yuv_texs.clone());
+    let _ = vd.textures_pub.try_broadcast((frame_idx, yuv_texs));
+    d!("Stored texture for frame {}", frame_idx);
+}

bin/app/src/ui/vid/decode/mod.rs

@@ -0,0 +1,49 @@
+/* This file is part of DarkFi (https://dark.fi)
+ *
+ * Copyright (C) 2020-2025 Dyne.org foundation
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+//! Platform-specific video decoding
+
+use parking_lot::Mutex as SyncMutex;
+use std::sync::Arc;
+
+use crate::gfx::RenderApi;
+
+use super::Av1VideoData;
+
+#[cfg(target_os = "android")]
+mod android;
+
+#[cfg(not(target_os = "android"))]
+mod rav1d;
+
+/// Spawn the decoder thread
+///
+/// Platform-specific implementation:
+/// - Android: Uses MediaCodec for H.264 hardware decoding
+/// - Desktop: Uses rav1d for AV1 software decoding
+pub fn spawn_decoder_thread(
+    path: String,
+    vid_data: Arc<SyncMutex<Option<Av1VideoData>>>,
+    render_api: RenderApi,
+) -> std::thread::JoinHandle<()> {
+    #[cfg(target_os = "android")]
+    return android::spawn_decoder_thread(path, vid_data, render_api);
+
+    #[cfg(not(target_os = "android"))]
+    return rav1d::spawn_decoder_thread(path, vid_data, render_api);
+}

bin/app/src/ui/vid/decode.rs → bin/app/src/ui/vid/decode/rav1d.rs

@@ -22,21 +22,14 @@ use rav1d::{
     Decoder as Rav1dDecoder, InloopFilterType, Picture as Rav1dPicture, PlanarImageComponent,
     Rav1dError, Settings as Rav1dSettings,
 };
-use std::{
-    sync::{
-        mpsc::{Receiver, Sender},
-        Arc,
-    },
-    time::Instant,
-};
+use std::{sync::Arc, time::Instant};
 
 use crate::{
     gfx::{gfxtag, RenderApi},
+    ui::vid::{ivf::IvfStreamingDemuxer, Av1VideoData, YuvTextures},
     util::spawn_thread,
 };
 
-use super::{ivf::IvfStreamingDemuxer, Av1VideoData, YuvTextures};
-
 macro_rules! d { ($($arg:tt)*) => { debug!(target: "ui:video::decode", $($arg)*); } }
 
 /// Spawn the decoder thread (Thread 2 of 2)

bin/app/src/ui/vid/mod.rs

@@ -19,7 +19,7 @@
 use async_trait::async_trait;
 use parking_lot::Mutex as SyncMutex;
 use rand::{rngs::OsRng, Rng};
-use std::sync::{mpsc, Arc};
+use std::sync::Arc;
 use tracing::instrument;
 
 use crate::{