feat(material): Implement initial skinned material

examples/hello/hello.fs

@@ -91,15 +91,22 @@ let init (_args: array<string>) =
         // let renderModel = (Graphics.Scene3D.model barrelModel) |> Transform.scale 1f;
         // let renderModel = Model.file ("ExplodingBarrel.glb") |> Graphics.Scene3D.model |> Transform.scale 0.5f;
         // let modify = Model.modify (MeshSelector.all ()) (MeshOverride.material (textureMaterial));
-        // let renderModel = Model.file ("vr_glove_model2.glb") |> modify |> Graphics.Scene3D.model |> Transform.scale 5f;
+        // let renderModel = Model.file ("vr_glove_model2.glb") |> Graphics.Scene3D.model |> Transform.scale 5f;
 
         let renderModel = 
-            "shark.glb"
+            "fish.glb"
             |> Model.file 
             |> Graphics.Scene3D.model 
             |> Transform.translateZ 10.0f
             |> Transform.scale 0.004f;
 
+        // let renderModel = 
+        //     "shark.glb"
+        //     |> Model.file 
+        //     |> Graphics.Scene3D.model 
+        //     |> Transform.translateZ 10.0f
+        //     |> Transform.scale 0.004f;
+
         group([|
             material (textureMaterial, [|
                 cylinder() |> Transform.translateY -1.0f;

runtime/functor-runtime-common/src/asset/pipelines/model_pipeline.rs

@@ -1,4 +1,5 @@
 use std::cmp::min;
+use std::collections::HashMap;
 use std::io::Cursor;
 
 use cgmath::num_traits::ToPrimitive;
@@ -224,17 +225,64 @@ fn process_node(
         let reader = skin.reader(|buffer| Some(&buffers[buffer.index()]));
 
         // TODO: Save inverse bind matrices with model
-        let _inverse_bind_matrices = reader
+        let inverse_bind_matrices = reader
             .read_inverse_bind_matrices()
-            .map(|v| v.collect::<Vec<_>>())
+            .map(|v| {
+                v.map(|mat_array| Matrix4::from(mat_array))
+                    .collect::<Vec<Matrix4<f32>>>()
+            })
             .unwrap_or_default();
 
-        let mut skeleton_builder = SkeletonBuilder::create();
+        let joints = skin.joints().collect::<Vec<_>>();
 
-        let maybe_root = skin.skeleton();
+        println!("SKIN: {}", skin.name().unwrap_or("NO NAME"));
 
-        if let Some(root) = maybe_root {
-            process_joints(&root, None, &mut skeleton_builder);
+        // Map joint index to position in the array
+        let mut joint_index_to_array_index: HashMap<usize, usize> = HashMap::new();
+        for (i, joint) in joints.iter().enumerate() {
+            joint_index_to_array_index.insert(joint.index(), i);
+            println!(
+                "-- joint name: {} idx: {} i: {}",
+                joint.name().unwrap_or("None"),
+                joint.index(),
+                i
+            );
+        }
+
+        // Figure out the parent index from joint index
+        let mut joint_index_to_parent_index: HashMap<usize, usize> = HashMap::new();
+        for joint in joints.iter() {
+            for children in joint.children() {
+                joint_index_to_parent_index.insert(children.index(), joint.index());
+            }
+        }
+
+        let mut skeleton_builder = SkeletonBuilder::create(inverse_bind_matrices);
+
+        for (i, joint) in joints.iter().enumerate() {
+            let name = node.name().unwrap_or("None");
+            let transform = node.transform().matrix().into();
+
+            // let parent_index = joint_index_to_parent_index.get(&joint.index());
+            // let parent_id = if let Some(parent_index) = parent_index {
+            //     Some(*joint_index_to_array_index.get(parent_index).unwrap() as i32)
+            // } else {
+            //     None
+            // };
+
+            // skeleton_builder.add_joint(i as i32, name.to_owned(), parent_id, transform);
+
+            let parent_index_i32 = joint_index_to_parent_index
+                .get(&joint.index())
+                .map(|u| *u as i32);
+
+            skeleton_builder.add_joint(
+                i,
+                joint.index() as i32,
+                name.to_owned(),
+                parent_index_i32,
+                transform,
+            );
         }
 
         *maybe_skeleton = Some(skeleton_builder.build());
@@ -245,20 +293,12 @@ fn process_node(
     }
 }
 
-fn process_joints(
-    node: &gltf::Node,
-    parent_id: Option<i32>,
-    skeleton_builder: &mut SkeletonBuilder,
-) {
-    let id = node.index().to_i32().unwrap();
-    let name = node.name().unwrap_or("None");
-    let transform = node.transform().matrix().into();
-    skeleton_builder.add_joint(id, name.to_owned(), parent_id, transform);
-
-    for node in node.children() {
-        process_joints(&node, Some(id), skeleton_builder);
-    }
-}
+// fn process_joints(
+//     node: &gltf::Node,
+//     parent_id: Option<i32>,
+//     skeleton_builder: &mut SkeletonBuilder,
+// ) {
+// }
 
 fn process_animations(document: &gltf::Document, buffers: &[gltf::buffer::Data]) -> Vec<Animation> {
     let mut animations = Vec::new();

runtime/functor-runtime-common/src/material/mod.rs

@@ -24,8 +24,10 @@ pub trait Material {
 
 mod basic_material;
 mod color_material;
+mod skinned_material;
 
 pub use basic_material::*;
 pub use color_material::*;
+pub use skinned_material::*;
 
 use crate::RenderContext;

runtime/functor-runtime-common/src/material/skinned_material.rs

@@ -0,0 +1,154 @@
+use cgmath::Matrix4;
+
+use crate::shader_program::ShaderProgram;
+use crate::shader_program::UniformLocation;
+use crate::RenderContext;
+
+use super::Material;
+
+const VERTEX_SHADER_SOURCE: &str = r#"
+        #define MAX_JOINTS 200
+
+        layout (location = 0) in vec3 inPos;
+        layout (location = 1) in vec2 inTex;
+        layout (location = 2) in vec4 inJointIndices;
+        layout (location = 3) in vec4 inWeights;
+
+        uniform mat4 jointTransforms[MAX_JOINTS];
+        uniform mat4 world;
+        uniform mat4 view;
+        uniform mat4 projection;
+
+        out vec2 texCoord;
+        out vec3 weights;
+
+        void main() {
+
+            // Compute the skinning matrix
+            mat4 skinMatrix = 
+                inWeights.x * jointTransforms[int(inJointIndices.x)] +
+                inWeights.y * jointTransforms[int(inJointIndices.y)] +
+                inWeights.z * jointTransforms[int(inJointIndices.z)] +
+                inWeights.w * jointTransforms[int(inJointIndices.w)];
+
+            texCoord = inTex;
+            weights = inJointIndices.xyz;
+
+            // Apply the skinning transformation
+            vec4 skinnedPos = skinMatrix * vec4(inPos, 1.0);
+
+            gl_Position = projection * view * world * skinnedPos;
+        }
+"#;
+
+const FRAGMENT_SHADER_SOURCE: &str = r#"
+        out vec4 fragColor;
+
+        in vec2 texCoord;
+        in vec3 weights;
+
+        uniform sampler2D texture1;
+
+        void main() {
+            //fragColor = vec4(texCoord.x, texCoord.y, 0.0, 1.0);
+            fragColor = texture(texture1, texCoord);
+            //fragColor = vec4(weights.x / 50.0, weights.y / 50.0, weights.z / 50.0, 1.0);
+        }
+"#;
+
+struct Uniforms {
+    world_loc: UniformLocation,
+    view_loc: UniformLocation,
+    projection_loc: UniformLocation,
+    texture_loc: UniformLocation,
+    joint_transforms_loc: UniformLocation,
+}
+
+// TODO: We'll have to re-think this pattern
+// Maybe we need a shader repository or something to pull from
+static mut SHADER_PROGRAM: Option<(ShaderProgram, Uniforms)> = None;
+
+pub struct SkinnedMaterial;
+
+use crate::shader::Shader;
+use crate::shader::ShaderType;
+
+impl Material for SkinnedMaterial {
+    fn initialize(&mut self, ctx: &RenderContext) {
+        unsafe {
+            if SHADER_PROGRAM.is_none() {
+                let vertex_shader = Shader::build(
+                    ctx.gl,
+                    ShaderType::Vertex,
+                    VERTEX_SHADER_SOURCE,
+                    ctx.shader_version,
+                );
+
+                // fragment shader
+                let fragment_shader = Shader::build(
+                    ctx.gl,
+                    ShaderType::Fragment,
+                    FRAGMENT_SHADER_SOURCE,
+                    ctx.shader_version,
+                );
+                // link shaders
+
+                let shader = crate::shader_program::ShaderProgram::link(
+                    &ctx.gl,
+                    &vertex_shader,
+                    &fragment_shader,
+                );
+
+                let uniforms = Uniforms {
+                    world_loc: shader.get_uniform_location(ctx.gl, "world"),
+                    view_loc: shader.get_uniform_location(ctx.gl, "view"),
+                    projection_loc: shader.get_uniform_location(ctx.gl, "projection"),
+                    texture_loc: shader.get_uniform_location(ctx.gl, "texture1"),
+                    joint_transforms_loc: shader.get_uniform_location(ctx.gl, "jointTransforms"),
+                };
+
+                SHADER_PROGRAM = Some((shader, uniforms));
+            }
+        }
+    }
+
+    fn draw_opaque(
+        &self,
+        ctx: &RenderContext,
+        projection_matrix: &Matrix4<f32>,
+        view_matrix: &Matrix4<f32>,
+        world_matrix: &Matrix4<f32>,
+        skinning_data: &[Matrix4<f32>],
+    ) -> bool {
+        unsafe {
+            // TODO: Find another approach to do this - maybe a shader repository?
+            #[allow(static_mut_refs)]
+            if let Some((shader, uniforms)) = &SHADER_PROGRAM {
+                let p = shader;
+                p.use_program(ctx.gl);
+
+                p.set_uniform_matrix4(ctx.gl, &uniforms.world_loc, world_matrix);
+                p.set_uniform_matrix4(ctx.gl, &uniforms.view_loc, view_matrix);
+                p.set_uniform_matrix4(ctx.gl, &uniforms.projection_loc, projection_matrix);
+                p.set_uniform_1i(ctx.gl, &uniforms.texture_loc, 0);
+
+                let num_joints = skinning_data.len();
+                let mut joint_matrices = Vec::with_capacity(num_joints * 16);
+                for i in 0..num_joints {
+                    let matrix_array: &[f32; 16] = skinning_data[i].as_ref();
+                    joint_matrices.extend_from_slice(matrix_array);
+                }
+
+                p.set_uniform_matrix4fv(ctx.gl, &uniforms.joint_transforms_loc, &joint_matrices);
+            }
+        }
+
+        true
+    }
+}
+
+impl SkinnedMaterial {
+    pub fn create() -> Box<dyn Material> {
+        Box::new(SkinnedMaterial)
+    }
+}

runtime/functor-runtime-common/src/model/skeleton.rs

@@ -18,6 +18,10 @@ pub struct Skeleton {
     // Use HashMap for joints because they could be sparse
     joint_info: HashMap<i32, Joint>,
     joint_absolute_transform: HashMap<i32, Matrix4<f32>>,
+
+    array_index_to_joint_id: HashMap<usize, i32>,
+
+    inverse_bind_matrices: Vec<Matrix4<f32>>,
 }
 
 impl Skeleton {
@@ -26,6 +30,8 @@ impl Skeleton {
             num_joints: 0,
             joint_info: HashMap::new(),
             joint_absolute_transform: HashMap::new(),
+            inverse_bind_matrices: Vec::new(),
+            array_index_to_joint_id: HashMap::new(),
         }
     }
 
@@ -140,17 +146,31 @@ impl Skeleton {
         }
         vec
     }
+
+    pub fn get_skinning_transforms(&self) -> Vec<Matrix4<f32>> {
+        let mut vec = Vec::new();
+        for i in 0..self.inverse_bind_matrices.len() {
+            let joint_idx = self.array_index_to_joint_id.get(&i).unwrap();
+            vec.push(
+                self.get_joint_absolute_transform(*joint_idx)
+                    * self.inverse_bind_matrices.get(i as usize).unwrap(),
+            )
+        }
+        vec
+    }
 }
 
 pub struct SkeletonBuilder {
     skeleton: Skeleton,
 }
 
 impl SkeletonBuilder {
-    pub fn create() -> SkeletonBuilder {
+    pub fn create(inverse_bind_matrices: Vec<Matrix4<f32>>) -> SkeletonBuilder {
         SkeletonBuilder {
             skeleton: Skeleton {
-                num_joints: 0,
+                array_index_to_joint_id: HashMap::new(),
+                num_joints: inverse_bind_matrices.len() as i32,
+                inverse_bind_matrices,
                 joint_info: HashMap::new(),
                 joint_absolute_transform: HashMap::new(),
             },
@@ -159,11 +179,15 @@ impl SkeletonBuilder {
 
     pub fn add_joint(
         &mut self,
+        array_index: usize,
         joint_index: i32,
         name: String,
         parent_index: Option<i32>,
         transform: Matrix4<f32>,
     ) {
+        self.skeleton
+            .array_index_to_joint_id
+            .insert(array_index, joint_index);
         let joint = Joint {
             name,
             transform,
@@ -303,12 +327,18 @@ mod tests {
         let transform_joint_1 = Matrix4::from_translation(Vector3::new(0.0, 2.0, 0.0));
         let transform_joint_2 = Matrix4::from_translation(Vector3::new(0.0, 0.0, 3.0));
 
+        let inv_bind_matrices = vec![
+            Matrix4::identity(),
+            Matrix4::identity(),
+            Matrix4::identity(),
+        ];
+
         // Create the SkinBuilder
-        let mut skin_builder = SkeletonBuilder::create();
+        let mut skin_builder = SkeletonBuilder::create(inv_bind_matrices);
 
-        skin_builder.add_joint(0, "Joint0".to_string(), None, transform_joint_0);
-        skin_builder.add_joint(1, "Joint1".to_string(), Some(0), transform_joint_1);
-        skin_builder.add_joint(2, "Joint2".to_string(), Some(1), transform_joint_2);
+        skin_builder.add_joint(0, 0, "Joint0".to_string(), None, transform_joint_0);
+        skin_builder.add_joint(1, 1, "Joint1".to_string(), Some(0), transform_joint_1);
+        skin_builder.add_joint(2, 2, "Joint2".to_string(), Some(1), transform_joint_2);
 
         // Build the Skin
         let skin = skin_builder.build();