diff --git a/examples/hello/hello.fs b/examples/hello/hello.fs index 1571407..574821d 100644 --- a/examples/hello/hello.fs +++ b/examples/hello/hello.fs @@ -91,15 +91,22 @@ let init (_args: array) = // let renderModel = (Graphics.Scene3D.model barrelModel) |> Transform.scale 1f; // let renderModel = Model.file ("ExplodingBarrel.glb") |> Graphics.Scene3D.model |> Transform.scale 0.5f; // let modify = Model.modify (MeshSelector.all ()) (MeshOverride.material (textureMaterial)); - // let renderModel = Model.file ("vr_glove_model2.glb") |> modify |> Graphics.Scene3D.model |> Transform.scale 5f; + // let renderModel = Model.file ("vr_glove_model2.glb") |> Graphics.Scene3D.model |> Transform.scale 5f; let renderModel = - "shark.glb" + "fish.glb" |> Model.file |> Graphics.Scene3D.model |> Transform.translateZ 10.0f |> Transform.scale 0.004f; + // let renderModel = + // "shark.glb" + // |> Model.file + // |> Graphics.Scene3D.model + // |> Transform.translateZ 10.0f + // |> Transform.scale 0.004f; + group([| material (textureMaterial, [| cylinder() |> Transform.translateY -1.0f; diff --git a/runtime/functor-runtime-common/src/asset/pipelines/model_pipeline.rs b/runtime/functor-runtime-common/src/asset/pipelines/model_pipeline.rs index e1cc8d4..1c5eecf 100644 --- a/runtime/functor-runtime-common/src/asset/pipelines/model_pipeline.rs +++ b/runtime/functor-runtime-common/src/asset/pipelines/model_pipeline.rs @@ -1,4 +1,5 @@ use std::cmp::min; +use std::collections::HashMap; use std::io::Cursor; use cgmath::num_traits::ToPrimitive; @@ -224,17 +225,64 @@ fn process_node( let reader = skin.reader(|buffer| Some(&buffers[buffer.index()])); // TODO: Save inverse bind matrices with model - let _inverse_bind_matrices = reader + let inverse_bind_matrices = reader .read_inverse_bind_matrices() - .map(|v| v.collect::>()) + .map(|v| { + v.map(|mat_array| Matrix4::from(mat_array)) + .collect::>>() + }) .unwrap_or_default(); - let mut skeleton_builder = SkeletonBuilder::create(); + let joints = skin.joints().collect::>(); - let maybe_root = skin.skeleton(); + println!("SKIN: {}", skin.name().unwrap_or("NO NAME")); - if let Some(root) = maybe_root { - process_joints(&root, None, &mut skeleton_builder); + // Map joint index to position in the array + let mut joint_index_to_array_index: HashMap = HashMap::new(); + for (i, joint) in joints.iter().enumerate() { + joint_index_to_array_index.insert(joint.index(), i); + println!( + "-- joint name: {} idx: {} i: {}", + joint.name().unwrap_or("None"), + joint.index(), + i + ); + } + + // Figure out the parent index from joint index + let mut joint_index_to_parent_index: HashMap = HashMap::new(); + for joint in joints.iter() { + for children in joint.children() { + joint_index_to_parent_index.insert(children.index(), joint.index()); + } + } + + let mut skeleton_builder = SkeletonBuilder::create(inverse_bind_matrices); + + for (i, joint) in joints.iter().enumerate() { + let name = node.name().unwrap_or("None"); + let transform = node.transform().matrix().into(); + + // let parent_index = joint_index_to_parent_index.get(&joint.index()); + // let parent_id = if let Some(parent_index) = parent_index { + // Some(*joint_index_to_array_index.get(parent_index).unwrap() as i32) + // } else { + // None + // }; + + // skeleton_builder.add_joint(i as i32, name.to_owned(), parent_id, transform); + + let parent_index_i32 = joint_index_to_parent_index + .get(&joint.index()) + .map(|u| *u as i32); + + skeleton_builder.add_joint( + i, + joint.index() as i32, + name.to_owned(), + parent_index_i32, + transform, + ); } *maybe_skeleton = Some(skeleton_builder.build()); @@ -245,20 +293,12 @@ fn process_node( } } -fn process_joints( - node: &gltf::Node, - parent_id: Option, - skeleton_builder: &mut SkeletonBuilder, -) { - let id = node.index().to_i32().unwrap(); - let name = node.name().unwrap_or("None"); - let transform = node.transform().matrix().into(); - skeleton_builder.add_joint(id, name.to_owned(), parent_id, transform); - - for node in node.children() { - process_joints(&node, Some(id), skeleton_builder); - } -} +// fn process_joints( +// node: &gltf::Node, +// parent_id: Option, +// skeleton_builder: &mut SkeletonBuilder, +// ) { +// } fn process_animations(document: &gltf::Document, buffers: &[gltf::buffer::Data]) -> Vec { let mut animations = Vec::new(); diff --git a/runtime/functor-runtime-common/src/material/mod.rs b/runtime/functor-runtime-common/src/material/mod.rs index 94d00f6..10c90a2 100644 --- a/runtime/functor-runtime-common/src/material/mod.rs +++ b/runtime/functor-runtime-common/src/material/mod.rs @@ -24,8 +24,10 @@ pub trait Material { mod basic_material; mod color_material; +mod skinned_material; pub use basic_material::*; pub use color_material::*; +pub use skinned_material::*; use crate::RenderContext; diff --git a/runtime/functor-runtime-common/src/material/skinned_material.rs b/runtime/functor-runtime-common/src/material/skinned_material.rs new file mode 100644 index 0000000..ad2aaba --- /dev/null +++ b/runtime/functor-runtime-common/src/material/skinned_material.rs @@ -0,0 +1,154 @@ +use cgmath::Matrix4; + +use crate::shader_program::ShaderProgram; +use crate::shader_program::UniformLocation; +use crate::RenderContext; + +use super::Material; + +const VERTEX_SHADER_SOURCE: &str = r#" + #define MAX_JOINTS 200 + + layout (location = 0) in vec3 inPos; + layout (location = 1) in vec2 inTex; + layout (location = 2) in vec4 inJointIndices; + layout (location = 3) in vec4 inWeights; + + uniform mat4 jointTransforms[MAX_JOINTS]; + uniform mat4 world; + uniform mat4 view; + uniform mat4 projection; + + out vec2 texCoord; + out vec3 weights; + + void main() { + + // Compute the skinning matrix + mat4 skinMatrix = + inWeights.x * jointTransforms[int(inJointIndices.x)] + + inWeights.y * jointTransforms[int(inJointIndices.y)] + + inWeights.z * jointTransforms[int(inJointIndices.z)] + + inWeights.w * jointTransforms[int(inJointIndices.w)]; + + texCoord = inTex; + weights = inJointIndices.xyz; + + // Apply the skinning transformation + vec4 skinnedPos = skinMatrix * vec4(inPos, 1.0); + + gl_Position = projection * view * world * skinnedPos; + } +"#; + +const FRAGMENT_SHADER_SOURCE: &str = r#" + out vec4 fragColor; + + in vec2 texCoord; + in vec3 weights; + + uniform sampler2D texture1; + + void main() { + //fragColor = vec4(texCoord.x, texCoord.y, 0.0, 1.0); + fragColor = texture(texture1, texCoord); + //fragColor = vec4(weights.x / 50.0, weights.y / 50.0, weights.z / 50.0, 1.0); + } +"#; + +struct Uniforms { + world_loc: UniformLocation, + view_loc: UniformLocation, + projection_loc: UniformLocation, + texture_loc: UniformLocation, + joint_transforms_loc: UniformLocation, +} + +// TODO: We'll have to re-think this pattern +// Maybe we need a shader repository or something to pull from +static mut SHADER_PROGRAM: Option<(ShaderProgram, Uniforms)> = None; + +pub struct SkinnedMaterial; + +use crate::shader::Shader; +use crate::shader::ShaderType; + +impl Material for SkinnedMaterial { + fn initialize(&mut self, ctx: &RenderContext) { + unsafe { + if SHADER_PROGRAM.is_none() { + let vertex_shader = Shader::build( + ctx.gl, + ShaderType::Vertex, + VERTEX_SHADER_SOURCE, + ctx.shader_version, + ); + + // fragment shader + let fragment_shader = Shader::build( + ctx.gl, + ShaderType::Fragment, + FRAGMENT_SHADER_SOURCE, + ctx.shader_version, + ); + // link shaders + + let shader = crate::shader_program::ShaderProgram::link( + &ctx.gl, + &vertex_shader, + &fragment_shader, + ); + + let uniforms = Uniforms { + world_loc: shader.get_uniform_location(ctx.gl, "world"), + view_loc: shader.get_uniform_location(ctx.gl, "view"), + projection_loc: shader.get_uniform_location(ctx.gl, "projection"), + texture_loc: shader.get_uniform_location(ctx.gl, "texture1"), + joint_transforms_loc: shader.get_uniform_location(ctx.gl, "jointTransforms"), + }; + + SHADER_PROGRAM = Some((shader, uniforms)); + } + } + } + + fn draw_opaque( + &self, + ctx: &RenderContext, + projection_matrix: &Matrix4, + view_matrix: &Matrix4, + world_matrix: &Matrix4, + skinning_data: &[Matrix4], + ) -> bool { + unsafe { + // TODO: Find another approach to do this - maybe a shader repository? + #[allow(static_mut_refs)] + if let Some((shader, uniforms)) = &SHADER_PROGRAM { + let p = shader; + p.use_program(ctx.gl); + + p.set_uniform_matrix4(ctx.gl, &uniforms.world_loc, world_matrix); + p.set_uniform_matrix4(ctx.gl, &uniforms.view_loc, view_matrix); + p.set_uniform_matrix4(ctx.gl, &uniforms.projection_loc, projection_matrix); + p.set_uniform_1i(ctx.gl, &uniforms.texture_loc, 0); + + let num_joints = skinning_data.len(); + let mut joint_matrices = Vec::with_capacity(num_joints * 16); + for i in 0..num_joints { + let matrix_array: &[f32; 16] = skinning_data[i].as_ref(); + joint_matrices.extend_from_slice(matrix_array); + } + + p.set_uniform_matrix4fv(ctx.gl, &uniforms.joint_transforms_loc, &joint_matrices); + } + } + + true + } +} + +impl SkinnedMaterial { + pub fn create() -> Box { + Box::new(SkinnedMaterial) + } +} diff --git a/runtime/functor-runtime-common/src/model/skeleton.rs b/runtime/functor-runtime-common/src/model/skeleton.rs index 51f6985..44b23ab 100644 --- a/runtime/functor-runtime-common/src/model/skeleton.rs +++ b/runtime/functor-runtime-common/src/model/skeleton.rs @@ -18,6 +18,10 @@ pub struct Skeleton { // Use HashMap for joints because they could be sparse joint_info: HashMap, joint_absolute_transform: HashMap>, + + array_index_to_joint_id: HashMap, + + inverse_bind_matrices: Vec>, } impl Skeleton { @@ -26,6 +30,8 @@ impl Skeleton { num_joints: 0, joint_info: HashMap::new(), joint_absolute_transform: HashMap::new(), + inverse_bind_matrices: Vec::new(), + array_index_to_joint_id: HashMap::new(), } } @@ -140,6 +146,18 @@ impl Skeleton { } vec } + + pub fn get_skinning_transforms(&self) -> Vec> { + let mut vec = Vec::new(); + for i in 0..self.inverse_bind_matrices.len() { + let joint_idx = self.array_index_to_joint_id.get(&i).unwrap(); + vec.push( + self.get_joint_absolute_transform(*joint_idx) + * self.inverse_bind_matrices.get(i as usize).unwrap(), + ) + } + vec + } } pub struct SkeletonBuilder { @@ -147,10 +165,12 @@ pub struct SkeletonBuilder { } impl SkeletonBuilder { - pub fn create() -> SkeletonBuilder { + pub fn create(inverse_bind_matrices: Vec>) -> SkeletonBuilder { SkeletonBuilder { skeleton: Skeleton { - num_joints: 0, + array_index_to_joint_id: HashMap::new(), + num_joints: inverse_bind_matrices.len() as i32, + inverse_bind_matrices, joint_info: HashMap::new(), joint_absolute_transform: HashMap::new(), }, @@ -159,11 +179,15 @@ impl SkeletonBuilder { pub fn add_joint( &mut self, + array_index: usize, joint_index: i32, name: String, parent_index: Option, transform: Matrix4, ) { + self.skeleton + .array_index_to_joint_id + .insert(array_index, joint_index); let joint = Joint { name, transform, @@ -303,12 +327,18 @@ mod tests { let transform_joint_1 = Matrix4::from_translation(Vector3::new(0.0, 2.0, 0.0)); let transform_joint_2 = Matrix4::from_translation(Vector3::new(0.0, 0.0, 3.0)); + let inv_bind_matrices = vec![ + Matrix4::identity(), + Matrix4::identity(), + Matrix4::identity(), + ]; + // Create the SkinBuilder - let mut skin_builder = SkeletonBuilder::create(); + let mut skin_builder = SkeletonBuilder::create(inv_bind_matrices); - skin_builder.add_joint(0, "Joint0".to_string(), None, transform_joint_0); - skin_builder.add_joint(1, "Joint1".to_string(), Some(0), transform_joint_1); - skin_builder.add_joint(2, "Joint2".to_string(), Some(1), transform_joint_2); + skin_builder.add_joint(0, 0, "Joint0".to_string(), None, transform_joint_0); + skin_builder.add_joint(1, 1, "Joint1".to_string(), Some(0), transform_joint_1); + skin_builder.add_joint(2, 2, "Joint2".to_string(), Some(1), transform_joint_2); // Build the Skin let skin = skin_builder.build(); diff --git a/runtime/functor-runtime-common/src/scene3d/mod.rs b/runtime/functor-runtime-common/src/scene3d/mod.rs index cda779f..7e041d3 100644 --- a/runtime/functor-runtime-common/src/scene3d/mod.rs +++ b/runtime/functor-runtime-common/src/scene3d/mod.rs @@ -12,7 +12,7 @@ use crate::{ AssetHandle, BuiltAssetPipeline, }, geometry::{self, Geometry, Mesh}, - material::{BasicMaterial, ColorMaterial, Material}, + material::{ColorMaterial, Material, SkinnedMaterial}, math::Angle, model::{Model, Skeleton}, texture::{RuntimeTexture, Texture2D}, @@ -166,7 +166,7 @@ impl Scene3D { let skinning_data = vec![]; match &self.obj { SceneObject::Model(model_description) => { - let mut basic_material = BasicMaterial::create(); + let mut basic_material = SkinnedMaterial::create(); basic_material.initialize(&render_context); match &model_description.handle { @@ -215,6 +215,24 @@ impl Scene3D { &[], ); } else { + let maybe_animation = hydrated_model.animations.get(0); + let joints = if let Some(animation) = maybe_animation { + let time = render_context.frame_time.tts % animation.duration; + let animated_skeleton = Skeleton::animate( + &hydrated_model.skeleton, + animation, + time, + ); + animated_skeleton.get_skinning_transforms() + } else { + let mut joints = Vec::new(); + + for i in 0..50 { + joints.push(Matrix4::identity()); + } + joints + }; + // Bind textures mesh.base_color_texture.bind(0, &render_context); basic_material.draw_opaque( @@ -222,7 +240,7 @@ impl Scene3D { projection_matrix, view_matrix, &matrix, - &[], + &joints, ); }; diff --git a/runtime/functor-runtime-common/src/shader_program.rs b/runtime/functor-runtime-common/src/shader_program.rs index efe94dd..6f8a3db 100644 --- a/runtime/functor-runtime-common/src/shader_program.rs +++ b/runtime/functor-runtime-common/src/shader_program.rs @@ -44,7 +44,7 @@ impl ShaderProgram { unsafe { let native_uniform_location = gl .get_uniform_location(self.program_id, uniform_name) - .expect("Cannot get uniform location"); + .expect(&format!("Cannot get uniform location: {}", &uniform_name)); UniformLocation { native_uniform_location, } @@ -103,4 +103,15 @@ impl ShaderProgram { ); } } + + pub fn set_uniform_matrix4fv( + &self, + gl: &glow::Context, + location: &UniformLocation, + values: &[f32], + ) { + unsafe { + gl.uniform_matrix_4_f32_slice(Some(&location.native_uniform_location), false, values) + } + } }