three.js raycast on skinning mesh

Question

I'm trying to raycast skinning mesh (of knowing issue) after some skeleton changes (without animation on it, so performance isn't a priority).

The tricky thing i imagine in this attempt is:

Load skinned mesh add to scene
Make some changes in positions of specific bones at loaded mesh
Copy geometries of transformed loaded mesh (maybe from buffer?)
Create new mesh (some kind of imitation ghost mesh) from copied geometries and apply to it
set raycast on ghost mesh with opacity material= 0.0

Above list should work, but I'm stuck third day on point 3 cause I can't get transformed vertices after skinning.

var scene, camera, renderer, mesh, ghostMesh;

var raycaster = new THREE.Raycaster();
var raycasterMeshHelper;

initScene();
render();

function initScene() {
  scene = new THREE.Scene();
  camera = new THREE.PerspectiveCamera(75, window.innerWidth / window.innerHeight, 0.1, 200);
  camera.position.set(20, 7, 3);
  renderer = new THREE.WebGLRenderer({
    antialias: true
  });
  renderer.setPixelRatio(window.devicePixelRatio);
  renderer.setSize(window.innerWidth, window.innerHeight);

  document.body.appendChild(renderer.domElement);
  window.addEventListener('resize', onWindowResize, false);

  var orbit = new THREE.OrbitControls(camera, renderer.domElement);

  //lights stuff
  var ambientLight = new THREE.AmbientLight(0xffffff, 0.3);
  scene.add(ambientLight);
  var lights = [];
  lights[0] = new THREE.PointLight(0xffffff, 1, 0);
  lights[1] = new THREE.PointLight(0xffffff, 1, 0);
  lights[2] = new THREE.PointLight(0xffffff, 1, 0);
  lights[0].position.set(0, 200, 0);
  lights[1].position.set(100, 200, 100);
  lights[2].position.set(-100, -200, -100);
  scene.add(lights[0]);
  scene.add(lights[1]);
  scene.add(lights[2]);

  //raycaster mesh 
  var raycasterMaterial = new THREE.MeshBasicMaterial({
    color: 0xdddddd,
    opacity: 0.7,
    transparent: true
  });
  var geometrySphere = new THREE.SphereGeometry(0.5, 16, 16);
  raycasterMeshHelper = new THREE.Mesh(geometrySphere, raycasterMaterial);
  raycasterMeshHelper.visible = false;
  scene.add(raycasterMeshHelper);

  renderer.domElement.addEventListener('mousemove', onMouseMove, false);

  //model Loading

  var loader = new THREE.JSONLoader();
  loader.load("https://raw.githubusercontent.com/visus100/skinnedTests/master/js_fiddle/skinned_mesh.json", function(geometry) {
    var meshMaterial = new THREE.MeshStandardMaterial({
      color: 0x00df15,
      skinning: true
    });

    mesh = new THREE.SkinnedMesh(geometry, meshMaterial);
    scene.add(mesh);

    var skeleton = new THREE.SkeletonHelper(mesh);
    scene.add(skeleton);

    //some experimental skeletonal changes
    mesh.skeleton.bones[1].rotation.z += 0.10;
    mesh.skeleton.bones[2].rotation.x += -0.65;
    mesh.skeleton.bones[3].rotation.y += -0.45;
    mesh.skeleton.bones[3].position.x += 0.11;

    //updates matrix
    mesh.updateMatrix();
    mesh.geometry.applyMatrix(mesh.matrix);
    mesh.updateMatrixWorld(true);

    //crate ghost mesh geometry
    createGhostMesh();

    //crate point cloud helper from buffergeometry
    var bufferGeometry = new THREE.BufferGeometry().fromGeometry(mesh.geometry);

    var particesMaterial = new THREE.PointsMaterial({
      color: 0xff00ea,
      size: 0.07,
      sizeAttenuation: false
    });
    particles = new THREE.Points(bufferGeometry, particesMaterial);
    particles.sortParticles = true;
    scene.add(particles);

  });
}

function createGhostMesh() {
  var geometryForGhostMesh = new THREE.Geometry();

  //push vertices and other stuff to geometry
  for (i = 0; i < mesh.geometry.vertices.length; i++) {
    var temp = new THREE.Vector3(mesh.geometry.vertices[i].x, mesh.geometry.vertices[i].y, mesh.geometry.vertices[i].z);
    geometryForGhostMesh.vertices.push(temp);

    //////
    //here should be the code for calc translation vertices of skinned mesh and added to geometryForGhostMesh
    //////

    geometryForGhostMesh.skinIndices.push(mesh.geometry.skinIndices[i]);
    geometryForGhostMesh.skinWeights.push(mesh.geometry.skinWeights[i]);
  }

  for (i = 0; i < mesh.geometry.faces.length; i++) {
    geometryForGhostMesh.faces.push(mesh.geometry.faces[i]);
  }

  //create material and add to scene

  var ghostMaterial = new THREE.MeshBasicMaterial({
    color: 0xff0000,
    opacity: 0.1,
    transparent: true,
    skinning: true
  });
  ghostMesh = new THREE.Mesh(geometryForGhostMesh, ghostMaterial);
  scene.add(ghostMesh);
}

function onWindowResize() {
  camera.aspect = window.innerWidth / window.innerHeight;
  camera.updateProjectionMatrix();
  renderer.setSize(window.innerWidth, window.innerHeight);
}

function render() {
  requestAnimationFrame(render);
  renderer.render(scene, camera);
};

function onMouseMove(event) {
  //raycaster for ghostMesh 
  if (ghostMesh) {
    var rect = renderer.domElement.getBoundingClientRect();
    var mouseX = ((event.clientX - rect.left) / rect.width) * 2 - 1;
    var mouseY = -((event.clientY - rect.top) / rect.height) * 2 + 1;
    raycaster.setFromCamera(new THREE.Vector2(mouseX, mouseY), camera);

    var intersects = raycaster.intersectObject(ghostMesh);
    if (intersects.length > 0) {
      raycasterMeshHelper.visible = true;
      raycasterMeshHelper.position.set(0, 0, 0);
      raycasterMeshHelper.lookAt(intersects[0].face.normal);
      raycasterMeshHelper.position.copy(intersects[0].point);
    } else {
      raycasterMeshHelper.visible = false;
    }
  }
}

body {
  margin: 0px;
  background-color: #000000;
  overflow: hidden;
}

<script src="https://cdnjs.cloudflare.com/ajax/libs/three.js/98/three.min.js"></script>
<script src="https://threejs.org/examples/js/controls/OrbitControls.js"></script>

Please note that I need this in thre.js build r98 or less, because the rest of my code (not included here) and without morph tangents only skinning bones.

I tried to write it clearly and please if anyone want help do it so because I'm not a pro.

I not including my approach of calculating transformed geometries because I failed too hard.

I dug a lot about this problem here e.g. issue6440 and for today it's still not fixed.

But there existing methods to work with it e.g https://jsfiddle.net/fnjkeg9x/1/ but after several of attempts I failed and my conclusion is that the stormtrooper works on morph tanges and this could be the reason I failed.

EDIT:

I created next codepen based on this topics get-the-global-position-of-a-vertex-of-a-skinned-mesh and Stormtrooper. Decided to start with simple box to make bounding around skinned transformed mesh.

Result is fail because it giving 0 at line:
boneMatrix.fromArray(skeleton.boneMatrices, si * 16);
Here i comparing stormtrooper with my example output from console: Screen shot image

Codpen with new progress: https://codepen.io/donkeyLuck0/pen/XQbBMQ

My other idea is to apply this bones form loaded model and rig as a morph tangent programmatically (but i don't even know if it is possible and how to figure it out)

Founded example of animated model Sketchfab animation with points tracking

If you are trying to raycast for picking you can use GPU picking which is described toward the bottom of [this article](https://threejsfundamentals.org/threejs/lessons/threejs-picking.html) — gman, Apr 01 '19 at 22:12
In fact i need a THREE.Vector3( ) mouse click position of a transformed skinned mesh (that why i tried to do something like ghost mesh). I analysed the artice u posted which i'm very glad. I will try to do some tests about it but i need some time. It would be very useful to me if i could get your opinion while i will build. Do you think that gpu picking will work in my case? — DonkeyLuck0, Apr 01 '19 at 23:09
GPU picking as implemented in that article will only give you which object was picked. It will not give you any kind of position. To get position with GPU picking you'd have to write a custom shader. You'd basically use GPU picking from that article, then which ever object it says was picked you'd render with a custom shader that outputs positions instead of colors. Like the GPU picking you'd only need to render 1 pixel. You'd then read the pixel to get the position. — gman, Apr 02 '19 at 00:50
Thank you for your advices, I'm not in advanced level to write own shaders so still counting for my topic solution ( calculate transformed mesh and apply to another) which would be great. Also i will take a look for three.js example https://github.com/mrdoob/three.js/blob/master/examples/webgl_interactive_cubes_gpu.html then see if i can do something with it. I will try to post result when it's ready. — DonkeyLuck0, Apr 02 '19 at 01:21

score 1 · Answer 1 · answered Mar 02 '20 at 01:03

This is super late to the game, but here's an example of GPU picking that works with skinned meshes and doesn't require a separate picking scene to keep in sync with your main scene, nor does it require the user to manage custom materials:

https://github.com/bzztbomb/three_js_gpu_picking

The trick that allows for easy material overriding and scene re-use is here: https://github.com/bzztbomb/three_js_gpu_picking/blob/master/gpupicker.js#L58

score 1 · Answer 2 · answered Apr 20 '21 at 16:56

1

A proper support for raycasting for skinned meshes was added in https://github.com/mrdoob/three.js/pull/19178 in revision 116.

answered Apr 20 '21 at 16:56

Suma

33,181
16
123
191

gman · Answer 3 · 2020-01-09T20:09:56.223

You can use GPU picking to "pick" skinned object. It won't give you a position though

Note: GPU picking requires rendering every pickable object with a custom material. How you implement that is up to you. This article does it by making 2 scenes. That might not be as useful for skinned objects.

Unfortunately three.js provides no way to override materials AFAICT. Here's an example that replaces the materials on the pickable objects before rendering for picking and then restores them after. You would also need to hide any objects you don't want picked.

const renderer = new THREE.WebGLRenderer({
  antialias: true,
  canvas: document.querySelector('canvas'),
});
const scene = new THREE.Scene();
const camera = new THREE.PerspectiveCamera(75, 2, 0.1, 200);
camera.position.set(20, 7, 3);

const orbit = new THREE.OrbitControls(camera, renderer.domElement);

//lights stuff
const ambientLight = new THREE.AmbientLight(0xffffff, 0.3);
scene.add(ambientLight);
const lights = [];
lights[0] = new THREE.PointLight(0xffffff, 1, 0);
lights[1] = new THREE.PointLight(0xffffff, 1, 0);
lights[2] = new THREE.PointLight(0xffffff, 1, 0);
lights[0].position.set(0, 200, 0);
lights[1].position.set(100, 200, 100);
lights[2].position.set(-100, -200, -100);
scene.add(lights[0]);
scene.add(lights[1]);
scene.add(lights[2]);


//raycaster mesh 
const raycasterMaterial = new THREE.MeshBasicMaterial({
  color: 0xdddddd,
  opacity: 0.7,
  transparent: true
});
const geometrySphere = new THREE.SphereGeometry(0.5, 16, 16);
raycasterMeshHelper = new THREE.Mesh(geometrySphere, raycasterMaterial);
raycasterMeshHelper.visible = false;
scene.add(raycasterMeshHelper);


//model Loading
const pickableObjects = [];

const loader = new THREE.JSONLoader();
loader.load("https://raw.githubusercontent.com/visus100/skinnedTests/master/js_fiddle/skinned_mesh.json", (geometry) => {
  const meshMaterial = new THREE.MeshStandardMaterial({
    color: 0x00df15,
    skinning: true
  });

  const mesh = new THREE.SkinnedMesh(geometry, meshMaterial);
  scene.add(mesh);

  const id = pickableObjects.length + 1;
  pickableObjects.push({
    mesh,
    renderingMaterial: meshMaterial,
    pickingMaterial: new THREE.MeshPhongMaterial({
      skinning: true,
      emissive: new THREE.Color(id),
      color: new THREE.Color(0, 0, 0),
      specular: new THREE.Color(0, 0, 0),
      //map: texture,
      //transparent: true,
      //side: THREE.DoubleSide,
      //alphaTest: 0.5,
      blending: THREE.NoBlending,
    }),
  });

  //some experimental skeletonal changes
  mesh.skeleton.bones[1].rotation.z += 0.10;
  mesh.skeleton.bones[2].rotation.x += -0.65;
  mesh.skeleton.bones[3].rotation.y += -0.45;
  mesh.skeleton.bones[3].position.x += 0.11;

  //updates matrix
  mesh.updateMatrix();
  mesh.geometry.applyMatrix(mesh.matrix);
  mesh.updateMatrixWorld(true);

});

class GPUPickHelper {
  constructor() {
    // create a 1x1 pixel render target
    this.pickingTexture = new THREE.WebGLRenderTarget(1, 1);
    this.pixelBuffer = new Uint8Array(4);
  }
  pick(cssPosition, scene, camera) {
    const {
      pickingTexture,
      pixelBuffer
    } = this;

    // set the view offset to represent just a single pixel under the mouse
    const pixelRatio = renderer.getPixelRatio();
    camera.setViewOffset(
      renderer.context.drawingBufferWidth, // full width
      renderer.context.drawingBufferHeight, // full top
      cssPosition.x * pixelRatio | 0, // rect x
      cssPosition.y * pixelRatio | 0, // rect y
      1, // rect width
      1, // rect height
    );
    // render the scene
    // r102
    //renderer.setRenderTarget(pickingTexture);
    //renderer.render(scene, camera);
    //renderer.setRenderTarget(null);
    // r98
    renderer.render(scene, camera, pickingTexture);
    // clear the view offset so rendering returns to normal
    camera.clearViewOffset();
    //read the pixel
    renderer.readRenderTargetPixels(
      pickingTexture,
      0, // x
      0, // y
      1, // width
      1, // height
      pixelBuffer);

    const id =
      (pixelBuffer[0] << 16) |
      (pixelBuffer[1] << 8) |
      (pixelBuffer[2]);
    return id;
  }
}

function resizeRendererToDisplaySize(renderer) {
  const canvas = renderer.domElement;
  const width = canvas.clientWidth;
  const height = canvas.clientHeight;
  const needResize = canvas.width !== width || canvas.height !== height;
  if (needResize) {
    renderer.setSize(width, height, false);
  }
  return needResize;
}

const pickPosition = {
  x: 0,
  y: 0,
};
const pickHelper = new GPUPickHelper();
let lastPickedId = 0;
let lastPickedObjectSavedEmissive;

function render(time) {
  time *= 0.001;  // convert to seconds;

  if (resizeRendererToDisplaySize(renderer)) {
    const canvas = renderer.domElement;
    camera.aspect = canvas.clientWidth / canvas.clientHeight;
    camera.updateProjectionMatrix();
  }

  if (lastPickedId) {
    pickableObjects[lastPickedId - 1].renderingMaterial.emissive.setHex(lastPickedObjectSavedEmissive);
    lastPickedId = 0;
  }

  for (pickableObject of pickableObjects) {
    pickableObject.mesh.material = pickableObject.pickingMaterial;
  }
  
  const id = pickHelper.pick(pickPosition, scene, camera, time);
  
  for (pickableObject of pickableObjects) {
    pickableObject.mesh.material = pickableObject.renderingMaterial;
  }
  
  const pickedObject = pickableObjects[id - 1];
  if (pickedObject) {
    lastPickedId = id;
    lastPickedObjectSavedEmissive = pickedObject.renderingMaterial.emissive.getHex();
    pickedObject.renderingMaterial.emissive.setHex((time * 8) % 2 > 1 ? 0xFFFF00 : 0xFF0000);
  }
  
  renderer.render(scene, camera);

  requestAnimationFrame(render);
};
requestAnimationFrame(render);

function setPickPosition(event) {
  pickPosition.x = event.clientX;
  pickPosition.y = event.clientY;
}

function clearPickPosition() {
  // unlike the mouse which always has a position
  // if the user stops touching the screen we want
  // to stop picking. For now we just pick a value
  // unlikely to pick something
  pickPosition.x = -100000;
  pickPosition.y = -100000;
}

window.addEventListener('mousemove', setPickPosition);
window.addEventListener('mouseout', clearPickPosition);
window.addEventListener('mouseleave', clearPickPosition);

window.addEventListener('touchstart', (event) => {
  // prevent the window from scrolling
  event.preventDefault();
  setPickPosition(event.touches[0]);
}, {
  passive: false
});

window.addEventListener('touchmove', (event) => {
  setPickPosition(event.touches[0]);
});

window.addEventListener('touchend', clearPickPosition);


window.addEventListener('mousemove', setPickPosition);
window.addEventListener('mouseout', clearPickPosition);
window.addEventListener('mouseleave', clearPickPosition);

body {
  margin: 0;
}
canvas {
  width: 100vw;
  height: 100vh;
  display: block;
}

<script src="https://threejsfundamentals.org/threejs/resources/threejs/r98/three.min.js"></script>
<script src="https://threejsfundamentals.org/threejs/resources/threejs/r98/js/controls/OrbitControls.js"></script>
<canvas></canvas>

Nice example. I think a lot about it and make also some tests and got conclusions. GPU pick isn't best way for this case i see a lot of problems preparing it, and this will be too painful to work with. — DonkeyLuck0, Apr 02 '19 at 15:37
If i understand this method clearly i should even set light condition for picking mesh from scene (i tests your codepen and it's not always work it have some gaps (when light reflection is making object white) which are not selecting. maybe it will work when invert color situation. example scene white and everything which not white should be picked). I'm still looking for better solution, so if u could please take a look at my second approach to the topic. — DonkeyLuck0, Apr 02 '19 at 15:38
There should be no light effects on the picking material as color and specular are set to 0. — gman, Apr 03 '19 at 01:44

three.js raycast on skinning mesh

3 Answers3