Add backface culling and option to keep unflipped hit normal in case a ray hits a back-facing triangle.

Usage: set RayResultCallback.m_flags to kF_FilterBackfaces, optionally combined with kF_KeepUnflippedNormal.
Thanks Andy O'Neil for the patch!

Remove the force_inline for some internal constraint solver methods, it makes re-use easier.
Workaround/avoid MSVC 2005 compiler error in LibXML/trionan.c

Extras/LibXML/trionan.c

@@ -174,11 +174,16 @@ trio_make_double
 TRIO_ARGS1((values),
 	   TRIO_CONST unsigned char *values)
 {
-  TRIO_VOLATILE double result;
+
+//remove the TRI_VOLATILE, because MSVC 2005 compiles crashes
+//	fatal error C1001: An internal error has occurred in the compiler.
+// (compiler file 'f:\rtm\vctools\compiler\utc\src\P2\main.c[0x00498D00:0x00498D00]', line 182)
+
+  double result;
   int i;
 
   for (i = 0; i < (int)sizeof(double); i++) {
-    ((TRIO_VOLATILE unsigned char *)&result)[TRIO_DOUBLE_INDEX(i)] = values[i];
+    (( unsigned char *)&result)[TRIO_DOUBLE_INDEX(i)] = values[i];
   }
   return result;
 }

src/BulletCollision/CollisionDispatch/btCollisionWorld.cpp

@@ -298,7 +298,8 @@ void	btCollisionWorld::rayTestSingle(const btTransform& rayFromTrans,const btTra
 
 					BridgeTriangleRaycastCallback( const btVector3& from,const btVector3& to,
 						btCollisionWorld::RayResultCallback* resultCallback, btCollisionObject* collisionObject,btTriangleMeshShape*	triangleMesh):
-						btTriangleRaycastCallback(from,to),
+                  //@BP Mod
+						btTriangleRaycastCallback(from,to, resultCallback->m_flags),
 							m_resultCallback(resultCallback),
 							m_collisionObject(collisionObject),
 							m_triangleMesh(triangleMesh)
@@ -347,7 +348,8 @@ void	btCollisionWorld::rayTestSingle(const btTransform& rayFromTrans,const btTra
 
 					BridgeTriangleRaycastCallback( const btVector3& from,const btVector3& to,
 						btCollisionWorld::RayResultCallback* resultCallback, btCollisionObject* collisionObject,btConcaveShape*	triangleMesh):
-						btTriangleRaycastCallback(from,to),
+                  //@BP Mod
+                  btTriangleRaycastCallback(from,to, resultCallback->m_flags),
 							m_resultCallback(resultCallback),
 							m_collisionObject(collisionObject),
 							m_triangleMesh(triangleMesh)

src/BulletCollision/CollisionDispatch/btCollisionWorld.h

@@ -186,6 +186,8 @@ public:
 		btCollisionObject*		m_collisionObject;
 		short int	m_collisionFilterGroup;
 		short int	m_collisionFilterMask;
+      //@BP Mod - Custom flags, currently used to enable backface culling on tri-meshes, see btRaycastCallback
+      unsigned int m_flags;
 
 		virtual ~RayResultCallback()
 		{
@@ -199,7 +201,9 @@ public:
 			:m_closestHitFraction(btScalar(1.)),
 			m_collisionObject(0),
 			m_collisionFilterGroup(btBroadphaseProxy::DefaultFilter),
-			m_collisionFilterMask(btBroadphaseProxy::AllFilter)
+			m_collisionFilterMask(btBroadphaseProxy::AllFilter),
+         //@BP Mod
+         m_flags(0)
 		{
 		}
 

src/BulletCollision/NarrowPhaseCollision/btRaycastCallback.cpp

@@ -23,10 +23,12 @@ subject to the following restrictions:
 #include "BulletCollision/NarrowPhaseCollision/btGjkEpaPenetrationDepthSolver.h"
 #include "btRaycastCallback.h"
 
-btTriangleRaycastCallback::btTriangleRaycastCallback(const btVector3& from,const btVector3& to)
+btTriangleRaycastCallback::btTriangleRaycastCallback(const btVector3& from,const btVector3& to, unsigned int flags)
 	:
 	m_from(from),
 	m_to(to),
+   //@BP Mod
+   m_flags(flags),
 	m_hitFraction(btScalar(1.))
 {
 
@@ -55,6 +57,12 @@ void btTriangleRaycastCallback::processTriangle(btVector3* triangle,int partId,
 	{
 		return ; // same sign
 	}
+   //@BP Mod - Backface filtering
+   if (((m_flags & kF_FilterBackfaces) != 0) && (dist_a > btScalar(0.0)))
+   {
+      // Backface, skip check
+      return;
+   }
 	
 	const btScalar proj_length=dist_a-dist_b;
 	const btScalar distance = (dist_a)/(proj_length);
@@ -89,14 +97,18 @@ void btTriangleRaycastCallback::processTriangle(btVector3* triangle,int partId,
 					
 					if ( (btScalar)(cp2.dot(triangleNormal)) >=edge_tolerance) 
 					{
+                  //@BP Mod
+                  // Triangle normal isn't normalized
+				      triangleNormal.normalize();
 
-						if ( dist_a > 0 )
+                  //@BP Mod - Allow for unflipped normal when raycasting against backfaces
+                  if (((m_flags & kF_KeepUnflippedNormal) != 0) || (dist_a <= btScalar(0.0)))
 						{
-							m_hitFraction = reportHit(triangleNormal,distance,partId,triangleIndex);
+							m_hitFraction = reportHit(-triangleNormal,distance,partId,triangleIndex);
 						}
 						else
 						{
-							m_hitFraction = reportHit(-triangleNormal,distance,partId,triangleIndex);
+                     m_hitFraction = reportHit(triangleNormal,distance,partId,triangleIndex);
 						}
 					}
 				}

src/BulletCollision/NarrowPhaseCollision/btRaycastCallback.h

@@ -29,9 +29,20 @@ public:
 	btVector3 m_from;
 	btVector3 m_to;
 
+   //@BP Mod - allow backface filtering and unflipped normals
+   enum EFlags
+   {
+      kF_None                 = 0,
+      kF_FilterBackfaces      = 1 << 0,
+      kF_KeepUnflippedNormal  = 1 << 1,   // Prevents returned face normal getting flipped when a ray hits a back-facing triangle
+
+      kF_Terminator        = 0xFFFFFFFF
+   };
+   unsigned int m_flags;
+
 	btScalar	m_hitFraction;
 
-	btTriangleRaycastCallback(const btVector3& from,const btVector3& to);
+	btTriangleRaycastCallback(const btVector3& from,const btVector3& to, unsigned int flags=0);
 	
 	virtual void processTriangle(btVector3* triangle, int partId, int triangleIndex);
 

src/BulletDynamics/ConstraintSolver/btSequentialImpulseConstraintSolver.cpp

@@ -55,7 +55,7 @@ static inline __m128 _vmathVfDot3( __m128 vec0, __m128 vec1 )
 #endif//USE_SIMD
 
 // Project Gauss Seidel or the equivalent Sequential Impulse
-SIMD_FORCE_INLINE void btSequentialImpulseConstraintSolver::resolveSingleConstraintRowGenericSIMD(btSolverBody& body1,btSolverBody& body2,const btSolverConstraint& c)
+void btSequentialImpulseConstraintSolver::resolveSingleConstraintRowGenericSIMD(btSolverBody& body1,btSolverBody& body2,const btSolverConstraint& c)
 {
 #ifdef USE_SIMD
 	__m128 cpAppliedImp = _mm_set1_ps(c.m_appliedImpulse);
@@ -89,7 +89,7 @@ SIMD_FORCE_INLINE void btSequentialImpulseConstraintSolver::resolveSingleConstra
 }
 
 // Project Gauss Seidel or the equivalent Sequential Impulse
-SIMD_FORCE_INLINE void btSequentialImpulseConstraintSolver::resolveSingleConstraintRowGeneric(btSolverBody& body1,btSolverBody& body2,const btSolverConstraint& c)
+ void btSequentialImpulseConstraintSolver::resolveSingleConstraintRowGeneric(btSolverBody& body1,btSolverBody& body2,const btSolverConstraint& c)
 {
 	btScalar deltaImpulse = c.m_rhs-btScalar(c.m_appliedImpulse)*c.m_cfm;
 	const btScalar deltaVel1Dotn	=	c.m_contactNormal.dot(body1.m_deltaLinearVelocity) 	+ c.m_relpos1CrossNormal.dot(body1.m_deltaAngularVelocity);
@@ -120,7 +120,7 @@ SIMD_FORCE_INLINE void btSequentialImpulseConstraintSolver::resolveSingleConstra
 		body2.applyImpulse(-c.m_contactNormal*body2.m_invMass,c.m_angularComponentB,deltaImpulse);
 }
 
-SIMD_FORCE_INLINE void btSequentialImpulseConstraintSolver::resolveSingleConstraintRowLowerLimitSIMD(btSolverBody& body1,btSolverBody& body2,const btSolverConstraint& c)
+ void btSequentialImpulseConstraintSolver::resolveSingleConstraintRowLowerLimitSIMD(btSolverBody& body1,btSolverBody& body2,const btSolverConstraint& c)
 {
 #ifdef USE_SIMD
 	__m128 cpAppliedImp = _mm_set1_ps(c.m_appliedImpulse);
@@ -151,7 +151,7 @@ SIMD_FORCE_INLINE void btSequentialImpulseConstraintSolver::resolveSingleConstra
 }
 
 // Project Gauss Seidel or the equivalent Sequential Impulse
-SIMD_FORCE_INLINE void btSequentialImpulseConstraintSolver::resolveSingleConstraintRowLowerLimit(btSolverBody& body1,btSolverBody& body2,const btSolverConstraint& c)
+ void btSequentialImpulseConstraintSolver::resolveSingleConstraintRowLowerLimit(btSolverBody& body1,btSolverBody& body2,const btSolverConstraint& c)
 {
 	btScalar deltaImpulse = c.m_rhs-btScalar(c.m_appliedImpulse)*c.m_cfm;
 	const btScalar deltaVel1Dotn	=	c.m_contactNormal.dot(body1.m_deltaLinearVelocity) 	+ c.m_relpos1CrossNormal.dot(body1.m_deltaAngularVelocity);