   (FIXED some questions but)....Nightmare on Orthographic Picking steet 8.911
  Hey all.Im stuck on trying to set up Orthographic projection picking.yes I have looked on google for a week now thats a lot of pages.I was led to believe that I could use the same picking as the perspective projection was using(Q1.is this true).Here is a description of the program.I load a mesh that contains 5 frames(this all works when I have a perspective Projection)display them with the camera set above it and at centre of the world(we are looking down on it)then do picking.This all works well when I have it set with a perspective projection but not with orthographic projection.So the Orthographic projection app is all the same but use's  thisD3DXMatrixOrthoLH(&OrthograpicProjection,//D3DXMATRIX *pOut,width,//FLOAT w,height,//FLOAT h,1.0,//FLOAT zn,1000.0f);//FLOAT zfwhere as the other perspective app used thisD3DXMatrixPerspectiveFovLH( &Projection, ( float )D3DX_PI * 0.5f ,( FLOAT )width / ( FLOAT )height, 0.1f, 100000.0f );The mesh has its transforms set in the render code( same code for both)Q2. can the ortho projection change the frames Hierarachy, it looks like I can't get the ray to the right space.here is the pick for perspective app came from the SDK works well.cant get it to work I have debug values to show for ray position(after the picking functions).D3DXMATRIX P = *pmproj;//GetCamera().proj();POINT ptCursor;ptCursor.x = sx;ptCursor.y = sy;  // Compute the vector of the pick ray in screen space    D3DXVECTOR3 v;    v.x = ( ( ( 2.0f * ptCursor.x ) / width ) - 1 ) / P._11;    v.y = -( ( ( 2.0f * ptCursor.y ) / height ) - 1 ) / P._22;    v.z = 1.0f;D3DXVECTOR3 rayOrigin(0.0f, 0.0f, 0.0f);D3DXVECTOR3 rayDir(v.x, v.y, v.z);D3DXVECTOR3 ScreenSpacePick(v); // Get the inverse view matrix    D3DXMATRIX matView = Camera.GetViewMatrix();// Collect only the closest intersection    UINT nFace;    FLOAT fBary1, fBary2, fDist;    UINT nHitCount = 0;//by using the list of button frames we can check for onclicksUINT NumFrames = FramePtr.size();Frame *frame = NULL;//holds the current frame matched by its nameID3DX10Mesh *d3dxmesh = NULL;for(UINT ctr = 0; ctr < NumButtonFrames; ctr++){frame = FramePtr[ctr];if(frame){//only when do onovers check to see if we not on the frame we last moved to{if(frame->MeshContainer){d3dxmesh = frame->MeshContainer->pMesh;}//end meshcontainerif(d3dxmesh){D3DXMATRIX matWorld = frame->matCombined;D3DXMATRIX mWorldView = matWorld * matView;D3DXMATRIX m;D3DXMatrixInverse( &m, NULL, &mWorldView );// Transform the screen space pick ray into 3D spacerayDir.x = v.x * m._11 + v.y * m._21 + v.z * m._31;rayDir.y = v.x * m._12 + v.y * m._22 + v.z * m._32;rayDir.z = v.x * m._13 + v.y * m._23 + v.z * m._33;rayOrigin.x = m._41;rayOrigin.y = m._42;rayOrigin.z = m._43;HRESULT hr = d3dxmesh->Intersect(&rayOrigin, &rayDir, &nHitCount, &nFace, &fBary1, &fBary2, &fDist, NULL);if( nHitCount > 0 ){//close all and quitreturn true;}//end hit}//end not the same buttone as lasttime    }//end valid mesh}//end valis framed3dxmesh = NULL;}//end all //this is another pick sample can't get this one to work eitherD3DXVECTOR3 v;// Compute picking ray in view space.float vx = ((+2.0f*ptCursor.x)/width  - 1.0f)/P(0,0);float vy = ((-2.0f*ptCursor.y)/height + 1.0f)/P(1,1);v.x = vx;v.y = vy;v.z = 1.0f;D3DXVECTOR3 rayOrigin(0.0f, 0.0f, 0.0f);D3DXVECTOR3 rayDir(v.x, v.y, v.z);D3D10_VIEWPORT m_ViewPort;m_ViewPort.TopLeftX = 0;m_ViewPort.TopLeftY = 0;m_ViewPort.Width    = width;;m_ViewPort.Height   = height;m_ViewPort.MinDepth = 0.0f;m_ViewPort.MaxDepth = 1.0f;// Get the inverse view matrix    D3DXMATRIX matView = Camera.GetViewMatrix();// Collect only the closest intersection    UINT nFace;    FLOAT fBary1, fBary2, fDist;    UINT nHitCount = 0;//by using the list of button frames we can check for onclicksUINT NumFrames = FramePtr.size();Frame *frame = NULL;//holds the current frame matched by its nameID3DX10Mesh *d3dxmesh = NULL;for(UINT ctr = 0; ctr < NumFrames; ctr++){frame = FramePtr[ctr];if(frame){//only when do onovers check to see if we not on the frame we last moved to{if(frame->pMeshContainer){d3dxmesh = frame->MeshContainer->pMesh;}//end meshcontainerif(d3dxmesh){D3DXMATRIX matWorld = frame->matCombined;D3DXMATRIX mWorldView = matWorld * matView;D3DXMATRIX m;D3DXMatrixInverse( &m, NULL, &mWorldView );D3DXMATRIX inverseV;D3DXMatrixInverse(&inverseV, 0, &matView);// Tranform to World space.D3DXVec3TransformCoord(&rayOrigin, &rayOrigin, &inverseV);D3DXVec3TransformNormal(&rayDir, &rayDir, &inverseV);// Transform to the mesh's local space.D3DXMATRIX inverseW;D3DXMatrixInverse(&inverseW, 0, &matWorld);//D3DXVec3TransformCoord(&rayOrigin, &rayOrigin, &inverseW);//D3DXVec3TransformNormal(&rayDir, &rayDir, &inverseW);//D3DXVec3Normalize(&rayDir, &rayDir);HRESULT hr = d3dxmesh->Intersect(&rayOrigin, &rayDir, &nHitCount, &nFace, &fBary1, &fBary2, &fDist, NULL);if( nHitCount > 0 ){//close all and quitreturn true;}//end hit}//end not the same buttone as lasttime    }//end valid mesh}//end valis framed3dxmesh = NULL;}//end all Here is the print out of the ray and the frames world database on a test with the back buffer size of 800 X 600 and the mouse points set to 400, 300 centre of the screenthe mesh is in the centre of the world ,and all of it's frame should be transformed when its renderedthe camera is setup like this.D3DXVECTOR3 pos(0.0f, 230.0f, 0.0f);D3DXVECTOR3 Mlook(0.0f, 0.0f, 0.0f);Mlook = Mlook - pos;Camera->setLook(&Mlook);Camera->setPosition(&pos);Camera->SetRight(D3DXVECTOR3(1.0f, 0.0f, 0.0f));Camera->SetUp(D3DXVECTOR3(0.0f, 1.0f, 0.0f));Top Pick Code Value's.does a hit on this frame only when the mouse points set to 400, 300 but not any where else(this frame is centre)Frame Nmae::Options01Click PointX: 400 Click Point Y: 300 Screen Space pick ray X: 0 Screen Space pick ray Y: -0 Screen Space pick ray Z: 1 rayOrigin X: -21.51 rayOrigin Y: 148.2 rayOrigin Z: 35.83 rayDir X: 0 rayDir Y: -0.6667 rayDir Z: 0 v.x: 0 v.y: -0 World.x: -14.86 World.y: 7.67 World.z: -60.9//the view matrix is calculated like sogetViewMatrix(D3DXMATRIX* V){D3DXMatrixIdentity(V);// Keep camera's axes orthogonal to eachotherD3DXVec3Normalize(&_look, &_look);D3DXVec3Cross(&_up, &_look, &_right);D3DXVec3Normalize(&_up, &_up);D3DXVec3Cross(&_right, &_up, &_look);D3DXVec3Normalize(&_right, &_right);// Build the view matrix:float x = -D3DXVec3Dot(&_right, &_pos);float y = -D3DXVec3Dot(&_up, &_pos);float z = -D3DXVec3Dot(&_look, &_pos);(*V)(0,0) = _right.x; (*V)(0, 1) = _up.x; (*V)(0, 2) = _look.x; (*V)(0, 3) = 0.0f;(*V)(1,0) = _right.y; (*V)(1, 1) = _up.y; (*V)(1, 2) = _look.y; (*V)(1, 3) = 0.0f;(*V)(2,0) = _right.z; (*V)(2, 1) = _up.z; (*V)(2, 2) = _look.z; (*V)(2, 3) = 0.0f;(*V)(3,0) = x;   (*V)(3, 1) = y;(*V)(3, 2) = z;  (*V)(3, 3) = 1.0f;}Frame Nmae::Quit01Click PointX: 400 Click Point Y: 300 Screen Space pick ray X: 0 Screen Space pick ray Y: -0 Screen Space pick ray Z: 1 rayOrigin X: -21.15 rayOrigin Y: 148.2 rayOrigin Z: 126.2 rayDir X: 0 rayDir Y: -0.6667 rayDir Z: 0 v.x: 0 v.y: -0 World.x: -110.5 World.y: 7.67 World.z: -156.8last Pick code Value's.//this should be position to the left of the screen in world space but it gets picked like its not transforming(in centre of mesh).Frame Name::Quit01Click PointX: 400 Click Point Y: 300 Screen Space pick ray X: 0 Screen Space pick ray Y: 0 rayOrigin X: 0 rayOrigin Y: 230 rayOrigin Z: 0 rayDir X: 0 rayDir Y: -1 rayDir Z: 0 v.x: 0 v.y: 0 World.x: -9.864 World.y: 10.41 World.z: -27.51Q3.What space do I need to convert to Q4.where is it wrong.[Edited by - ankhd on January 25, 2010 9:04:16 PM]
  I believe the mesh is transformed its in world.I pass the world view and projection to the shader like this.//set the transformvp = view * projection;mworld = frame->matCombined;wvp = mworld * (*vp);the shader takes thisWhat space Im I in.if using this here code// Compute picking ray in view space.float vx = ((+2.0f*ptCursor.x)/width  - 1.0f)/P(0,0);float vy = ((-2.0f*ptCursor.y)/height + 1.0f)/P(1,1);now if i click 0,0 in window the above code give's me the following valuesvx = -400vy = 300is this world or view space it say's view space in doc, but my meshes combined matrix x, y and z values match. Does this mean my object is in view space..one more thing to not if i change my camera's position nothing changes, is this right with a orthographic projection or do I have some thing mesed up.[Edited by - ankhd on January 20, 2010 10:13:14 PM];  I've try all I can to fix this but I can't find any thing.What is the D3DXMatrixOrthoLH function doing to the pick ray.When using this D3DXMatrixPerspectiveFovLH( &m_Projection, ( float )D3DX_PI * 0.5f , ( FLOAT )width / ( FLOAT )height, 0.1f, 100000.0f );It works.But if I set the projection to this//create a orthographic projectionD3DXMatrixOrthoLH(&m_OrthograpicProjection,//D3DXMATRIX *pOut,  width,//FLOAT w,  height,//FLOAT h,  -10000.0,//FLOAT zn,  10000.0f);//FLOAT zfno picking it like it does not get transformed right whyHas anyone got picking to work with a ortho projection.   ;  Thanks All, Im now one step closer to Craziness(who said that) now I can sleep...well I found a way to do ray picking in ortho graphic projection mode works well but still cant get it to work for perspective but thats okI have a working perspective picking function as well.but I do have some questions on why is the following code work when the books I have, have picking but not looking like this or woking.my main problem was using the objects world location in the unproject function when i should have been using the identity matrix, ahyes i know your about to say i want to be able to place my object any where in the world, yes this lets you still place objects anywhere plus scale them. we only want to make a ray in world based on where we clicked that is the main step.anyhow check out the code....//------------------------------------------------------------------------------//this will create a ray for the mouse points xy and convert them to object space to do a pick test//for a Orthographic projection//------------------------------------------------------------------------------bool PickOrthographic(HWND hwnd,//used to get the pos of the mouseCamera *Camera,//cameraD3DXMATRIX *pmproj,//the projection matrixint sx, int sy,//mouseint width,int height)//screen width and height {D3DXMATRIX P = *pmproj;POINT ptCursor;ptCursor.x = sx;ptCursor.y = sy;  D3DXVECTOR3 v;D3DXVECTOR3 rayOrigin(0.0f, 0.0f, 0.0f);D3DXVECTOR3 rayDir(0.0f, 0.0f, 0.0f);    // Get the view matrix   D3DXMATRIX matView = Camera->GetViewMatrix();    D3D10_VIEWPORT m_ViewPort;m_ViewPort.TopLeftX = 0;m_ViewPort.TopLeftY = 0;m_ViewPort.Width    = width;;m_ViewPort.Height   = height;m_ViewPort.MinDepth = 0.0f;m_ViewPort.MaxDepth = 1.0f;// Collect only the closest intersection    UINT nFace;    FLOAT fBary1, fBary2, fDist;    UINT nHitCount = 0;//we need the Identity matrix not the objects world matrix//this is because unproject needs a reference to work from like the centre of the worldD3DXMATRIX Identityworld;D3DXMatrixIdentity( &Identityworld );ID3DX10Mesh *d3dxmesh = NULL;if(d3dxmesh){D3DXMATRIX matWorld = frame->matCombined;D3DXMATRIX ToObjectSpaceMat;D3DXMatrixInverse( &ToObjectSpaceMat, NULL, &matWorld );v.x = ptCursor.x;v.y = ptCursor.y;v.z = 0.0f;//unproject the origin to its world location with a z = 0.0fD3DXVec3Unproject(&rayOrigin,&v,&m_ViewPort,&P,&matView, &Identityworld);// second call has z set to onev.x = ptCursor.x;v.y = ptCursor.y;v.z = 1000.0f;//unproject the direction to its world location with a z = 1000.0f the z here can fit into any z range you may want maybe?????D3DXVec3Unproject(&rayDir,&v,&m_ViewPort,pmproj,&matView,&Identityworld);//now transform them to object spaceD3DXVec3TransformCoord(&rayOrigin, &rayOrigin, &ToObjectSpaceMat);D3DXVec3TransformNormal(&rayDir, &rayDir, &ToObjectSpaceMat);//the ray is now ready to be used in intersectHRESULT hr = d3dxmesh->Intersect(&rayOrigin, &rayDir, &nHitCount, &nFace, &fBary1, &fBary2, &fDist, NULL);if( nHitCount > 0 ){OutPutDebugText("Have Collisiondata....",//the message to display(TCHAR *)frame->Name.c_str(),//the location the message was called Eg. what function//close all and quitreturn true;}//end hit}//end valid mesh    return false;}//end  PickOrthographic//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////