Kinect for Windows v2深度到彩色图像不对齐 [英] Kinect for Windows v2 depth to color image misalignment
问题描述
目前我正在为Kinect for Windows v2开发一个工具（类似于XBOX ONE中的工具）。我试着跟随一些例子，并有一个工作示例，显示相机图像，深度图像，以及使用opencv将深度映射到rgb的图像。但是我看到它在映射时复制了我的手，我认为这是由于坐标映射器部分出错了。
这里是一个例子：
这里是创建图像的代码片段（示例中为rgbd图像）
void KinectViewer :: create_rgbd（cv :: Mat& cv :: Mat& rgb_im，cv :: Mat& rgbd_im）{
HRESULT hr = m_pCoordinateMapper-> MapDepthFrameToColorSpace（cDepthWidth * cDepthHeight，（UINT16 *）depth_im.data，cDepthWidth * cDepthHeight，m_pColorCoordinates）;
rgbd_im = cv :: Mat :: zeros（depth_im.rows，depth_im.cols，CV_8UC3）;
double minVal，maxVal;
cv :: minMaxLoc（depth_im，& minVal，& maxVal）;
for（int i = 0; i （i，j）>（i，j）> 0&&& depth_im.at< UINT16>（i，j）< maxVal *（max_z / 100）& * min_z / 100）{
double a = i * cDepthWidth + j;
ColorSpacePoint colorPoint = m_pColorCoordinates [i * cDepthWidth + j];
int colorX =（int）（floor（colorPoint.X + 0.5））;
int colorY =（int）（floor（colorPoint.Y + 0.5））;
if（（colorX> = 0）&&（colorX< cColorWidth）&&（colorY> = 0）&&（colorY< cColorHeight））
{
rgbd_im.at< cv :: Vec3b>（i，j）= rgb_im.at< cv :: Vec3b>（colorY，colorX）
}
}
}
}
}
有没有人有一个如何解决这个问题的线索？如何防止此重复？
提前感谢
更新： p>
如果我做一个简单的深度图像阈值，我得到下面的图像：
这或多或少是我期望发生的，并且在后台没有重复的手。有没有办法在后台阻止这个重复的手？
我建议你使用BodyIndexFrame来识别一个特定的值属于玩家。这样，您可以拒绝不属于播放器的任何RGB像素，并保留其余的。我不认为CoordinateMapper在说谎。
一些注释：
- 使用MapColorFrameToDepthSpace代替MapDepthFrameToColorSpace;
- 找到相应的DepthSpacePoint和depthX，depthY，而不是ColorSpacePoint和colorX，colorY
ul>
这是我的方法，当一个框架到达（它在C＃）：
depthFrame.CopyFrameDataToArray（_depthData）;
colorFrame.CopyConvertedFrameDataToArray（_colorData，ColorImageFormat.Bgra）;
bodyIndexFrame.CopyFrameDataToArray（_bodyData）;
_coordinateMapper.MapColorFrameToDepthSpace（_depthData，_depthPoints）;
Array.Clear（_displayPixels，0，_displayPixels.Length）;
for（int colorIndex = 0; colorIndex< _depthPoints.Length; ++ colorIndex）
{
DepthSpacePoint depthPoint = _depthPoints [colorIndex];
if（！float.IsNegativeInfinity（depthPoint.X）&&！float.IsNegativeInfinity（depthPoint.Y））
{
int depthX =（int） .X + 0.5f）;
int depthY =（int）（depthPoint.Y + 0.5f）;
if（（depthX> = 0）&&（depthX< _depthWidth）&&（depthY> = 0）& b $ b {
int depthIndex =（depthY * _depthWidth）+ depthX;
byte player = _bodyData [depthIndex];
//确定该点是否属于玩家
if（player！= 0xff）
{
int sourceIndex = colorIndex * BYTES_PER_PIXEL;
_displayPixels [sourceIndex] = _colorData [sourceIndex ++]; // B
_displayPixels [sourceIndex] = _colorData [sourceIndex ++]; // G
_displayPixels [sourceIndex] = _colorData [sourceIndex ++]; // R
_displayPixels [sourceIndex] = 0xff; // A
}
}
}
} b $ b 这里是数组的初始化：
BYTES_PER_PIXEL =（PixelFormats.Bgr32.BitsPerPixel + 7）/ 8;
_colorWidth = colorFrame.FrameDescription.Width;
_colorHeight = colorFrame.FrameDescription.Height;
_depthWidth = depthFrame.FrameDescription.Width;
_depthHeight = depthFrame.FrameDescription.Height;
_bodyIndexWidth = bodyIndexFrame.FrameDescription.Width;
_bodyIndexHeight =bodyIndexFrame.FrameDescription.Height;
_depthData = new ushort [_depthWidth * _depthHeight];
_bodyData = new byte [_depthWidth * _depthHeight];
_colorData = new byte [_colorWidth * _colorHeight * BYTES_PER_PIXEL];
_displayPixels =新字节[_colorWidth * _colorHeight * BYTES_PER_PIXEL];
_depthPoints = new DepthSpacePoint [_colorWidth * _colorHeight];
请注意，_depthPoints数组的大小为1920x1080。
同样，最重要的是使用BodyIndexFrame源。
currently I am developing a tool for the Kinect for Windows v2 (similar to the one in XBOX ONE). I tried to follow some examples, and have a working example that shows the camera image, the depth image, and an image that maps the depth to the rgb using opencv. But I see that it duplicates my hand when doing the mapping, and I think it is due to something wrong in the coordinate mapper part.
here is an example of it:
And here is the code snippet that creates the image (rgbd image in the example)
void KinectViewer::create_rgbd(cv::Mat& depth_im, cv::Mat& rgb_im, cv::Mat& rgbd_im){
HRESULT hr = m_pCoordinateMapper->MapDepthFrameToColorSpace(cDepthWidth * cDepthHeight, (UINT16*)depth_im.data, cDepthWidth * cDepthHeight, m_pColorCoordinates);
rgbd_im = cv::Mat::zeros(depth_im.rows, depth_im.cols, CV_8UC3);
double minVal, maxVal;
cv::minMaxLoc(depth_im, &minVal, &maxVal);
for (int i=0; i < cDepthHeight; i++){
for (int j=0; j < cDepthWidth; j++){
if (depth_im.at<UINT16>(i, j) > 0 && depth_im.at<UINT16>(i, j) < maxVal * (max_z / 100) && depth_im.at<UINT16>(i, j) > maxVal * min_z /100){
double a = i * cDepthWidth + j;
ColorSpacePoint colorPoint = m_pColorCoordinates[i*cDepthWidth+j];
int colorX = (int)(floor(colorPoint.X + 0.5));
int colorY = (int)(floor(colorPoint.Y + 0.5));
if ((colorX >= 0) && (colorX < cColorWidth) && (colorY >= 0) && (colorY < cColorHeight))
{
rgbd_im.at<cv::Vec3b>(i, j) = rgb_im.at<cv::Vec3b>(colorY, colorX);
}
}
}
}
}
Does anyone have a clue of how to solve this? How to prevent this duplication?
Thanks in advance
UPDATE:
If I do a simple depth image thresholding I obtain the following image:
This is what more or less I expected to happen, and not having a duplicate hand in the background. Is there a way to prevent this duplicate hand in the background?
I suggest you use the BodyIndexFrame to identify whether a specific value belongs to a player or not. This way, you can reject any RGB pixel that does not belong to a player and keep the rest of them. I do not think that CoordinateMapper is lying.
A few notes:
- Include the BodyIndexFrame source to your frame reader
- Use MapColorFrameToDepthSpace instead of MapDepthFrameToColorSpace; this way, you'll get the HD image for the foreground
- Find the corresponding DepthSpacePoint and depthX, depthY, instead of ColorSpacePoint and colorX, colorY
Here is my approach when a frame arrives (it's in C#):
depthFrame.CopyFrameDataToArray(_depthData);
colorFrame.CopyConvertedFrameDataToArray(_colorData, ColorImageFormat.Bgra);
bodyIndexFrame.CopyFrameDataToArray(_bodyData);
_coordinateMapper.MapColorFrameToDepthSpace(_depthData, _depthPoints);
Array.Clear(_displayPixels, 0, _displayPixels.Length);
for (int colorIndex = 0; colorIndex < _depthPoints.Length; ++colorIndex)
{
DepthSpacePoint depthPoint = _depthPoints[colorIndex];
if (!float.IsNegativeInfinity(depthPoint.X) && !float.IsNegativeInfinity(depthPoint.Y))
{
int depthX = (int)(depthPoint.X + 0.5f);
int depthY = (int)(depthPoint.Y + 0.5f);
if ((depthX >= 0) && (depthX < _depthWidth) && (depthY >= 0) && (depthY < _depthHeight))
{
int depthIndex = (depthY * _depthWidth) + depthX;
byte player = _bodyData[depthIndex];
// Identify whether the point belongs to a player
if (player != 0xff)
{
int sourceIndex = colorIndex * BYTES_PER_PIXEL;
_displayPixels[sourceIndex] = _colorData[sourceIndex++]; // B
_displayPixels[sourceIndex] = _colorData[sourceIndex++]; // G
_displayPixels[sourceIndex] = _colorData[sourceIndex++]; // R
_displayPixels[sourceIndex] = 0xff; // A
}
}
}
}
Here is the initialization of the arrays:
BYTES_PER_PIXEL = (PixelFormats.Bgr32.BitsPerPixel + 7) / 8;
_colorWidth = colorFrame.FrameDescription.Width;
_colorHeight = colorFrame.FrameDescription.Height;
_depthWidth = depthFrame.FrameDescription.Width;
_depthHeight = depthFrame.FrameDescription.Height;
_bodyIndexWidth = bodyIndexFrame.FrameDescription.Width;
_bodyIndexHeight = bodyIndexFrame.FrameDescription.Height;
_depthData = new ushort[_depthWidth * _depthHeight];
_bodyData = new byte[_depthWidth * _depthHeight];
_colorData = new byte[_colorWidth * _colorHeight * BYTES_PER_PIXEL];
_displayPixels = new byte[_colorWidth * _colorHeight * BYTES_PER_PIXEL];
_depthPoints = new DepthSpacePoint[_colorWidth * _colorHeight];
Notice that the _depthPoints array has a 1920x1080 size.
Once again, the most important thing is to use the BodyIndexFrame source.
这篇关于Kinect for Windows v2深度到彩色图像不对齐的文章就介绍到这了，希望我们推荐的答案对大家有所帮助，也希望大家多多支持IT屋！
