相机校准opencv python的鸟瞰透视图转换 [英] Bird's eye view perspective transformation from camera calibration opencv python
问题描述
我正在尝试从相机的固有，外在矩阵和失真系数中获得鸟瞰视角的变换.
我尝试使用这个问题的答案. >
使用的图像是来自opencv官方github存储库的样本图像left02.jpg
我校准了相机，发现了固有的，非固有的矩阵和失真系数.
我未扭曲图像并找到姿势.检查参数是否正确.
我用来查找透视变换矩阵的方程是(请参阅上面的链接):
Hr = K * R.inv() * K.inv()，其中R是旋转矩阵(来自cv2.Rodrigues())，而K是来自cv2.getoptimalnewcameramatrix()
[ 1 0 | ]
Ht = [ 0 1 | -K*C/Cz ]
[ 0 0 | ]
其中C=-R.inv()*T其中T是来自cv2.solvePnP()的翻译向量
Cz是C向量的第三个分量
所需的转换是:H = Ht * Hr
我用来构造上述等式的代码是:
K = newcameramtx # from cv2.getoptimalnewcameramatrix()
ret,rvec,tvec = cv2.solvePnP(world_points,corners2,K,dist)
R,_ = cv2.Rodrigues(rvec)
_,R_inv = cv2.invert(R)
_,K_inv = cv2.invert(K)
Hr = np.matmul(K,np.matmul(R_inv,K_inv))
C = np.matmul(-R_inv,tvec)
Cz = C[2]
temp_vector = np.matmul(-K,C/Cz)
Ht = np.identity(3)
for i,val in enumerate(temp_vector):
Ht[i][2] = val
homography = np.matmul(Ht,Hr)
warped_img =cv2.warpPerspective(img,homography,(img.shape[1],img.shape[0]))
# where img is the above undistored image with visualized pose
生成的变形图像不正确.
如果我使用以下代码从单应性中删除译文
homography = Hr.copy()
warped_img =cv2.warpPerspective(img,homography,(img.shape[1],img.shape[0]))
我得到以下图像
我认为上图显示我的旋转部分是正确的，但是我的翻译是错误的.
由于平移矩阵(Ht)是扩充矩阵，所以不确定上述矩阵的构造是否正确.
我特别想从相机校准中找出鸟瞰视角的变换.
所以，我该如何校正上述等式，以便获得棋盘图像的完美鸟瞰图
任何人也可以解释一下如何推导有关Ht和Hr的上述方程的数学方法吗?我没有太多接触线性代数的知识，所以这些方程式对我来说不是很明显.
更新:
homography = np.matmul(Ht,Hr)
warped_img =cv2.warpPerspective(img,homography,(img.shape[1],img.shape[0]),flags=cv2.WARP_INVERSE_MAP)
cv2.WARP_INVERSE_MAP标志给了我不同的结果
仍然不是我要找的结果！
本教程中说明了您要实现的目标:此处(对于摄像头框架)，则允许鸟瞰视角的摄像头姿势所需的旋转为:
只需按照本教程操作，您将获得与以下内容类似的单应性矩阵:
H:
[0.935, -0.337, 40.383;
-0.116, 0.729, 64.381;
0.000408, -0.001299, 1]
使用warpPerspective:
另一个例子:
由于棋盘是平坦的(对于一般场景，单应性仅适用于纯旋转相机运动)，因此您还可以进行以下平移操作:
编辑:从本教程获得的代码
#include <opencv2/opencv.hpp>
#include <opencv2/aruco.hpp>
using namespace std;
using namespace cv;
namespace
{
enum Pattern { CHESSBOARD, CIRCLES_GRID, ASYMMETRIC_CIRCLES_GRID };
void calcChessboardCorners(Size boardSize, float squareSize, vector<Point3f>& corners, Pattern patternType = CHESSBOARD)
{
corners.resize(0);
switch (patternType)
{
case CHESSBOARD:
case CIRCLES_GRID:
//! [compute-chessboard-object-points]
for( int i = 0; i < boardSize.height; i++ )
for( int j = 0; j < boardSize.width; j++ )
//To try to center the chessboard frame, we substract the image size
corners.push_back(Point3f(float((j-boardSize.width/2)*squareSize),
float((i-boardSize.height/2)*squareSize), 0));
//! [compute-chessboard-object-points]
break;
case ASYMMETRIC_CIRCLES_GRID:
for( int i = 0; i < boardSize.height; i++ )
for( int j = 0; j < boardSize.width; j++ )
corners.push_back(Point3f(float((2*j + i % 2)*squareSize),
float(i*squareSize), 0));
break;
default:
CV_Error(Error::StsBadArg, "Unknown pattern type\n");
}
}
void computeC2MC1(const Mat &R1, const Mat &tvec1, const Mat &R2, const Mat &tvec2,
Mat &R_1to2, Mat &tvec_1to2)
{
//c2Mc1 = c2Mo * oMc1 = c2Mo * c1Mo.inv()
R_1to2 = R2 * R1.t();
tvec_1to2 = R2 * (-R1.t()*tvec1) + tvec2;
}
} //namespace
int main()
{
Mat img = imread("left02.jpg");
Mat img_corners = img.clone(), img_pose = img.clone(), img_bird_eye_view = img.clone();
vector<Point2f> corners;
Size patternSize(9,6);
bool found = findChessboardCorners(img, patternSize, corners);
drawChessboardCorners(img_corners, patternSize, corners, found);
imshow("Chessboard corners detection", img_corners);
vector<Point3f> objectPoints;
float squareSize = 2.5e-2;
calcChessboardCorners(patternSize, squareSize, objectPoints);
FileStorage fs("left_intrinsics.yml", FileStorage::READ);
Mat cameraMatrix, distCoeffs;
fs["camera_matrix"] >> cameraMatrix;
fs["distortion_coefficients"] >> distCoeffs;
Mat rvec, tvec;
solvePnP(objectPoints, corners, cameraMatrix, distCoeffs, rvec, tvec);
aruco::drawAxis(img_pose, cameraMatrix, distCoeffs, rvec, tvec, 2*squareSize);
imshow("Pose", img_pose);
Mat R_desired = (Mat_<double>(3,3) <<
0, 1, 0,
-1, 0, 0,
0, 0, 1);
Mat R;
Rodrigues(rvec, R);
Mat normal = (Mat_<double>(3,1) << 0, 0, 1);
Mat normal1 = R*normal;
Mat origin(3, 1, CV_64F, Scalar(0));
Mat origin1 = R*origin + tvec;
double d_inv1 = 1.0 / normal1.dot(origin1);
Mat R_1to2, tvec_1to2;
Mat tvec_desired = tvec.clone();
computeC2MC1(R, tvec, R_desired, tvec_desired, R_1to2, tvec_1to2);
Mat H = R_1to2 + d_inv1 * tvec_1to2*normal1.t();
H = cameraMatrix * H * cameraMatrix.inv();
H = H/H.at<double>(2,2);
std::cout << "H:\n" << H << std::endl;
warpPerspective(img_pose, img_bird_eye_view, H, img.size());
Mat compare;
hconcat(img_pose, img_bird_eye_view, compare);
imshow("Bird eye view", compare);
waitKey();
return 0;
}
I am trying to get the bird's eye view perspective transform from camera intrinsic, extrinsic matrices and distortion coefficients.
I tried using the answer from this question.
The image used is the sample image left02.jpg from the opencv official github repo
I calibrated the camera and found the intrinsic, extrinsic matrices and the distortion co-efficients.
I undistored the image and found the pose. To check if the params are right.
The equations I used to find the perspective transformation matrix are (Refer the above link):
Hr = K * R.inv() * K.inv() where R is rotational matrix (from cv2.Rodrigues()) and K is obtained from cv2.getoptimalnewcameramatrix()
[ 1 0 | ]
Ht = [ 0 1 | -K*C/Cz ]
[ 0 0 | ]
Where C=-R.inv()*T Where T is translational vector from cv2.solvePnP()
and Cz is the 3rd component of the C vector
The required transformation is: H = Ht * Hr
The code I used to construct the above equation is:
K = newcameramtx # from cv2.getoptimalnewcameramatrix()
ret,rvec,tvec = cv2.solvePnP(world_points,corners2,K,dist)
R,_ = cv2.Rodrigues(rvec)
_,R_inv = cv2.invert(R)
_,K_inv = cv2.invert(K)
Hr = np.matmul(K,np.matmul(R_inv,K_inv))
C = np.matmul(-R_inv,tvec)
Cz = C[2]
temp_vector = np.matmul(-K,C/Cz)
Ht = np.identity(3)
for i,val in enumerate(temp_vector):
Ht[i][2] = val
homography = np.matmul(Ht,Hr)
warped_img =cv2.warpPerspective(img,homography,(img.shape[1],img.shape[0]))
# where img is the above undistored image with visualized pose
The resulting warped image is not correct.
If I remove the translation from the homography by using the below code
homography = Hr.copy()
warped_img =cv2.warpPerspective(img,homography,(img.shape[1],img.shape[0]))
I am getting the following image
I think the above image shows that my rotational part is correct but my translation is wrong.
Since the translational matrix (Ht) is an augmented matrix am unsure whether my construction of the above matrix is correct.
I specifically want to figure out the bird's eye perspective transformation from the camera calibration.
So, How do I correct the above equations so that I am getting the perfect bird's eye view of the chessboard image
Could anyone also please explain the math on how the above equations for Ht and Hr are derived? I don't have much exposure to Linear algebra so these equations are not very obvious to me.
UPDATE:
homography = np.matmul(Ht,Hr)
warped_img =cv2.warpPerspective(img,homography,(img.shape[1],img.shape[0]),flags=cv2.WARP_INVERSE_MAP)
cv2.WARP_INVERSE_MAP flag gave me a different result
Still not the result I am looking for!
What you want to achieve is explained in the tutorial: Demo 3: Homography from the camera displacement.
You have the current camera pose (rotation + translation), you can compute the desired camera pose that allows to view the chessboard from a bird eye view.
As the chessboard frame is different from the camera frame (see here for the camera frame), the desired rotation for the camera pose that allows a bird eye view is:
Just follow the tutorial and you should get a homography matrix similar to:
H:
[0.935, -0.337, 40.383;
-0.116, 0.729, 64.381;
0.000408, -0.001299, 1]
With warpPerspective:
Another example:
As the chessboard is flat (for a generic scene a homography is valid only for a pure rotational camera movement), you can also play with the translation:
Edit: the code derived from the tutorial
#include <opencv2/opencv.hpp>
#include <opencv2/aruco.hpp>
using namespace std;
using namespace cv;
namespace
{
enum Pattern { CHESSBOARD, CIRCLES_GRID, ASYMMETRIC_CIRCLES_GRID };
void calcChessboardCorners(Size boardSize, float squareSize, vector<Point3f>& corners, Pattern patternType = CHESSBOARD)
{
corners.resize(0);
switch (patternType)
{
case CHESSBOARD:
case CIRCLES_GRID:
//! [compute-chessboard-object-points]
for( int i = 0; i < boardSize.height; i++ )
for( int j = 0; j < boardSize.width; j++ )
//To try to center the chessboard frame, we substract the image size
corners.push_back(Point3f(float((j-boardSize.width/2)*squareSize),
float((i-boardSize.height/2)*squareSize), 0));
//! [compute-chessboard-object-points]
break;
case ASYMMETRIC_CIRCLES_GRID:
for( int i = 0; i < boardSize.height; i++ )
for( int j = 0; j < boardSize.width; j++ )
corners.push_back(Point3f(float((2*j + i % 2)*squareSize),
float(i*squareSize), 0));
break;
default:
CV_Error(Error::StsBadArg, "Unknown pattern type\n");
}
}
void computeC2MC1(const Mat &R1, const Mat &tvec1, const Mat &R2, const Mat &tvec2,
Mat &R_1to2, Mat &tvec_1to2)
{
//c2Mc1 = c2Mo * oMc1 = c2Mo * c1Mo.inv()
R_1to2 = R2 * R1.t();
tvec_1to2 = R2 * (-R1.t()*tvec1) + tvec2;
}
} //namespace
int main()
{
Mat img = imread("left02.jpg");
Mat img_corners = img.clone(), img_pose = img.clone(), img_bird_eye_view = img.clone();
vector<Point2f> corners;
Size patternSize(9,6);
bool found = findChessboardCorners(img, patternSize, corners);
drawChessboardCorners(img_corners, patternSize, corners, found);
imshow("Chessboard corners detection", img_corners);
vector<Point3f> objectPoints;
float squareSize = 2.5e-2;
calcChessboardCorners(patternSize, squareSize, objectPoints);
FileStorage fs("left_intrinsics.yml", FileStorage::READ);
Mat cameraMatrix, distCoeffs;
fs["camera_matrix"] >> cameraMatrix;
fs["distortion_coefficients"] >> distCoeffs;
Mat rvec, tvec;
solvePnP(objectPoints, corners, cameraMatrix, distCoeffs, rvec, tvec);
aruco::drawAxis(img_pose, cameraMatrix, distCoeffs, rvec, tvec, 2*squareSize);
imshow("Pose", img_pose);
Mat R_desired = (Mat_<double>(3,3) <<
0, 1, 0,
-1, 0, 0,
0, 0, 1);
Mat R;
Rodrigues(rvec, R);
Mat normal = (Mat_<double>(3,1) << 0, 0, 1);
Mat normal1 = R*normal;
Mat origin(3, 1, CV_64F, Scalar(0));
Mat origin1 = R*origin + tvec;
double d_inv1 = 1.0 / normal1.dot(origin1);
Mat R_1to2, tvec_1to2;
Mat tvec_desired = tvec.clone();
computeC2MC1(R, tvec, R_desired, tvec_desired, R_1to2, tvec_1to2);
Mat H = R_1to2 + d_inv1 * tvec_1to2*normal1.t();
H = cameraMatrix * H * cameraMatrix.inv();
H = H/H.at<double>(2,2);
std::cout << "H:\n" << H << std::endl;
warpPerspective(img_pose, img_bird_eye_view, H, img.size());
Mat compare;
hconcat(img_pose, img_bird_eye_view, compare);
imshow("Bird eye view", compare);
waitKey();
return 0;
}
这篇关于相机校准opencv python的鸟瞰透视图转换的文章就介绍到这了，希望我们推荐的答案对大家有所帮助，也希望大家多多支持IT屋！
