Question

在这里的第一篇文章，我会尽力描述问题。我正在尝试使用类似于this software

这样的精美应用程序的opencv，dlib和网络摄像头数据来实时制作3D虚拟角色的面部动画。

在example here之后，使用网络摄像头，我可以获得屏幕空间中的实时2D面部界标数据，还可以估算出头部姿势的3D翻译数据。但是我真正想知道的是面部标志的估计3D世界坐标。例如，当头部向侧面倾斜30度时，嘴巴看起来就像演员用嘴发出“ w”声一样。

谁能告诉我使用估计的头部姿势将这些2d面部界标转换为3D数据的策略是什么，或者这是什么使这些实时面部动画应用如此昂贵？


std::vector<cv::Point3d> object_pts;
std::vector<cv::Point2d> image_pts;
std::vector<cv::Point3d> reprojectsrc;
std::vector<cv::Point2d> reprojectdst;
cv::Mat rotation_vec;
cv::Mat rotation_mat;
cv::Mat translation_vec;
cv::Mat pose_mat;
cv::Mat euler_angle;
cv::Mat cam_matrix;
cv::Mat dist_coeffs;
cv::Mat out_intrinsics;
cv::Mat out_rotation;
cv::Mat out_translation;

int UpdateFace(dlib::cv_image<dlib::bgr_pixel> cimg, dlib::rectangle face)
{
    // get facial landmarks
    shape = predictor(cimg, face);

    // draw facial landmarks
    for (unsigned int i = 0; i < 68; ++i)
    {
        cv::circle(frame, cv::Point(shape.part(i).x(), shape.part(i).y()), 2, cv::Scalar(0, 0, 255), -1);
    }

    // clear data and fill in 2D ref points
    image_pts.clear();
    //#17 left brow left corner
    image_pts.push_back(cv::Point2d(shape.part(17).x(), shape.part(17).y())); 
    //#21 left brow right corner
    image_pts.push_back(cv::Point2d(shape.part(21).x(), shape.part(21).y())); 
    //#22 right brow left corner
    image_pts.push_back(cv::Point2d(shape.part(22).x(), shape.part(22).y())); 
    //#26 right brow right corner
    image_pts.push_back(cv::Point2d(shape.part(26).x(), shape.part(26).y())); 
    //#36 left eye left corner
    image_pts.push_back(cv::Point2d(shape.part(36).x(), shape.part(36).y())); 
    //#39 left eye right corner
    image_pts.push_back(cv::Point2d(shape.part(39).x(), shape.part(39).y())); 
    //#42 right eye left corner
    image_pts.push_back(cv::Point2d(shape.part(42).x(), shape.part(42).y())); 
    //#45 right eye right corner
    image_pts.push_back(cv::Point2d(shape.part(45).x(), shape.part(45).y())); 
    //#31 nose left corner
    image_pts.push_back(cv::Point2d(shape.part(31).x(), shape.part(31).y())); 
    //#35 nose right corner
    image_pts.push_back(cv::Point2d(shape.part(35).x(), shape.part(35).y())); 
    //#48 mouth left corner
    image_pts.push_back(cv::Point2d(shape.part(48).x(), shape.part(48).y())); 
    //#54 mouth right corner
    image_pts.push_back(cv::Point2d(shape.part(54).x(), shape.part(54).y())); 
    //#57 mouth central bottom corner
    image_pts.push_back(cv::Point2d(shape.part(57).x(), shape.part(57).y())); 
    //#8 chin corner
    image_pts.push_back(cv::Point2d(shape.part(8).x(), shape.part(8).y()));   

    // calculate the head pose
    cv::solvePnP(object_pts, image_pts, cam_matrix, dist_coeffs, rotation_vec, translation_vec);

    // reproject
    cv::projectPoints(reprojectsrc, rotation_vec, translation_vec, cam_matrix, dist_coeffs, reprojectdst);

    //draw 3d box around the actors head
    cv::line(frame, reprojectdst[0], reprojectdst[1], cv::Scalar(0, 0, 255));
    cv::line(frame, reprojectdst[1], reprojectdst[2], cv::Scalar(0, 0, 255));
    cv::line(frame, reprojectdst[2], reprojectdst[3], cv::Scalar(0, 0, 255));
    cv::line(frame, reprojectdst[3], reprojectdst[0], cv::Scalar(0, 0, 255));
    cv::line(frame, reprojectdst[4], reprojectdst[5], cv::Scalar(0, 0, 255));
    cv::line(frame, reprojectdst[5], reprojectdst[6], cv::Scalar(0, 0, 255));
    cv::line(frame, reprojectdst[6], reprojectdst[7], cv::Scalar(0, 0, 255));
    cv::line(frame, reprojectdst[7], reprojectdst[4], cv::Scalar(0, 0, 255));
    cv::line(frame, reprojectdst[0], reprojectdst[4], cv::Scalar(0, 0, 255));
    cv::line(frame, reprojectdst[1], reprojectdst[5], cv::Scalar(0, 0, 255));
    cv::line(frame, reprojectdst[2], reprojectdst[6], cv::Scalar(0, 0, 255));
    cv::line(frame, reprojectdst[3], reprojectdst[7], cv::Scalar(0, 0, 255));

    //calculate euler angle to move avatars head
    cv::Rodrigues(rotation_vec, rotation_mat);
    cv::hconcat(rotation_mat, translation_vec, pose_mat);
    cv::decomposeProjectionMatrix(pose_mat, out_intrinsics, out_rotation, 
        out_translation, cv::noArray(), cv::noArray(), cv::noArray(), euler_angle);

    // calculate 3D transformation of facial landmarks to animate avatars facial features
    // code needed here ...
}

Answer 1

这不是一个容易解决的问题。

您可以将3D可变形模型视为起点。

https://cvssp.org/faceweb/3dmm/

然后，您可以查看在Google学术搜索中引用该论文的所有最新论文，以找到最新技术。

希望有帮助。

您如何将2D面部地标转换为3D世界坐标？

1 个答案: