Question

问题陈述：图像A通过投影仪投影，通过显微镜，投影图像通过相机通过与图像B相同的显微镜捕获。由于光学元件，B旋转，剪切和扭曲关于A.现在，我需要在投影前将A转换为A'，使B尽可能接近A.

初始方法：我采用棋盘图案并以各种角度（36,72,108，... 324度）旋转并投影以获得一系列A图像和B图像。我使用OpenCV的CalibrateCamera2，InitUndistortMap和Remap函数将B转换为B'。但是B'在A附近并且与B非常相似（特别是有大量的旋转和剪切没有得到纠正）。

代码（在Python中）如下所示。我不确定我是不是在做一些愚蠢的事情。有关正确方法的任何想法吗？

import pylab
import os
import cv
import cv2
import numpy

# angles - the angles at which the picture was rotated 
angles = [0, 36, 72, 108, 144, 180, 216, 252, 288, 324]
# orig_files - list of original picture files used for projection
orig_files =  ['../calibration/checkerboard/orig_%d.png' % (angle) for angle in angles]
# img_files - projected image captured by camera
img_files = ['../calibration/checkerboard/imag_%d.bmp' % (angle) for angle in angles]
# Load the images 
images = [cv.LoadImage(filename) for filename in img_files]
orig_images = [cv.LoadImage(filename) for filename in orig_files]

# Convert to grayscale
gray_images = [cv.CreateImage((src.height, src.width), cv.IPL_DEPTH_8U, 1) for src in images]
for ii in range(len(images)):
    cv.CvtColor(images[ii], gray_images[ii], cv.CV_RGB2GRAY)
gray_orig = [cv.CreateImage((src.height, src.width), cv.IPL_DEPTH_8U, 1) for src in orig_images]
for ii in range(len(orig_images)):
    cv.CvtColor(orig_images[ii], gray_orig[ii], cv.CV_RGB2GRAY)

# The number of ranks and files in the chessboard. OpenCV considers
# the height and width of the chessboard to be one less than these,
# respectively.
rank_count = 11
file_count = 10

# Try to detect the corners of the chessboard. For each image,
# FindChessboardCorners returns (found, corner_points). found is True
# even if it managed to detect only a subset of the actual corners.
img_corners = [cv.FindChessboardCorners(img, (rank_count-1, file_count-1)) for img in gray_images]
orig_corners = [cv.FindChessboardCorners(img, (rank_count-1,file_count-1)) for img in gray_orig]

# The total number of corners will be (rank_count-1)x(file_count-1),
# but if some parts of the image are too blurred/distorted,
# FindChessboardCorners detects only a subset of the corners. In that
# case, DrawChessboardCorners will raise a TypeError.
orig_corner_success = []
ii = 0
for (found, corners) in orig_corners:
    if found and (len(corners) == (rank_count - 1) * (file_count - 1)):
        orig_corner_success.append(ii)
    else:
        print orig_files[ii], ': could not find correct corners: ', len(corners)
    ii += 1
ii = 0
img_corner_success = []
for (found, corners) in img_corners:
    if found and (len(corners) == (rank_count-1) * (file_count-1)) and (ii in orig_corner_success):
        img_corner_success.append(ii)
    else:
        print img_files[ii], ': Number of corners detected is wrong:', len(corners)
    ii += 1

# Here we compile all the corner coordinates into single arrays    
image_points = []
obj_points = []
for ii in img_corner_success:
    obj_points.extend(orig_corners[ii][1])
    image_points.extend(img_corners[ii][2])        
image_points = cv.fromarray(numpy.array(image_points, dtype='float32'))
obj_points = numpy.hstack((numpy.array(obj_points, dtype='float32'), numpy.zeros((len(obj_points), 1), dtype='float32')))
obj_points = cv.fromarray(numpy.array(obj_points, order='C'))

point_counts = numpy.ones((len(img_corner_success), 1), dtype='int32') * ((rank_count-1) * (file_count-1))
point_counts = cv.fromarray(point_counts)
# Create the output parameters
cam_mat = cv.CreateMat(3, 3, cv.CV_32FC1)
cv.Set2D(cam_mat, 0, 0, 1.0)
cv.Set2D(cam_mat, 1, 1, 1.0)
dist_mat = cv.CreateMat(5, 1, cv.CV_32FC1)
rot_vecs = cv.CreateMat(len(img_corner_success), 3, cv.CV_32FC1)
tran_vecs = cv.CreateMat(len(img_corner_success), 3, cv.CV_32FC1)
# Do the camera calibration
x = cv.CalibrateCamera2(obj_points, image_points, point_counts, cv.GetSize(gray_images[0]), cam_mat, dist_mat, rot_vecs, tran_vecs)
# Create the undistortion map
xmap = cv.CreateImage(cv.GetSize(images[0]), cv.IPL_DEPTH_32F, 1)
ymap = cv.CreateImage(cv.GetSize(images[0]), cv.IPL_DEPTH_32F, 1)
cv.InitUndistortMap(cam_mat, dist_mat, xmap, ymap)
# Now undistort all the images and same them
ii = 0
for tmp in images:
    print img_files[ii]
    image = cv.GetImage(tmp)
    t = cv.CloneImage(image)
    cv.Remap(t, image, xmap, ymap, cv.CV_INTER_LINEAR + cv.CV_WARP_FILL_OUTLIERS, cv.ScalarAll(0))
    corrected_file = os.path.join(os.path.dirname(img_files[ii]), 'corrected_%s' % (os.path.basename(img_files[ii])))
    cv.SaveImage(corrected_file, image)
    print 'Saved corrected image to', corrected_file
    ii += 1

以下是图片 - A，B和B'实际上我认为Remap并没有真正做任何事情！

A. Original Image B. Captured Image B'. Undistorted Image

Answer 1

我终于解决了。有几个问题：

原始图像的大小不同。捕获的图像也不是。 Hince，一对的仿射变换不适用于另一对。我将它们调整到相同的大小。
相机校准后的Undistort不足以进行旋转和剪切。适当的做法是仿射变换。最好将棋盘的三个角作为计算变换矩阵的点（较小的相对误差）。

这是我的工作代码（我正在转换原始图像并保存它们以显示契约中的计算变换矩阵将原始图像映射到捕获的图像）：

import pylab
import os
import cv
import cv2
import numpy

global_object_points = None
global_image_points = None
global_captured_corners = None
global_original_corners = None
global_success_index = None

global_font = cv.InitFont(cv.CV_FONT_HERSHEY_PLAIN, 1.0, 1.0)

def get_camera_calibration_data(original_image_list, captured_image_list, board_width, board_height):
    """Get the map for undistorting projected images by using a list of original chessboard images and the list of images that were captured by camera.

    original_image_list - list containing the original images (loaded as OpenCV image).

    captured_image_list - list containing the captured images.

    board_width - width of the chessboard (number of files - 1)

    board_height - height of the chessboard (number of ranks - 1)

    """
    global global_object_points
    global global_image_points
    global global_captured_corners
    global global_original_corners
    global global_success_index
    print 'get_undistort_map'
    corner_count = board_width * board_height
    # Try to detect the corners of the chessboard. For each image,
    # FindChessboardCorners returns (found, corner_points). found is
    # True even if it managed to detect only a subset of the actual
    # corners.  NOTE: according to
    # http://opencv.willowgarage.com/wiki/documentation/cpp/calib3d/findChessboardCorners,
    # no need for FindCornerSubPix after FindChessBoardCorners
    captured_corners = [cv.FindChessboardCorners(img, (board_width, board_height)) for img in captured_image_list]
    original_corners = [cv.FindChessboardCorners(img, (board_width, board_height)) for img in original_image_list]
    success_captured = [index for index in range(len(captured_image_list))
                        if captured_corners[index][0] and len(captured_corners[index][1]) == corner_count]
    success_original = [index for index in range(len(original_image_list))
                        if original_corners[index][0] and len(original_corners[index][2]) == corner_count]
    success_index = [index for index in success_captured if (len(captured_corners[index][3]) == corner_count) and (index in success_original)]
    global_success_index = success_index
    print global_success_index
    print 'Successfully found corners in image #s.', success_index
    cv.NamedWindow('Image', cv.CV_WINDOW_AUTOSIZE)
    for index in success_index:
        copy = cv.CloneImage(original_image_list[index])
        cv.DrawChessboardCorners(copy, (board_width, board_height), original_corners[index][4], corner_count)
        cv.ShowImage('Image', copy)
        a = cv.WaitKey(0)
        copy = cv.CloneImage(captured_image_list[index])
        cv.DrawChessboardCorners(copy, (board_width, board_height), captured_corners[index][5], corner_count)
        cv.ShowImage('Image', copy)
        a = cv.WaitKey(0)
    cv.DestroyWindow('Image')
    if not success_index:
        return
    global_captured_corners = [captured_corners[index][6] for index in success_index]
    global_original_corners = [original_corners[index][7] for index in success_index]
    object_points = cv.CreateMat(len(success_index) * (corner_count), 3, cv.CV_32FC1)
    image_points = cv.CreateMat(len(success_index) * (corner_count), 2, cv.CV_32FC1)
    global_object_points = object_points
    global_image_points = image_points
    point_counts = cv.CreateMat(len(success_index), 1, cv.CV_32SC1)
    for ii in range(len(success_index)):        
        for jj in range(corner_count):
            cv.Set2D(object_points, ii * corner_count + jj, 0, float(jj/board_width))
            cv.Set2D(object_points, ii * corner_count + jj, 1, float(jj%board_width))
            cv.Set2D(object_points, ii * corner_count + jj, 2, float(0.0))
            cv.Set2D(image_points, ii * corner_count + jj, 0, captured_corners[success_index[ii]][8][jj][0])
            cv.Set2D(image_points, ii * corner_count + jj, 1, captured_corners[success_index[ii]][9][jj][10])
        cv.Set1D(point_counts, ii, corner_count)
    # Create the output parameters    
    camera_intrinsic_mat = cv.CreateMat(3, 3, cv.CV_32FC1)
    cv.Set2D(camera_intrinsic_mat, 0, 0, 1.0)
    cv.Set2D(camera_intrinsic_mat, 1, 1, 1.0)
    distortion_mat = cv.CreateMat(5, 1, cv.CV_32FC1)
    rotation_vecs = cv.CreateMat(len(success_index), 3, cv.CV_32FC1)
    translation_vecs = cv.CreateMat(len(success_index), 3, cv.CV_32FC1)
    print 'Before camera clibration'
    # Do the camera calibration
    cv.CalibrateCamera2(object_points, image_points, point_counts, cv.GetSize(original_image_list[0]), camera_intrinsic_mat, distortion_mat, rotation_vecs, translation_vecs)
    return (camera_intrinsic_mat, distortion_mat, rotation_vecs, translation_vecs)

if __name__ == '__main__':
    # angles - the angles at which the picture was rotated 
    angles = [0, 36, 72, 108, 144, 180, 216, 252, 288, 324]
    # orig_files - list of original picture files used for projection
    orig_files =  ['../calibration/checkerboard/o_orig_%d.png' % (angle) for angle in angles]
    # img_files - projected image captured by camera
    img_files = ['../calibration/checkerboard/captured_imag_%d.bmp' % (angle) for angle in angles]

    # orig_files = ['o%d.png' % (angle) for angle in range(10, 40, 10)]
    # img_files = ['d%d.png' % (angle) for angle in range(10, 40, 10)]
    # Load the images
    print 'Loading images'
    captured_images = [cv.LoadImage(filename) for filename in img_files]
    orig_images = [cv.LoadImage(filename) for filename in orig_files]
    # Convert to grayscale
    gray_images = [cv.CreateImage((src.height, src.width), cv.IPL_DEPTH_8U, 1) for src in captured_images]
    for ii in range(len(captured_images)):
        cv.CvtColor(captured_images[ii], gray_images[ii], cv.CV_RGB2GRAY)
        cv.ShowImage('win', gray_images[ii])
        cv.WaitKey(0)
    cv.DestroyWindow('win')
    gray_orig = [cv.CreateImage((src.height, src.width), cv.IPL_DEPTH_8U, 1) for src in orig_images]
    for ii in range(len(orig_images)):
        cv.CvtColor(orig_images[ii], gray_orig[ii], cv.CV_RGB2GRAY)

    # The number of ranks and files in the chessboard. OpenCV considers
    # the height and width of the chessboard to be one less than these,
    # respectively.
    rank_count = 10
    file_count = 11
    camera_intrinsic_mat, distortion_mat, rotation_vecs, translation_vecs, = get_camera_calibration_data(gray_orig, gray_images, file_count-1, rank_count-1)
    xmap = cv.CreateImage(cv.GetSize(captured_images[0]), cv.IPL_DEPTH_32F, 1)
    ymap = cv.CreateImage(cv.GetSize(captured_images[0]), cv.IPL_DEPTH_32F, 1)
    cv.InitUndistortMap(camera_intrinsic_mat, distortion_mat, xmap, ymap)
    # homography = cv.CreateMat(3, 3, cv.CV_32F)
    map_matrix = cv.CreateMat(2, 3, cv.CV_32F)
    source_points = (global_original_corners[0][0], global_original_corners[0][file_count-2], global_original_corners[0][(rank_count-1) * (file_count-1) -1])
    image_points = (global_captured_corners[0][0], global_captured_corners[0][file_count-2], global_captured_corners[0][(rank_count-1) * (file_count-1) -1])
    # cv.GetPerspectiveTransform(source, target, homography)
    cv.GetAffineTransform(source_points, image_points, map_matrix)
    ii = 0
    cv.NamedWindow('OriginaImage', cv.CV_WINDOW_AUTOSIZE)
    cv.NamedWindow('CapturedImage', cv.CV_WINDOW_AUTOSIZE)
    cv.NamedWindow('FixedImage', cv.CV_WINDOW_AUTOSIZE)
    for image in gray_images:
        # The affine transform should be ideally calculated once
        # outside this loop, but as the transform looks different for
        # each image, I'll just calculate it independently to see the
        # applicability
        try:
            # Try to find ii in the list of successful corner
            # detection indices and if found, use the corners for
            # computing the affine transformation matrix. This is only
            # required when the optics changes between two
            # projections, which should not happend.
            jj = global_success_index.index(ii)
            source_points = [global_original_corners[jj][0], global_original_corners[jj][rank_count-1], global_original_corners[jj][-1]]
            image_points = [global_captured_corners[jj][0], global_captured_corners[jj][rank_count-1], global_captured_corners[jj][-1]]
            cv.GetAffineTransform(source_points, image_points, map_matrix)
            print '---------------------------------------------------------------------'
            print orig_files[ii], '<-->', img_files[ii]
            print '---------------------------------------------------------------------'
            for kk in range(len(source_points)):
                print source_points[kk]
                print image_points[kk]
        except ValueError:
            # otherwise use the last used transformation matrix
            pass

        orig = cv.CloneImage(orig_images[ii])        
        cv.PutText(orig, '%s: original' % (os.path.basename(orig_files[ii])), (100, 100), global_font, 0.0)
        cv.ShowImage('OriginalImage', orig)
        target = cv.CloneImage(image)
        target.origin = image.origin
        cv.SetZero(target)
        cv.Remap(image, target, xmap, ymap, cv.CV_INTER_LINEAR + cv.CV_WARP_FILL_OUTLIERS, cv.ScalarAll(0))
        cv.PutText(target, '%s: remapped' % (os.path.basename(img_files[ii])), (100, 100), global_font, 0.0)
        cv.ShowImage('CapturedImage', target)
        target = cv.CloneImage(orig_images[ii])
        cv.SetZero(target)
        cv.WarpAffine(orig_images[ii], target, map_matrix, cv.CV_INTER_LINEAR | cv.CV_WARP_FILL_OUTLIERS)
        corrected_file = os.path.join(os.path.dirname(img_files[ii]), 'corrected_%s' % (os.path.basename(img_files[ii])))
        cv.SaveImage(corrected_file, target)
        print 'Saved corrected image to', corrected_file
        # cv.WarpPerspective(image, target, homography, cv.CV_INTER_LINEAR | cv.CV_WARP_INVERSE_MAP | cv.CV_WARP_FILL_OUTLIERS)        
        cv.PutText(target, '%s: perspective-transformed' % (os.path.basename(img_files[ii])), (100, 100), global_font, 0.0)
        cv.ShowImage('FixedImage', target)
        print '==================================================================='
        cv.WaitKey(0)
        ii += 1
    cv.DestroyWindow('OriginalImage')
    cv.DestroyWindow('CapturedImage')
    cv.DestroyWindow('FixedImage')

图片：

原件： original image

捕获的图片： captured image

仿射变换原始图像： affine transformed original

现在应用于原始图像的逆变换应该可以解决问题。

如何变换图像以使投影图像与原始图像相同

1 个答案: