Question

在我的情况下，OpenCV StereoBM Depth Map返回的数据没有意义，无论参数调整如何。

我正在研究涉及OpenCV并使用立体视觉生成深度图的设计项目。我目前成功地加载了我的网络摄像头并使用StereoBM生成深度图。但是，结果数据目前并不有用，因为我的屏幕截图如下所示。所以我创建了一个小python应用程序，帮助我调整StereoBM参数，这些参数没有帮助。

我的问题是必须校准相机以便与StereoBM功能一起使用吗？

如果没有，有哪些替代方案可以帮助我提高结果（即提高分辨率，使用StereoSBGM等）

代码

import cv2
import time
import numpy as np
from Tkinter import *

oldVal = 15
def oddVals(n):
        global oldVal
        n = int(n)
        if not n % 2:
                window_size.set(n+1 if n > oldVal else n-1)
                oldVal = window_size.get()

minDispValues = [16,32,48,64]
def minDispCallback(n):
        n = int(n)
        newvalue = min(minDispValues, key=lambda x:abs(x-float(n)))
        min_disp.set(newvalue)

# Display the sliders to control the stereo vision 
master = Tk()

master.title("StereoBM Settings");

min_disp = Scale(master, from_=16, to=64, command=minDispCallback, length=600, orient=HORIZONTAL, label="Minimum Disparities")
min_disp.pack()
min_disp.set(16)

window_size = Scale(master, from_=5, to=255, command=oddVals, length=600, orient=HORIZONTAL, label="Window Size")
window_size.pack()
window_size.set(15)

Disp12MaxDiff = Scale(master, from_=5, to=30, length=600, orient=HORIZONTAL, label="Max Difference")
Disp12MaxDiff.pack()
Disp12MaxDiff.set(0)

UniquenessRatio = Scale(master, from_=0, to=30, length=600, orient=HORIZONTAL, label="Uniqueness Ratio")
UniquenessRatio.pack()
UniquenessRatio.set(15)

SpeckleRange = Scale(master, from_=0, to=60, length=600, orient=HORIZONTAL, label="Speckle Range")
SpeckleRange.pack()
SpeckleRange.set(34)

SpeckleWindowSize = Scale(master, from_=60, to=150, length=600, orient=HORIZONTAL, label="Speckle Window Size")
SpeckleWindowSize.pack()
SpeckleWindowSize.set(100)

master.update()

vcLeft = cv2.VideoCapture(0) # Load video campture for the left camera
#vcLeft.set(cv2.cv.CV_CAP_PROP_FRAME_WIDTH,420);
#vcLeft.set(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT,340);
vcLeft.set(3,640) # Set camera width
vcLeft.set(4,480) # Set camera height

vcRight = cv2.VideoCapture(1) # Load video capture for the right camera
#vcRight.set(cv2.cv.CV_CAP_PROP_FRAME_WIDTH,420);
#vcRight.set(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT,340);

firstTime = time.time() # First time log

totalFramesPassed = 0 # Number of frames passed

if vcLeft.isOpened() and vcRight.isOpened():
        rvalLeft, frameLeft = vcLeft.read()
        rvalRight, frameRight = vcRight.read()

else:
        rvalLeft = False
        rvalRight = False

while rvalLeft and rvalRight: # If the cameras are opened

        rvalLeft, frameLeft = vcLeft.read()

        rvalRight, frameRight = vcRight.read()

        cv2.putText(frameLeft, "FPS : " + str(totalFramesPassed / (time.time() - firstTime)),(40, 40), cv2.FONT_HERSHEY_SIMPLEX, 0.8, 150, 2, 10)

        cv2.imshow("Left Camera", frameLeft)

        cv2.putText(frameRight, "FPS : " + str(totalFramesPassed / (time.time() - firstTime)),(40, 40), cv2.FONT_HERSHEY_SIMPLEX, 0.8, 150, 2, 10)

        cv2.imshow("Right Camera", frameRight)

        frameLeftNew = cv2.cvtColor(frameLeft, cv2.COLOR_BGR2GRAY)

        frameRightNew = cv2.cvtColor(frameRight, cv2.COLOR_BGR2GRAY)

        num_disp = 112 - min_disp.get()

        stereo = cv2.StereoBM_create(numDisparities = num_disp, blockSize = window_size.get())

        stereo.setMinDisparity(min_disp.get())

        stereo.setNumDisparities(num_disp)

        stereo.setBlockSize(window_size.get())

        stereo.setDisp12MaxDiff(Disp12MaxDiff.get())

        stereo.setUniquenessRatio(UniquenessRatio.get())

        stereo.setSpeckleRange(SpeckleRange.get())

        stereo.setSpeckleWindowSize(SpeckleWindowSize.get())

        disparity = stereo.compute(frameLeftNew, frameRightNew).astype(np.float32) / 16.0

        disp_map = (disparity - min_disp.get())/num_disp

        cv2.imshow("Disparity", disp_map)

        master.update() # Update the slider options

        key = cv2.waitKey(20)

        totalFramesPassed = totalFramesPassed + 1 # One frame passed, increment

        if key == 27:

                break


vcLeft.release()

vcRight.release()

Answer 1

正如StereoBM opencv stereoBM doc的opencv文档中所述，这两个图像需要是“整流的立体声对”。

这意味着在计算差异之前，您需要纠正两个摄像头。

在计算差异之前，请查看stereo_match，了解如何纠正这两个摄像头。

当您使用stereoBM计算视差时，您正在查看两个图像中的并行极线的对应关系。这意味着期望图像以这样的距离对齐，使得两个图像中的相同行对应于空间中的相同行。整改过程负责这一点。

有关详细信息，请查看Rectification with opencv

Answer 2

我发现我们需要纠正这对以便使用StereoBM功能。此外，我发现虽然资源密集程度更高，但StereoSGBM功能给了我更优的结果。

如果将来有人需要校准他们的相机，您可以使用此代码来帮助您这样做：

# Imports
import cv2
import numpy as np

# Constants
leftCameraNumber = 2 # Number for left camera
rightCameraNumber = 1 # Number for right camera

numberOfChessRows = 6
numberOfChessColumns = 8
chessSquareSize = 30 # Length of square in millimeters

numberOfChessColumns = numberOfChessColumns - 1 # Update to reflect how many corners are inside the chess board
numberOfChessRows = numberOfChessRows - 1

objp = np.zeros((numberOfChessColumns*numberOfChessRows,3), np.float32)
objp[:,:2] = np.mgrid[0:numberOfChessRows,0:numberOfChessColumns].T.reshape(-1,2)*chessSquareSize

objectPoints = []
leftImagePoints = []
rightImagePoints = []

parameterCriteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)

# Code
print("Press \"n\" when you're done caputing checkerboards.")

vcLeft = cv2.VideoCapture(leftCameraNumber) # Load video campture for the left camera
vcLeft.set(cv2.CAP_PROP_FRAME_WIDTH,640*3/2);
vcLeft.set(cv2.CAP_PROP_FRAME_HEIGHT,480*3/2);

vcRight = cv2.VideoCapture(rightCameraNumber) # Load video capture for the right camera
vcRight.set(cv2.CAP_PROP_FRAME_WIDTH,640*3/2);
vcRight.set(cv2.CAP_PROP_FRAME_HEIGHT,480*3/2);

if vcLeft.isOpened() and vcRight.isOpened():
    rvalLeft, frameLeft = vcLeft.read()
    rvalRight, frameRight = vcRight.read()

else:
    rvalLeft = False
    rvalRight = False

# Number of succesful recognitions
checkerboardRecognitions = 0

while rvalLeft and rvalRight: # If the cameras are opened

    vcLeft.grab();

    vcRight.grab();

    rvalLeft, frameLeft = vcLeft.retrieve()

    rvalRight, frameRight = vcRight.retrieve()

    frameLeftNew = cv2.cvtColor(frameLeft, cv2.COLOR_BGR2GRAY)

    frameRightNew = cv2.cvtColor(frameRight, cv2.COLOR_BGR2GRAY)

    foundPatternLeft, cornersLeft = cv2.findChessboardCorners(frameLeftNew, (numberOfChessRows, numberOfChessColumns), None, cv2.CALIB_CB_ADAPTIVE_THRESH + cv2.CALIB_CB_NORMALIZE_IMAGE + cv2.CALIB_CB_FAST_CHECK)

    foundPatternRight, cornersRight = cv2.findChessboardCorners(frameRightNew, (numberOfChessRows, numberOfChessColumns), None, cv2.CALIB_CB_ADAPTIVE_THRESH + cv2.CALIB_CB_NORMALIZE_IMAGE + cv2.CALIB_CB_FAST_CHECK)


    if foundPatternLeft and foundPatternRight: # If found corners in this frame

        # Process the images and display the count of checkboards in our array
        checkerboardRecognitions = checkerboardRecognitions + 1
        print("Checker board recognitions: " + str(checkerboardRecognitions))

        objectPoints.append(objp)

        exactCornersLeft = cv2.cornerSubPix(frameLeftNew, cornersLeft, (11, 11), (-1, -1), parameterCriteria);
        leftImagePoints.append(exactCornersLeft)

        exactCornersRight = cv2.cornerSubPix(frameRightNew, cornersRight, (11, 11), (-1, -1), parameterCriteria);
        rightImagePoints.append(exactCornersRight)

        frameLeft = cv2.drawChessboardCorners(frameLeft, (numberOfChessRows, numberOfChessColumns), (exactCornersLeft), True);

        frameRight = cv2.drawChessboardCorners(frameRight, (numberOfChessRows, numberOfChessColumns), (exactCornersRight), True);


    # Display current webcams regardless if board was found or not
    cv2.imshow("Left Camera", frameLeft)

    cv2.imshow("Right Camera", frameRight)


    key = cv2.waitKey(250) # Give the frame some time

    if key == ord('n'):

        break

cameraMatrixLeft = np.zeros( (3,3) )
cameraMatrixRight = np.zeros( (3,3) )
distortionLeft = np.zeros( (8,1) )
distortionRight = np.zeros( (8,1) )
height, width = frameLeft.shape[:2]

rms, leftMatrix, leftDistortion, rightMatrix, rightDistortion, R, T, E, F = cv2.stereoCalibrate(objectPoints, leftImagePoints, rightImagePoints,  cameraMatrixLeft, distortionLeft, cameraMatrixRight, distortionRight, (width, height),parameterCriteria, flags=0)

arr1 = np.arange(8).reshape(2, 4)
arr2 = np.arange(10).reshape(2, 5)
np.savez('camera_calibration.npz', leftMatrix=leftMatrix, leftDistortion=leftDistortion, rightMatrix=rightMatrix, rightDistortion=rightDistortion, R=R, T=T, E=E, F=F)
print("Calibration Settings Saved to File!")

print("RMS:")
print(rms)
print("Left Matrix:")
print(leftMatrix)
print("Left Distortion:")
print(leftDistortion)
print("Right Matrix:")
print(rightMatrix)
print("Right Distortion:")
print(rightDistortion)
print("R:")
print(R)
print("T:")
print(T)
print("E:")
print(E)
print("F:")
print(F)


leftRectTransform, rightRectTransform, leftProjMatrix, rightProjMatrix, _, _, _ = cv2.stereoRectify(leftMatrix, leftDistortion, rightMatrix, rightDistortion,  (width, height), R, T, alpha=-1);
leftMapX, leftMapY = cv2.initUndistortRectifyMap(leftMatrix, leftDistortion, leftRectTransform, leftProjMatrix, (width, height), cv2.CV_32FC1);
rightMapX, rightMapY = cv2.initUndistortRectifyMap(rightMatrix, rightDistortion, rightRectTransform, rightProjMatrix, (width, height), cv2.CV_32FC1);

minimumDisparities = 0
maximumDisparities = 128

stereo = cv2.StereoSGBM_create(minimumDisparities, maximumDisparities, 18)

while True: # If the cameras are opened
    vcLeft.grab();

    vcRight.grab();

    rvalLeft, frameLeft = vcLeft.retrieve()

    rvalRight, frameRight = vcRight.retrieve()

    frameLeftNew = cv2.cvtColor(frameLeft, cv2.COLOR_BGR2GRAY)

    frameRightNew = cv2.cvtColor(frameRight, cv2.COLOR_BGR2GRAY)

    leftRectified = cv2.remap(frameLeftNew, leftMapX, leftMapY, cv2.INTER_LINEAR);

    rightRectified = cv2.remap(frameRightNew, rightMapX, rightMapY, cv2.INTER_LINEAR);

    disparity = stereo.compute(leftRectified, rightRectified)

    cv2.filterSpeckles(disparity, 0, 6000, maximumDisparities);

    cv2.imshow("Normalized Disparity", (disparity/16.0 - minimumDisparities)/maximumDisparities);

    cv2.imshow("Left Camera", leftRectified)

    cv2.imshow("Right Camera", rightRectified)


    key = cv2.waitKey(10) # Give the frame some time

    if key == 27:

        break

print("Finished!")

是否必须校准相机才能与StereoBM一起使用？

2 个答案: