我使用numpy和OpenCV创建了两帧Lucas-Kanade比例金字塔光流的实现，但是它的输出看起来不那么“清晰”，因为我使用的测试图像数据集的地面实况图像表明他们应该使用成为。我感觉好像算法有一个小错误导致它产生“合理”的结果，（例如，在没有太多运动的区域中运动非常少，并且似乎有一些方向的一致感），并且输出的“合理性”使我很难找到我所犯的错误，如果有的话。这是代码：
类MultiResLucasKanade2：
def __init__(self, frames, num_pyramid_layers, scale_factor = 2):
self.frames = frames
self.scale_factor = scale_factor
self.num_pyramid_layers = num_pyramid_layers
self.calculate_flows()
def calculate_flows(self):
'''function doesn't "save" any flows calculated right now -- just shows the "flow
image" of the flows it calculates'''
for frame_index in range(1, self.frames.shape[0]):
base_frame1 = self.frames[frame_index - 1]
base_frame2 = self.frames[frame_index]
frame1_pyr = self.build_pyramid(base_frame1)
frame2_pyr = self.build_pyramid(base_frame2)
flow_x_pyr = self.build_pyramid(np.zeros(base_frame1.shape))
flow_y_pyr = self.build_pyramid(np.zeros(base_frame2.shape))
flow_x_pyr[0], flow_y_pyr[0] = self.get_flows_between_frames(frame1_pyr[0], frame2_pyr[0])
upscaled_previous_flow_x = cv2.resize(flow_x_pyr[0], frame1_pyr[1].shape[::-1])*self.scale_factor
upscaled_previous_flow_y = cv2.resize(flow_y_pyr[0], frame1_pyr[1].shape[::-1])*self.scale_factor
flow_x_sum = cv2.resize(upscaled_previous_flow_x, base_frame1.shape[::-1], cv2.INTER_LINEAR)
flow_y_sum = cv2.resize(upscaled_previous_flow_y, base_frame1.shape[::-1], cv2.INTER_LINEAR)
for pyr_index in range(1, len(frame1_pyr)):
frame2 = frame2_pyr[pyr_index]
frame1 = self.flow_interpolate_image(frame1_pyr[pyr_index], upscaled_previous_flow_x, upscaled_previous_flow_y)
flow_x_pyr[pyr_index], flow_y_pyr[pyr_index] = self.get_flows_between_frames(frame1, frame2)
if pyr_index < len(frame1_pyr) - 1:
layer_scale_factor = (self.scale_factor)**(len(frame1_pyr) - 1 - pyr_index)
upscaled_x_pyr = cv2.resize(flow_x_pyr[pyr_index], base_frame1.shape[::-1], cv2.INTER_LINEAR) * layer_scale_factor
upscaled_y_pyr = cv2.resize(flow_y_pyr[pyr_index], base_frame1.shape[::-1], cv2.INTER_LINEAR) * layer_scale_factor
flow_x_sum += upscaled_x_pyr
flow_y_sum += upscaled_y_pyr
downscale_to_next_layer_factor = 1.0/((self.scale_factor)**(len(frame1_pyr) - 2 - pyr_index))
upscaled_previous_flow_x = cv2.resize(flow_x_sum, frame1_pyr[pyr_index+1].shape[::-1], cv2.INTER_LINEAR)*downscale_to_next_layer_factor
upscaled_previous_flow_y = cv2.resize(flow_y_sum, frame1_pyr[pyr_index+1].shape[::-1], cv2.INTER_LINEAR)*downscale_to_next_layer_factor
full_flow_image = self.get_flow_image(flow_x_sum, flow_y_sum)
Image.fromarray(full_flow_image).show()
Image.fromarray(self.flow_interpolate_image(frame1, flow_x_sum, flow_y_sum)).show()
def build_pyramid(self, image):
pyramid = [image]
for i in range(0, self.num_pyramid_layers - 1):
prev_pyr_image = pyramid[len(pyramid)-1]
append_pyr_image = cv2.GaussianBlur(prev_pyr_image, (5,5), 1.0)
append_pyr_image = cv2.resize(append_pyr_image, (int((prev_pyr_image.shape[1]+1)/self.scale_factor), int((prev_pyr_image.shape[0]+1)/self.scale_factor)))
pyramid.append(append_pyr_image)
pyramid = list(reversed(pyramid))
return pyramid
'''interpolates the first frame to fit the second'''
def flow_interpolate_image(self, image, x_flows, y_flows):
x_indices_mat = np.array([np.arange(image.shape[1]) for j in range(0, image.shape[0])])
y_indices_mat = np.array([np.arange(image.shape[0]) for j in range(0, image.shape[1])]).T
x_map = (x_indices_mat - x_flows).astype(np.float32)
y_map = (y_indices_mat - y_flows).astype(np.float32)
warp_image = cv2.remap(image, x_map, y_map, cv2.INTER_LINEAR)
return warp_image
def get_flows_between_frames(self, frame1, frame2):
WINDOW_SIZE = 3
WINDOW_MARGIN = (WINDOW_SIZE - 1)//2
x_flows = np.zeros((frame1.shape[0], frame1.shape[1]))
y_flows = np.zeros((frame1.shape[0], frame1.shape[1]))
frame2_grad_x = cv2.Scharr(frame2, cv2.CV_32F, 1, 0)
frame2_grad_y = cv2.Scharr(frame2, cv2.CV_32F, 0, 1)
partial_frame2_partial_time = frame2 - frame1
for x in range(WINDOW_MARGIN, x_flows.shape[1] - WINDOW_MARGIN):
for y in range(WINDOW_MARGIN, x_flows.shape[0] - WINDOW_MARGIN):
frame2_gradx_window = frame2_grad_x[y-WINDOW_MARGIN : y+WINDOW_MARGIN+1, x-WINDOW_MARGIN : x+WINDOW_MARGIN+1].flatten()
frame2_grady_window = frame2_grad_y[y-WINDOW_MARGIN : y+WINDOW_MARGIN+1, x-WINDOW_MARGIN : x+WINDOW_MARGIN+1].flatten()
frame2_gradients_window = np.zeros((frame2_gradx_window.shape[0], 2))
frame2_gradients_window[:, 0] = frame2_gradx_window
frame2_gradients_window[:, 1] = frame2_grady_window
frame2_partial_time_window = partial_frame2_partial_time[y-WINDOW_MARGIN : y+WINDOW_MARGIN+1, x-WINDOW_MARGIN : x+WINDOW_MARGIN+1].flatten()
covar_mat = frame2_gradients_window.T.dot(frame2_gradients_window)
if MatrixHelper.is_invertible(covar_mat):
mat_product = frame2_gradients_window.T.dot(-frame2_partial_time_window)
flow_steps = np.linalg.inv(covar_mat).dot(mat_product)
x_flows[y, x] = flow_steps[0]
y_flows[y, x] = flow_steps[1]
x_flows = cv2.resize(x_flows, frame1.shape[::-1], cv2.INTER_LINEAR)
y_flows = cv2.resize(y_flows, frame1.shape[::-1], cv2.INTER_LINEAR)
return x_flows, y_flows
def get_flow_image(self, x_flows, y_flows):
flow_stacks = np.dstack((x_flows, y_flows))
hsv_angles = np.arctan2(y_flows, x_flows)
hsv_angles = np.rad2deg(hsv_angles)%360
hsv_saturations = (np.linalg.norm(flow_stacks, axis = 2))
hsv_saturations /= np.amax(hsv_saturations)
flow_hsv_image = np.dstack((hsv_angles, hsv_saturations, np.ones((x_flows.shape[0], x_flows.shape[1])))).astype(np.float32)
flow_rgb_image = np.uint8(255 * cv2.cvtColor(flow_hsv_image, cv2.COLOR_HSV2RGB))
return flow_rgb_image
这里有一些示例帧和算法输出，它们是通过实例化Lucas-Kanade而创建的：
op_flow = MultiResLucasKanade2(np.array([bw_base_image, bw_fit_image]), 5, scale_factor = 2.0)
Input frames, algorithm output, ground truth, etc. of running the above code