我构建了此线程类以使用TensorRT运行推理：
class GPUThread(threading.Thread):
def __init__(self, engine_path):
threading.Thread.__init__(self)
self.engine_path = engine_path
self.engine = self.open_engine(engine_path)
def run(self):
cuda.init()
#self.dev = cuda.Device(0)
#self.ctx = self.dev.make_context()
self.rt_run()
#self.ctx.pop()
#del self.ctx
return
def rt_run(self):
with self.engine.create_execution_context() as context:
inputs, outputs, bindings, stream = self.allocate_buffers(self.engine)
# ... Retrieve image
self.load_input(inputs[0].host, image)
[output] = self.do_inference(
context,
bindings=bindings,
inputs=inputs,
outputs=outputs,
stream=stream
)
return
def load_input(self, pagelocked_buffer, image):
# ... Image transformations ...
# Copy to the pagelocked input buffer
np.copyto(pagelocked_buffer, crop_img)
return
def allocate_buffers(self, engine):
inputs = []
outputs = []
bindings = []
stream = cuda.Stream()
for binding in engine:
size = trt.volume(engine.get_binding_shape(binding)) * engine.max_batch_size
dtype = trt.nptype(engine.get_binding_dtype(binding))
# Allocate host and device buffers
host_mem = cuda.pagelocked_empty(size, dtype)
device_mem = cuda.mem_alloc(host_mem.nbytes)
# Append the device buffer to device bindings.
bindings.append(int(device_mem))
# Append to the appropriate list.
if engine.binding_is_input(binding):
inputs.append(HostDeviceMem(host_mem, device_mem))
else:
outputs.append(HostDeviceMem(host_mem, device_mem))
return inputs, outputs, bindings, stream
def run_inference(self, context, bindings, inputs, outputs, stream, batch_size=1):
# Transfer input data to the GPU.
[cuda.memcpy_htod_async(inp.device, inp.host, stream) for inp in inputs]
# Run inference.
context.execute_async(batch_size=batch_size, bindings=bindings, stream_handle=stream.handle)
# Transfer predictions back from the GPU.
[cuda.memcpy_dtoh_async(out.host, out.device, stream) for out in outputs]
# Synchronize the stream
stream.synchronize()
# Return only the host outputs.
return [out.host for out in outputs]
运行上面的代码时，出现错误：stream = cuda.Stream() pycuda._driver.LogicError: explicit_context_dependent failed: invalid device context - no currently active context?在上面的cuda.Stream()中调用了此函数allocate_buffers。
因此，我然后在run中尝试以下操作（请注意，这是上面的注释掉的代码）：
self.dev = cuda.Device(0)
self.ctx = self.dev.make_context()
self.rt_run()
self.ctx.pop()
del self.ctx
这导致我的系统在调用rt_run的{{1}}时完全冻结。我猜想在创建PyCuda上下文与创建TensorRT执行上下文之间存在冲突吗？我正在Jetson Nano上运行它。
如果删除create_execution_context代码，则可以分配缓冲区，并且上下文似乎处于活动状态并且可以在工作线程中找到。但是，如果没有TensorRT execution 上下文，我将无法进行推理。 create_execution_context不是上面execute_async的方法。
请注意，从主线程运行时，这些问题均不会出现。我可以像上面的代码那样使用PyCuda的autoinit并创建一个执行上下文。
因此，总而言之，在工作线程中，除非调用self.ctx，否则我将无法分配缓冲区，但这会导致self.dev.make_context调用使系统崩溃。如果我不调用create_execution_context，则无法在执行上下文中分配缓冲区，因为在self.dev.make_context中调用invalid device context时收到错误cuda.Stream()。
我正在运行什么：