Question

我试图用C ++用YOLO模型编写一个推理程序。我已经搜索了有关Darknet的一些信息，但是它必须使用.cfg文件来导入模型结构（这对我来说有点太复杂了...），因此我想使用tensorflow进行程序。

（我的模型权重从.hdf5（在python中使用）转换为.pb（在C ++中使用））

我发现了一些用python编写的示例，似乎它们在推理过程之前已经做了一些工作... Source

def yolo_eval(yolo_outputs,
              anchors,
              num_classes,
              image_shape,
              max_boxes=50,
              score_threshold=.6,
              iou_threshold=.5):

    """Evaluate YOLO model on given input and return filtered boxes."""
    num_layers = len(yolo_outputs)
    anchor_mask = [[6,7,8], [3,4,5], [0,1,2]] if num_layers==3 else [[3,4,5], [1,2,3]] # default setting
    input_shape = K.shape(yolo_outputs[0])[1:3] * 32
    boxes = []
    box_scores = []
    for l in range(num_layers):
        _boxes, _box_scores = yolo_boxes_and_scores(yolo_outputs[l],
            anchors[anchor_mask[l]], num_classes, input_shape, image_shape)
        boxes.append(_boxes)
        box_scores.append(_box_scores)
    boxes = K.concatenate(boxes, axis=0)
    box_scores = K.concatenate(box_scores, axis=0)

    mask = box_scores >= score_threshold
    max_boxes_tensor = K.constant(max_boxes, dtype='int32')
    boxes_ = []
    scores_ = []
    classes_ = []
    for c in range(num_classes):
        # TODO: use keras backend instead of tf.
        class_boxes = tf.boolean_mask(boxes, mask[:, c])
        class_box_scores = tf.boolean_mask(box_scores[:, c], mask[:, c])
        nms_index = tf.image.non_max_suppression(
            class_boxes, class_box_scores, max_boxes_tensor, iou_threshold=iou_threshold)
        class_boxes = K.gather(class_boxes, nms_index)
        class_box_scores = K.gather(class_box_scores, nms_index)
        classes = K.ones_like(class_box_scores, 'int32') * c
        boxes_.append(class_boxes)
        scores_.append(class_box_scores)
        classes_.append(classes)
    boxes_ = K.concatenate(boxes_, axis=0)
    scores_ = K.concatenate(scores_, axis=0)
    classes_ = K.concatenate(classes_, axis=0)

    return boxes_, scores_, classes_

我已经打印出返回值看起来像这样

boxes-> Tensor（“ concat_11：0”，shape =（?, 4），dtype = float32）

得分-> Tensor（“ concat_12：0”，shape =（?,），dtype = float32）

classes-> Tensor（“ concat_13：0”，shape =（?,），dtype = int32）

我的YOLO模型（.hdf5）的原始输出是（我是通过打印出 model.output 得到的）

tf.Tensor'conv2d_59_1 / BiasAdd：0'shape =（?,？，？，21）dtype = float32

tf.Tensor'conv2d_67_1 / BiasAdd：0'shape =（?,？，？，21）dtype = float32

tf.Tensor'conv2d_75_1 / BiasAdd：0'shape =（?,？，？，21）dtype = float32

Python代码的推论部分是

out_boxes, out_scores, out_classes = sess.run(
                                    [boxes, scores, classes],
                                    feed_dict={
                                        yolo_model.input: image_data,
                                        input_image_shape: [image.size[1], image.size[0]],
                                        K.learning_phase(): 0
                                    })

与推理代码的python版本相比， C ++部分是...（Reference）

int main()
{
    string image = "test.jpg";
    string graph = "yolo_weight.pb";
    string labels = "coco.names";
    int32 input_width = 416;
    int32 input_height = 416;
    float input_mean = 0;
    float input_std = 255;
    string input_layer = "input_1:0";
    std::vector<std::string> output_layer = {"conv2d_59/BiasAdd:0", "conv2d_67/BiasAdd:0", "conv2d_75/BiasAdd:0" };

    std::unique_ptr<tensorflow::Session> session;
    string graph_path = tensorflow::io::JoinPath(root_dir, graph);
    Status load_graph_status = LoadGraph(graph_path, &session);

    std::vector<Tensor> resized_tensors;
    string image_path = tensorflow::io::JoinPath(root_dir, image);
    Status read_tensor_status = ReadTensorFromImageFile(image_path, input_height, input_width, 
    input_mean, input_std, &resized_tensors);

    Tensor inpTensor = Tensor(DT_FLOAT, TensorShape({ 1, input_height, input_width, 3 }));
    std::vector<Tensor> outputs;
    cv::Mat srcImage = cv::imread(image);
    cv::resize(srcImage, srcImage, cv::Size(input_width, input_height));
    srcImage.convertTo(srcImage, CV_32FC3);
    srcImage = srcImage / 255;  
    string ty = type2str(srcImage.type());
    float *p = (&inpTensor)->flat<float>().data();
    cv::Mat tensorMat(input_height, input_width, CV_32FC3, p);
    srcImage.convertTo(tensorMat, CV_32FC3);
    Status run_status = session->Run({{ input_layer, inpTensor }}, { output_layer }, {}, &outputs);
    int cc = 1;
    auto output_detection_class = outputs[0].tensor<float, 4>();
    std::cout << "detection scores" << std::endl;
    std::cout << "typeid(output_detection_scoreclass).name->" << typeid(output_detection_class).name() << std::endl;
    for (int i = 0; i < 13; ++i)
    {
        for (int j = 0; j < 13; ++j)
        {
            for (int k = 0; k < 21; ++k)
            {
                // using (index_1, index_2, index_3) to access the element in a tensor
                printf("i->%d, j->%d, k->%d\t", i, j, k);
                std::cout << output_detection_class(1, i, j, k) << "\t";
                cc += 1;
                if (cc % 4 == 0)
                {
                    std::cout << "\n";
                }
            }
        }
        std::cout << std::endl;
    }
    return 0;
}

c ++版本推断部分的输出是

outputs.size（）-> 3

outputs [0] .shape（）-> [1,13,13,21]

outputs [1] .shape（）-> [1,26,26,21]

outputs [2] .shape（）-> [1,52,52,21]

但是我得到的输出很奇怪...

（outputs [0]的输出值似乎不是分数，类或坐标中的任何一个...） Yolo output result

所以我想知道这是因为我在推断之前错过了用python编写的部分吗？还是我使用错误的方式获取输出数据？

我已经检查了一些相关的问题和答案...

1.Yolo v3 model output clarification with keras

2.Convert YoloV3 output to coordinates of bounding box, label and confidence

3.How to access tensorflow::Tensor C++

但是我仍然不知道该怎么做：（

我还发现了一个repo可能会有所帮助，我看过它的 yolo.cpp ，但是它的模型输出张量的形状与我的不同，我不确定是否可以直接修改代码，它的输出张量是

tf.Tensor'import / output：0'shape =（?, 735）dtype = float32

感谢任何帮助或建议...

Answer 1

如果您仍在为此苦苦挣扎，我看不到您将Sigmoid和Exp应用于输出图层值的地方。

您可能会看这篇介绍如何处理输出的论文。

https://medium.com/analytics-vidhya/yolo-v3-theory-explained-33100f6d193

Answer 2

正如 Bryan 所说，输出层仍有一些操作需要完成。

因此，在我的情况下（根据 this repo），我将其添加到 YOLO 类（在文件 yolo.py 中）以在保存模型时添加这些后处理：

def output_pb(self, out_dir, out_pb):

    out_bx = self.boxes.name.split(":")[0]
    out_sc = self.scores.name.split(":")[0]
    out_cs = self.classes.name.split(":")[0]
    print(out_bx, out_sc, out_cs)
    frozen_graph = tf.graph_util.remove_training_nodes(tf.graph_util.convert_variables_to_constants(self.sess, self.sess.graph.as_graph_def(), [out_bx, out_sc, out_cs]))
    tf.io.write_graph(frozen_graph, out_dir, out_pb, as_text=False)
    print("===== FINISH saving new pb file =====")

保存模型时，我这样调用函数：

yolo = YOLO(**config)
yolo.output_pb(output_dir, output_pb_name)

在 C++ 中进行推理时，整个过程是这样的：

// initialize model
YOLO* YOLO_data = (YOLO*)Init_DllODM_object(config);
// do some stuff to set data in YOLO_data
cv::Mat input_pic = "whatever_pic.png";
predict(YOLO_data, input_pic, YOLO_data ->bbox_res, YOLO_data ->score_res, YOLO_data ->class_res);
// draw result on pic
cv::Mat res = show_result(YOLO_data, input_pic);

详细代码在这里：

// yolo_cpp.h

struct YOLO
{
    float score_thres;
    std::vector<int> class_res;
    std::vector<float> bbox_res, score_res;

    std::string inp_tensor_name;
    std::string placeholder_name;
    std::vector<std::string> out_tensors;
    Session* session;

    Tensor t, inpTensor;
    std::vector<tensorflow::Tensor> outTensor;

    std::vector<int> MD_size;
    std::vector<int> inp_pic_size;
    std::vector<std::string> md_class_list;
    std::vector<cv::Scalar> color_list;
    int show_score;
    int score_type;
    int return_origin;
};

// yolo_cpp.cpp

void* Init_DllODM_object(json config)
{
    std::string model_path = config["model"].get<std::string>();
    YOLO* YOLO_data = new YOLO();
    auto options = tensorflow::SessionOptions();
    GraphDef graphdef;
    // loading model to graph
    Status status_load = ReadBinaryProto(Env::Default(), model_path, &graphdef);

    options.config.mutable_gpu_options()->set_per_process_gpu_memory_fraction(0.7);
    options.config.mutable_gpu_options()->set_allow_growth(true);

    int node_count = graphdef.node_size();
    for (int i = 0; i < node_count; i++)
    {
        auto n = graphdef.node(i);
        if (n.name().find("input_") != string::npos)
        {
            YOLO_data->inp_tensor_name = n.name();
        }
        else if (n.name().find("Placeholder_") != string::npos)
        {
            YOLO_data->placeholder_name = n.name();
        }
        else if (i == node_count - 5)
        {
            YOLO_data->out_tensors.push_back(n.name());
        }
        else if (i == node_count - 3)
        {
            YOLO_data->out_tensors.push_back(n.name());
        }
        else if (i == node_count - 1)
        {
            YOLO_data->out_tensors.push_back(n.name());
        }

    }
    if (!status_load.ok()) {
        std::cout << "ERROR: Loading model failed..." << std::endl;
        std::cout << model_path << status_load.ToString() << "\n";
    }

    std::vector<int> MD_size_ = config["input_size"];
    YOLO_data->MD_size = MD_size_;
    std::vector<int> inp_pic_size_ = config["input_pic_size"];
    YOLO_data->inp_pic_size = inp_pic_size_;

    YOLO_data->inpTensor = Tensor(DT_FLOAT, TensorShape({ 1, YOLO_data->MD_size[0], YOLO_data->MD_size[1], 3 }));  // input tensor
    YOLO_data->t = Tensor(DT_FLOAT, TensorShape({ 2 }));
    //ref: https://stackoverflow.com/questions/36804714/define-a-feed-dict-in-c-for-tensorflow-models
    auto t_matrix = YOLO_data->t.tensor<float, 1>();
    t_matrix(0) = YOLO_data->inp_pic_size[0];
    t_matrix(1) = YOLO_data->inp_pic_size[1];
    // create session
    Status status_newsess = NewSession(options, &YOLO_data->session); //for the usage of gpu setting
    Status status_create = YOLO_data->session->Create(graphdef);
    if (!status_create.ok()) {
        std::cout << "ERROR: Creating graph in session failed.." << status_create.ToString() << std::endl;
    }
    else {
        std::cout << "----------- Successfully created session and load graph -------------" << std::endl;
    }

    return YOLO_data;
}

int predict(YOLO* YOLO_, cv::Mat srcImage, std::vector<float>& bbox_res, std::vector<float>& score_res, std::vector<int>& class_res)
{
    // read image -> input image
    if (srcImage.empty())   // check if image can open correctly
    {
        std::cout << "can't open the image!!!!!!!" << std::endl;
        int res = -1;
        return res;
    }
    // ref: https://ppt.cc/f7ERNx
    std::vector<std::pair<string, tensorflow::Tensor>> inputs = {
        { YOLO_->inp_tensor_name, YOLO_->inpTensor },
        { YOLO_->placeholder_name, YOLO_->t },
    };
    srcImage = letterbox_image(srcImage, YOLO_->MD_size[0], YOLO_->MD_size[1]);
    convertCVMatToTensor(YOLO_, srcImage);
    Status status_run = YOLO_->session->Run({ inputs }, { YOLO_->out_tensors }, {}, &YOLO_->outTensor);
    if (!status_run.ok()) {
        std::cout << "ERROR: RUN failed..." << std::endl;
        std::cout << status_run.ToString() << "\n";
        int res = -1;
        return res;
    }

    TTypes<float>::Flat pp1 = YOLO_->outTensor[0].flat<float>();
    TTypes<float>::Flat pp2 = YOLO_->outTensor[1].flat<float>();
    TTypes<int>::Flat pp3 = YOLO_->outTensor[2].flat<int>();
    int pp1_idx;

    for (int i = 0; i < pp2.size(); i++)
    {
        pp1_idx = i * 4;
        bbox_res.push_back(pp1(pp1_idx));
        bbox_res.push_back(pp1(pp1_idx + 1));
        bbox_res.push_back(pp1(pp1_idx + 2));
        bbox_res.push_back(pp1(pp1_idx + 3));
        score_res.push_back(pp2(i));
        class_res.push_back(pp3(i));
    }

    return 0;
}

cv::Mat show_result(YOLO* inf_obj, cv::Mat inp_pic)
{
    int bbox_idx;
    std::string plot_str;
    bool under_thresh = false;
    std::vector<int> del_idx;
    for (int i = 0; i < inf_obj->class_res.size(); i++)
    {
        int y_min, y_max, x_min, x_max;
        bbox_idx = i * 4;
        y_min = std::max(0, (int)floor(inf_obj->bbox_res[bbox_idx] + 0.5));
        x_min = std::max(0, (int)floor(inf_obj->bbox_res[bbox_idx + 1] + 0.5));
        y_max = std::max(0, (int)floor(inf_obj->bbox_res[bbox_idx + 2] + 0.5));
        x_max = std::max(0, (int)floor(inf_obj->bbox_res[bbox_idx + 3] + 0.5));
        //std::cout << md_class_list[class_res[i]] << ", ";
        //std::cout << score_res[i] << ",";
        //std::cout << "[" << x_min << ", " << y_min << ", " << x_max << ", " << y_max << "]\n";

        if (inf_obj->show_score)
        {
            if (inf_obj->score_type)
                plot_str = inf_obj->md_class_list[inf_obj->class_res[i]] + ", " + std::to_string(rounding(inf_obj->score_res[i] * 100, 2)).substr(0, 5) + "%";
            else
                plot_str = inf_obj->md_class_list[inf_obj->class_res[i]] + ", " + std::to_string(rounding(inf_obj->score_res[i], 2)).substr(0, 4);
        }

        else
            plot_str = inf_obj->md_class_list[inf_obj->class_res[i]];

        if (inf_obj->score_res[i] >= inf_obj->score_thres)
        {
            inp_pic = plot_one_box(inp_pic, x_min, y_min, x_max, y_max, plot_str, inf_obj->color_list[inf_obj->class_res[i]]);
        }
        else
        {
            //std::cout << "score_res[i]->" << score_res[i] << "under thresh!!" << std::endl;
            under_thresh = true;
            del_idx.push_back(i);
        }
    }

    if (under_thresh)
    {
        //std::cout << "*** deleting element" << std::endl;
        for (int x = 0; x < del_idx.size(); x++)
        {
            bbox_idx = (del_idx[x] - x) * 4;
            inf_obj->bbox_res.erase(inf_obj->bbox_res.begin() + bbox_idx + 3);
            inf_obj->bbox_res.erase(inf_obj->bbox_res.begin() + bbox_idx + 2);
            inf_obj->bbox_res.erase(inf_obj->bbox_res.begin() + bbox_idx + 1);
            inf_obj->bbox_res.erase(inf_obj->bbox_res.begin() + bbox_idx);
            inf_obj->score_res.erase(inf_obj->score_res.begin() + del_idx[x] - x);
            inf_obj->class_res.erase(inf_obj->class_res.begin() + del_idx[x] - x);
        }
        del_idx.clear();
    }

    return inp_pic;
}

由于我的代码是用于dll的，所以我是这样安排的。还有一些多余的代码我没有删除，但我认为到目前为止，可以使用这些提供的代码完成整个过程。希望这有助于：D

如何使用带有C ++的tensorflow从YOLO模型正确获取输出？

2 个答案: