caffe error == cudaSuccess(77 vs. 0)遇到非法内存访问

时间:2017-01-08 08:58:10

标签: c++ neural-network deep-learning caffe

最近,我需要在caffe中更改softmax_loss_layer的代码。 caffe版本是:https://github.com/BVLC/caffe

...之间的代码是我的代码,其余的是相同的。这就是我修改代码的方式 首先,我在loss_layers.hpp中添加了rs_

class SoftmaxWithLossLayer : public LossLayer<Dtype> {
...
Blob<Dtype> rs_;
...
}

然后,我在softmax_loss_layer.cpp

重塑rs_
void SoftmaxWithLossLayer<Dtype>::LayerSetUp(
const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) {
...
rs_.Reshape(bottom[2]->num(),bottom[2]->channels(),bottom[2]->height(),
bottom[2]->width());
}

Bottom [2]来自dense_image_data_layer.cpp

template <typename Dtype>
void DenseImageDataLayer<Dtype>::DataLayerSetUp(constvector<Blob<Dtype>*>& bottom,
  const vector<Blob<Dtype>*>& top) {
top[2]->Reshape(batch_size, 1, height, width);
this->prefetch_weight_.Reshape(batch_size, 1, height, width);
this->transformed_weight_.Reshape(1, 1, height, width);
}
Dtype* top_data = top[2]->mutable_cpu_data();
const int count = top[2]->count();
if(crop_size > 0) {
for(int index = 0;index < count;++index)
{
const int ww = index % crop_size;
const int wh = (index / crop_size) % crop_size;
top_data[index]=static_cast<Dtype>(cv_weight.at<uchar>(wh,ww));
}
}
else{
for(int index = 0;index < count;++index)
{
const int ww = index % width;
const int wh = (index / width) % height;
top_data[index]=static_cast<Dtype>(cv_weight.at<uchar>(wh,ww));
}
}

最后,我在softmax_loss_layer.cu

中添加了这段代码
template <typename Dtype>
void SoftmaxWithLossLayer<Dtype>::Forward_gpu(
const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) {
...
Dtype loss;
Dtype em;
bool add_weight=this->layer_param_.loss_param().add_weight();
Dtype* weight=bottom[2]->mutable_gpu_data();
Dtype z=0;
Dtype* rs=rs_.mutable_gpu_data();
if(add_weight)
{
SoftmaxLossWeightForwardGPU<Dtype><<<CAFFE_GET_BLOCKS(nthreads),
  CAFFE_CUDA_NUM_THREADS>>>(nthreads, prob_data, label, 
  weight_by_label_freqs_, label_count_data , loss_data, 
  outer_num_, dim, inner_num_, 
  has_ignore_label_, ignore_label_, counts,weight,rs);
const Dtype*rs1=rs_.gpu_data();
caffe_gpu_asum(nthreads, loss_data, &loss);
em=loss/nthreads;
count=nthreads;
Dtype am=1/2*log((1-em)/em);
CalculateZ<Dtype><<<CAFFE_GET_BLOCKS(nthreads),
CAFFE_CUDA_NUM_THREADS>>>(nthreads,weight,rs1,am,z,inner_num_);
WeightUpdate<Dtype><<<CAFFE_GET_BLOCKS(nthreads),
  CAFFE_CUDA_NUM_THREADS>>>(nthreads,weight,rs1,am,z,inner_num_);
...
}

template <typename Dtype>
void SoftmaxWithLossLayer<Dtype>::Backward_gpu(const    vector<Blob<Dtype>*>& top,const vector<bool>& propagate_down, const  vector<Blob<Dtype>*>& bottom) {
...
const Dtype* weight=bottom[2]->gpu_data();
bool add_weight=this->layer_param_.loss_param().add_weight();
if(add_weight)
SoftmaxLossWeightBackwardGPU<Dtype><<<CAFFE_GET_BLOCKS(nthreads),
    CAFFE_CUDA_NUM_THREADS>>>(nthreads, top_data, label,
    weight_by_label_freqs_, label_count_data, bottom_diff, 
    outer_num_, dim, inner_num_, has_ignore_label_, 
    ignore_label_, counts,weight);
...
}

错误是:

F0108 10:45:48.291290  2859 math_functions.cu:81] Check failed: error ==     cudaSuccess (77 vs. 0)  an illegal memory access was encountered
*** Check failure stack trace: ***
@     0x7f0b3bcfbdaa  (unknown)
@     0x7f0b3bcfbce4  (unknown)
@     0x7f0b3bcfb6e6  (unknown)
@     0x7f0b3bcfe687  (unknown)
@     0x7f0b3c16dd48  caffe::caffe_gpu_memcpy()
@     0x7f0b3c09b67e  caffe::SyncedMemory::gpu_data()
@     0x7f0b3c054472  caffe::Blob<>::gpu_data()
@     0x7f0b3c0b0568  caffe::Net<>::ForwardFromTo()
@     0x7f0b3c0b0947  caffe::Net<>::ForwardPrefilled()
@     0x7f0b3c08e555  caffe::Solver<>::Step()
@     0x7f0b3c08ee8f  caffe::Solver<>::Solve()
@           0x407806  train()
@           0x405d41  main
@     0x7f0b3b20dec5  (unknown)
@           0x4062ed  (unknown)
@              (nil)  (unknown)

我认为这个错误是因为我没有正确使用rs_或者底部[2]不正确。 我更改的所有代码都在上面发布,所以有人可以告诉我该怎么做? 如果您需要额外的信息,请告诉我?

1 个答案:

答案 0 :(得分:1)

很难遵循您的代码和更改 一些评论;

  1. os.path.realpath reshape rs_应该在reshape()方法中,而不是在setup()中。
  2. 最好使用rs_.ReshapeLike(*bottom[2])而不是明确枚举numchannels等。如果您要使用不同维度的Blob,该怎么办?
  3. 您是否测试了您修改过的图层?来自caffe wiki

      

    test/test_your_layer.cpp中编写测试。使用test/test_gradient_check_util.hpp检查您的前向和后向实现是否符合数字协议。