Question

我正在使用boost::tokenizer来读取类似CSV的文件。我将令牌存储在std::vector中。它运行良好，但我想为每个标记仅存储boost::iterator。

我试过了：

#include <string>
#include <boost/tokenizer.hpp>
#include <boost/range/iterator_range.hpp>

typedef std::string::const_iterator string_iter;
typedef boost::iterator_range<string_iter> string_view;

int main(){
    std::string line;

    std::vector<string_view> contents;

    boost::tokenizer<boost::escaped_list_separator<char>, string_iter, string_view> tok(line.begin(), line.end());
    contents.assing(tok.begin(), tok.end());
}

但它无法编译：

/usr/include/boost/token_functions.hpp：实例化'bool boost :: escaped_list_separator :: operator（）（InputIterator＆amp;，InputIterator，Token＆amp;）[with InputIterator = __gnu_cxx :: __ normal_iterator＆gt ;;令牌= boost :: iterator_range＆lt; __ gnu_cxx :: __ normal_iterator＆gt;取代; Char = char;特征= std :: char_traits]'：/ usr /include/boost/token_iterator.hpp:70:11：来自'void boost :: token_iterator :: initialize（）[与TokenizerFunc = 提高:: escaped_list_separator;迭代器= __gnu_cxx :: __ normal_iterator＆gt ;; Type = boost :: iterator_range＆lt; __ gnu_cxx :: __ normal_iterator＆gt; ＆GT;]” /usr/include/boost/token_iterator.hpp:77:63：需要 'boost :: token_iterator :: token_iterator（TokenizerFunc，Iterator，Iterator）[with TokenizerFunc = boost :: escaped_list_separator;迭代器= __gnu_cxx :: __ normal_iterator＆gt ;; Type = boost :: iterator_range＆lt; __ gnu_cxx :: __ normal_iterator＆gt; ＆gt;]'/usr/include/boost/tokenizer.hpp:86:53：需要'boost :: tokenizer :: iter boost :: tokenizer :: begin（）const [with TokenizerFunc = boost :: escaped_list_separator;迭代器= __gnu_cxx :: __ normal_iterator＆gt ;; Type = boost :: iterator_range＆lt; __ gnu_cxx :: __ normal_iterator＆gt;取代; boost :: tokenizer :: iter = 提高:: token_iterator， __gnu_cxx :: __ normal_iterator＆gt ;, boost :: iterator_range＆lt; __ gnu_cxx :: __ normal_iterator＆gt; ＆GT; ＆GT;]” /home/wichtounet/dev/gooda-to-afdo-converter/src/gooda_reader.cpp:58:37：从这里需要/usr/include/boost/token_functions.hpp:187:16：错误：'tok + =中没有匹配'operator + ='（＆amp; 下） - ＆GT; __ gnu_cxx :: __ normal_iterator＆LT; _Iterator， _Container＆gt; :: operator *＆gt;（）'/ usr /include/boost/token_functions.hpp:193:11：错误：不匹配 “to + +”中的'operator + ='（＆amp; 下） - ＆GT; __ gnu_cxx :: __ normal_iterator＆LT; _Iterator， _Container＆gt; :: operator *＆gt;（）'/ usr /include/boost/token_functions.hpp：在'void的实例化中提高:: escaped_list_separator :: do_escape（迭代器和放大器;， iterator，Token＆amp;）[with iterator = __gnu_cxx :: __ normal_iterator＆gt ;;令牌= boost :: iterator_range＆lt; __ gnu_cxx :: __ normal_iterator＆gt;取代; Char = char;特征= 的std :: char_traits]”： /usr/include/boost/token_functions.hpp:176:11：来自'bool boost :: escaped_list_separator :: operator（）（InputIterator＆amp;，InputIterator，Token＆amp;）[with InputIterator = __gnu_cxx :: __ normal_iterator＆gt ;;令牌= boost :: iterator_range＆lt; __ gnu_cxx :: __ normal_iterator＆gt;取代; Char = char;特征= std :: char_traits]'/usr/include/boost/token_iterator.hpp:70:11：来自'void boost :: token_iterator :: initialize（）[与TokenizerFunc = 提高:: escaped_list_separator;迭代器= __gnu_cxx :: __ normal_iterator＆gt ;; Type = boost :: iterator_range＆lt; __ gnu_cxx :: __ normal_iterator＆gt; ＆GT;]” /usr/include/boost/token_iterator.hpp:77:63：需要 'boost :: token_iterator :: token_iterator（TokenizerFunc，Iterator，Iterator）[with TokenizerFunc = boost :: escaped_list_separator;迭代器= __gnu_cxx :: __ normal_iterator＆gt ;; Type = boost :: iterator_range＆lt; __ gnu_cxx :: __ normal_iterator＆gt; ＆gt;]'/usr/include/boost/tokenizer.hpp:86:53：需要'boost :: tokenizer :: iter boost :: tokenizer :: begin（）const [with TokenizerFunc = boost :: escaped_list_separator;迭代器= __gnu_cxx :: __ normal_iterator＆gt ;; Type = boost :: iterator_range＆lt; __ gnu_cxx :: __ normal_iterator＆gt;取代; boost :: tokenizer :: iter = 提高:: token_iterator， __gnu_cxx :: __ normal_iterator＆gt ;, boost :: iterator_range＆lt; __ gnu_cxx :: __ normal_iterator＆gt; ＆GT; ＆GT;]” /home/wichtounet/dev/gooda-to-afdo-converter/src/gooda_reader.cpp:58:37：从这里需要/usr/include/boost/token_functions.hpp:130:9：错误：'tok + ='\ 012''中的'operator + ='不匹配 /usr/include/boost/token_functions.hpp:134:9：错误：不匹配 “to + +”中的'operator + ='（＆amp; 下） - ＆GT; __ gnu_cxx :: __ normal_iterator＆LT; _Iterator， _Container＆gt; :: operator *＆gt;（）'/ usr /include/boost/token_functions.hpp:138:9：错误：不匹配 “to + +”中的'operator + ='（＆amp; 下） - ＆GT; __ gnu_cxx :: __ normal_iterator＆LT; _Iterator， _Container＆gt; :: operator *＆gt;（）'/ usr /include/boost/token_functions.hpp:142:9：错误：不匹配 “to + +”中的'operator + ='（＆amp; 下） - ＆GT; __ gnu_cxx :: __ normal_iterator＆LT; _Iterator， _Container＆gt; :: operator *＆gt;（）'

我也只是尝试使用boost::token_iterator自己计算两个迭代器，但到目前为止我还没有成功。

是否有解决方案只获取每个令牌的迭代器范围而不是字符串以节省一些性能？

Answer 1

啊！你需要一个包括：

#include <iostream>
#include <boost/tokenizer.hpp>
#include <boost/range/iterator_range.hpp>
#include <string>

int main()
{
    std::string line;
    typedef std::string::const_iterator string_iter;
    typedef boost::iterator_range<string_iter> string_view;

    boost::tokenizer<boost::escaped_list_separator<char>, string_iter, string_view> tok(line.begin(), line.end());
}

compiles fine

Answer 2

这不起作用。 tokenizer需要一个类型（第三个模板参数），它可以附加tokenizer函数的结果。具体而言，它必须提供运营商+= ( tokenizer<...>::iterator::value_type )。下面的code snippet应该会让你更进一步，虽然我不确定这是否值得努力......

#include <string>
#include <boost/tokenizer.hpp>
#include <boost/range/iterator_range.hpp>
#include <iostream>
#include <cstddef>

typedef std::string::const_iterator string_iter;
typedef boost::iterator_range<string_iter> string_view;

// a constant size character buffer, skips anything beyond CSize...
template< std::size_t CSize >
class assignable_view {
   std::size_t m_size;
   char m_buffer[CSize];

   friend std::ostream& operator << (std::ostream& p_out, assignable_view const & p_view)
   {
      if (p_view.m_size > 0u) {
         std::copy(p_view.m_buffer, p_view.m_buffer + p_view.m_size, std::ostream_iterator<char>(p_out));
      }
      return p_out;
   }

public:
   template <class TIter>
   void operator += (TIter p_input) 
   {
      if (m_size < CSize) {
         m_buffer[m_size++] = p_input;
      }   
   }   
   assignable_view() 
      : m_size(0u) {}
};

int main(){
    std::string line
        = "Field 1,\"putting quotes around fields, allows commas\",Field 3";

    std::vector<string_view> contents;

    boost::tokenizer<
       boost::escaped_list_separator<char>, 
       string_iter, 
       assignable_view<11>    
    > tok(line.begin(), line.end());

    for (auto const & t_s : tok) {
       std::cout << t_s << std::endl;
    }
    //contents.assing(tok.begin(), tok.end());
}

将boost :: tokenizer与boost :: iterator_range一起使用

2 个答案: