boost/tokenizer.hpp

   1 // Boost tokenizer.hpp  -----------------------------------------------------//
   2
   3 // (c) Copyright Jeremy Siek and John R. Bandela 2001.
   4
   5 // Distributed under the Boost Software License, Version 1.0. (See
   6 // accompanying file LICENSE_1_0.txt or copy at
   7 // http://www.boost.org/LICENSE_1_0.txt)
   8
   9 // See http://www.boost.org/libs/tokenizer for documenation
  10
  11 // Revision History:
  12 // 03 Jul 2003   John Bandela
  13 //      Converted to new iterator adapter
  14 // 02 Feb 2002   Jeremy Siek
  15 //      Removed tabs and a little cleanup.
  16
  17 #ifndef BOOST_TOKENIZER_JRB070303_HPP_
  18 #define BOOST_TOKENIZER_JRB070303_HPP_
  19
  20 #include <boost/token_iterator.hpp>
  21
  22 namespace boost {
  23
  24
  25   //===========================================================================
  26   // A container-view of a tokenized "sequence"
  27   template <
  28     typename TokenizerFunc = char_delimiters_separator<char>,
  29     typename Iterator = std::string::const_iterator,
  30     typename Type = std::string
  31   >
  32   class tokenizer {
  33   private:
  34     typedef token_iterator_generator<TokenizerFunc,Iterator,Type> TGen;
  35
  36     // It seems that MSVC does not like the unqualified use of iterator,
  37     // Thus we use iter internally when it is used unqualified and
  38     // the users of this class will always qualify iterator.
  39     typedef typename TGen::type iter;
  40
  41   public:
  42
  43     typedef iter iterator;
  44     typedef iter const_iterator;
  45     typedef Type value_type;
  46     typedef value_type& reference;
  47     typedef const value_type& const_reference;
  48     typedef value_type* pointer;
  49     typedef const pointer const_pointer;
  50     typedef void size_type;
  51     typedef void difference_type;
  52
  53     tokenizer(Iterator first, Iterator last,
  54               const TokenizerFunc& f = TokenizerFunc())
  55       : first_(first), last_(last), f_(f) { }
  56
  57     template <typename Container>
  58     tokenizer(const Container& c)
  59       : first_(c.begin()), last_(c.end()), f_() { }
  60
  61     template <typename Container>
  62     tokenizer(const Container& c,const TokenizerFunc& f)
  63       : first_(c.begin()), last_(c.end()), f_(f) { }
  64
  65     void assign(Iterator first, Iterator last){
  66       first_ = first;
  67       last_ = last;
  68     }
  69
  70     void assign(Iterator first, Iterator last, const TokenizerFunc& f){
  71       assign(first,last);
  72       f_ = f;
  73     }
  74
  75     template <typename Container>
  76     void assign(const Container& c){
  77       assign(c.begin(),c.end());
  78     }
  79
  80
  81     template <typename Container>
  82     void assign(const Container& c, const TokenizerFunc& f){
  83       assign(c.begin(),c.end(),f);
  84     }
  85
  86     iter begin() const { return iter(f_,first_,last_); }
  87     iter end() const { return iter(f_,last_,last_); }
  88
  89   private:
  90     Iterator first_;
  91     Iterator last_;
  92     TokenizerFunc f_;
  93   };
  94
  95
  96 } // namespace boost
  97
  98 #endif