base/string_tokenizer_unittest.cc

   1 // Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/string_tokenizer.h"
   6 #include "testing/gtest/include/gtest/gtest.h"
   7
   8 using std::string;
   9
  10 namespace {
  11 class StringTokenizerTest : public testing::Test {};
  12 }
  13
  14 TEST(StringTokenizerTest, Simple) {
  15   string input = "this is a test";
  16   StringTokenizer t(input, " ");
  17
  18   EXPECT_TRUE(t.GetNext());
  19   EXPECT_EQ(string("this"), t.token());
  20
  21   EXPECT_TRUE(t.GetNext());
  22   EXPECT_EQ(string("is"), t.token());
  23
  24   EXPECT_TRUE(t.GetNext());
  25   EXPECT_EQ(string("a"), t.token());
  26
  27   EXPECT_TRUE(t.GetNext());
  28   EXPECT_EQ(string("test"), t.token());
  29
  30   EXPECT_FALSE(t.GetNext());
  31 }
  32
  33 TEST(StringTokenizerTest, RetDelims) {
  34   string input = "this is a test";
  35   StringTokenizer t(input, " ");
  36   t.set_options(StringTokenizer::RETURN_DELIMS);
  37
  38   EXPECT_TRUE(t.GetNext());
  39   EXPECT_EQ(string("this"), t.token());
  40
  41   EXPECT_TRUE(t.GetNext());
  42   EXPECT_EQ(string(" "), t.token());
  43
  44   EXPECT_TRUE(t.GetNext());
  45   EXPECT_EQ(string("is"), t.token());
  46
  47   EXPECT_TRUE(t.GetNext());
  48   EXPECT_EQ(string(" "), t.token());
  49
  50   EXPECT_TRUE(t.GetNext());
  51   EXPECT_EQ(string("a"), t.token());
  52
  53   EXPECT_TRUE(t.GetNext());
  54   EXPECT_EQ(string(" "), t.token());
  55
  56   EXPECT_TRUE(t.GetNext());
  57   EXPECT_EQ(string("test"), t.token());
  58
  59   EXPECT_FALSE(t.GetNext());
  60 }
  61
  62 TEST(StringTokenizerTest, ManyDelims) {
  63   string input = "this: is, a-test";
  64   StringTokenizer t(input, ": ,-");
  65
  66   EXPECT_TRUE(t.GetNext());
  67   EXPECT_EQ(string("this"), t.token());
  68
  69   EXPECT_TRUE(t.GetNext());
  70   EXPECT_EQ(string("is"), t.token());
  71
  72   EXPECT_TRUE(t.GetNext());
  73   EXPECT_EQ(string("a"), t.token());
  74
  75   EXPECT_TRUE(t.GetNext());
  76   EXPECT_EQ(string("test"), t.token());
  77
  78   EXPECT_FALSE(t.GetNext());
  79 }
  80
  81 TEST(StringTokenizerTest, ParseHeader) {
  82   string input = "Content-Type: text/html ; charset=UTF-8";
  83   StringTokenizer t(input, ": ;=");
  84   t.set_options(StringTokenizer::RETURN_DELIMS);
  85
  86   EXPECT_TRUE(t.GetNext());
  87   EXPECT_FALSE(t.token_is_delim());
  88   EXPECT_EQ(string("Content-Type"), t.token());
  89
  90   EXPECT_TRUE(t.GetNext());
  91   EXPECT_TRUE(t.token_is_delim());
  92   EXPECT_EQ(string(":"), t.token());
  93
  94   EXPECT_TRUE(t.GetNext());
  95   EXPECT_TRUE(t.token_is_delim());
  96   EXPECT_EQ(string(" "), t.token());
  97
  98   EXPECT_TRUE(t.GetNext());
  99   EXPECT_FALSE(t.token_is_delim());
 100   EXPECT_EQ(string("text/html"), t.token());
 101
 102   EXPECT_TRUE(t.GetNext());
 103   EXPECT_TRUE(t.token_is_delim());
 104   EXPECT_EQ(string(" "), t.token());
 105
 106   EXPECT_TRUE(t.GetNext());
 107   EXPECT_TRUE(t.token_is_delim());
 108   EXPECT_EQ(string(";"), t.token());
 109
 110   EXPECT_TRUE(t.GetNext());
 111   EXPECT_TRUE(t.token_is_delim());
 112   EXPECT_EQ(string(" "), t.token());
 113
 114   EXPECT_TRUE(t.GetNext());
 115   EXPECT_FALSE(t.token_is_delim());
 116   EXPECT_EQ(string("charset"), t.token());
 117
 118   EXPECT_TRUE(t.GetNext());
 119   EXPECT_TRUE(t.token_is_delim());
 120   EXPECT_EQ(string("="), t.token());
 121
 122   EXPECT_TRUE(t.GetNext());
 123   EXPECT_FALSE(t.token_is_delim());
 124   EXPECT_EQ(string("UTF-8"), t.token());
 125
 126   EXPECT_FALSE(t.GetNext());
 127   EXPECT_FALSE(t.token_is_delim());
 128 }
 129
 130 TEST(StringTokenizerTest, ParseQuotedString) {
 131   string input = "foo bar 'hello world' baz";
 132   StringTokenizer t(input, " ");
 133   t.set_quote_chars("'");
 134
 135   EXPECT_TRUE(t.GetNext());
 136   EXPECT_EQ(string("foo"), t.token());
 137
 138   EXPECT_TRUE(t.GetNext());
 139   EXPECT_EQ(string("bar"), t.token());
 140
 141   EXPECT_TRUE(t.GetNext());
 142   EXPECT_EQ(string("'hello world'"), t.token());
 143
 144   EXPECT_TRUE(t.GetNext());
 145   EXPECT_EQ(string("baz"), t.token());
 146
 147   EXPECT_FALSE(t.GetNext());
 148 }
 149
 150 TEST(StringTokenizerTest, ParseQuotedString_Malformed) {
 151   string input = "bar 'hello wo";
 152   StringTokenizer t(input, " ");
 153   t.set_quote_chars("'");
 154
 155   EXPECT_TRUE(t.GetNext());
 156   EXPECT_EQ(string("bar"), t.token());
 157
 158   EXPECT_TRUE(t.GetNext());
 159   EXPECT_EQ(string("'hello wo"), t.token());
 160
 161   EXPECT_FALSE(t.GetNext());
 162 }
 163
 164 TEST(StringTokenizerTest, ParseQuotedString_Multiple) {
 165   string input = "bar 'hel\"lo\" wo' baz\"";
 166   StringTokenizer t(input, " ");
 167   t.set_quote_chars("'\"");
 168
 169   EXPECT_TRUE(t.GetNext());
 170   EXPECT_EQ(string("bar"), t.token());
 171
 172   EXPECT_TRUE(t.GetNext());
 173   EXPECT_EQ(string("'hel\"lo\" wo'"), t.token());
 174
 175   EXPECT_TRUE(t.GetNext());
 176   EXPECT_EQ(string("baz\""), t.token());
 177
 178   EXPECT_FALSE(t.GetNext());
 179 }
 180
 181 TEST(StringTokenizerTest, ParseQuotedString_EscapedQuotes) {
 182   string input = "foo 'don\\'t do that'";
 183   StringTokenizer t(input, " ");
 184   t.set_quote_chars("'");
 185
 186   EXPECT_TRUE(t.GetNext());
 187   EXPECT_EQ(string("foo"), t.token());
 188
 189   EXPECT_TRUE(t.GetNext());
 190   EXPECT_EQ(string("'don\\'t do that'"), t.token());
 191
 192   EXPECT_FALSE(t.GetNext());
 193 }
 194
 195 TEST(StringTokenizerTest, ParseQuotedString_EscapedQuotes2) {
 196   string input = "foo='a, b', bar";
 197   StringTokenizer t(input, ", ");
 198   t.set_quote_chars("'");
 199
 200   EXPECT_TRUE(t.GetNext());
 201   EXPECT_EQ(string("foo='a, b'"), t.token());
 202
 203   EXPECT_TRUE(t.GetNext());
 204   EXPECT_EQ(string("bar"), t.token());
 205
 206   EXPECT_FALSE(t.GetNext());
 207 }
 208