base/string_tokenizer_unittest.cc

   1 // Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/string_tokenizer.h"
   6 #include "testing/gtest/include/gtest/gtest.h"
   7
   8 using std::string;
   9
  10 namespace {
  11 class StringTokenizerTest : public testing::Test {};
  12 }
  13
  14 TEST(StringTokenizerTest, Simple) {
  15   string input = "this is a test";
  16   StringTokenizer t(input, " ");
  17
  18   EXPECT_TRUE(t.GetNext());
  19   EXPECT_EQ(string("this"), t.token());
  20
  21   EXPECT_TRUE(t.GetNext());
  22   EXPECT_EQ(string("is"), t.token());
  23
  24   EXPECT_TRUE(t.GetNext());
  25   EXPECT_EQ(string("a"), t.token());
  26
  27   EXPECT_TRUE(t.GetNext());
  28   EXPECT_EQ(string("test"), t.token());
  29
  30   EXPECT_FALSE(t.GetNext());
  31 }
  32
  33 TEST(StringTokenizerTest, Reset) {
  34   string input = "this is a test";
  35   StringTokenizer t(input, " ");
  36
  37   for (int i = 0; i < 2; ++i) {
  38     EXPECT_TRUE(t.GetNext());
  39     EXPECT_EQ(string("this"), t.token());
  40
  41     EXPECT_TRUE(t.GetNext());
  42     EXPECT_EQ(string("is"), t.token());
  43
  44     EXPECT_TRUE(t.GetNext());
  45     EXPECT_EQ(string("a"), t.token());
  46
  47     EXPECT_TRUE(t.GetNext());
  48     EXPECT_EQ(string("test"), t.token());
  49
  50     EXPECT_FALSE(t.GetNext());
  51     t.Reset();
  52   }
  53 }
  54
  55 TEST(StringTokenizerTest, RetDelims) {
  56   string input = "this is a test";
  57   StringTokenizer t(input, " ");
  58   t.set_options(StringTokenizer::RETURN_DELIMS);
  59
  60   EXPECT_TRUE(t.GetNext());
  61   EXPECT_EQ(string("this"), t.token());
  62
  63   EXPECT_TRUE(t.GetNext());
  64   EXPECT_EQ(string(" "), t.token());
  65
  66   EXPECT_TRUE(t.GetNext());
  67   EXPECT_EQ(string("is"), t.token());
  68
  69   EXPECT_TRUE(t.GetNext());
  70   EXPECT_EQ(string(" "), t.token());
  71
  72   EXPECT_TRUE(t.GetNext());
  73   EXPECT_EQ(string("a"), t.token());
  74
  75   EXPECT_TRUE(t.GetNext());
  76   EXPECT_EQ(string(" "), t.token());
  77
  78   EXPECT_TRUE(t.GetNext());
  79   EXPECT_EQ(string("test"), t.token());
  80
  81   EXPECT_FALSE(t.GetNext());
  82 }
  83
  84 TEST(StringTokenizerTest, ManyDelims) {
  85   string input = "this: is, a-test";
  86   StringTokenizer t(input, ": ,-");
  87
  88   EXPECT_TRUE(t.GetNext());
  89   EXPECT_EQ(string("this"), t.token());
  90
  91   EXPECT_TRUE(t.GetNext());
  92   EXPECT_EQ(string("is"), t.token());
  93
  94   EXPECT_TRUE(t.GetNext());
  95   EXPECT_EQ(string("a"), t.token());
  96
  97   EXPECT_TRUE(t.GetNext());
  98   EXPECT_EQ(string("test"), t.token());
  99
 100   EXPECT_FALSE(t.GetNext());
 101 }
 102
 103 TEST(StringTokenizerTest, ParseHeader) {
 104   string input = "Content-Type: text/html ; charset=UTF-8";
 105   StringTokenizer t(input, ": ;=");
 106   t.set_options(StringTokenizer::RETURN_DELIMS);
 107
 108   EXPECT_TRUE(t.GetNext());
 109   EXPECT_FALSE(t.token_is_delim());
 110   EXPECT_EQ(string("Content-Type"), t.token());
 111
 112   EXPECT_TRUE(t.GetNext());
 113   EXPECT_TRUE(t.token_is_delim());
 114   EXPECT_EQ(string(":"), t.token());
 115
 116   EXPECT_TRUE(t.GetNext());
 117   EXPECT_TRUE(t.token_is_delim());
 118   EXPECT_EQ(string(" "), t.token());
 119
 120   EXPECT_TRUE(t.GetNext());
 121   EXPECT_FALSE(t.token_is_delim());
 122   EXPECT_EQ(string("text/html"), t.token());
 123
 124   EXPECT_TRUE(t.GetNext());
 125   EXPECT_TRUE(t.token_is_delim());
 126   EXPECT_EQ(string(" "), t.token());
 127
 128   EXPECT_TRUE(t.GetNext());
 129   EXPECT_TRUE(t.token_is_delim());
 130   EXPECT_EQ(string(";"), t.token());
 131
 132   EXPECT_TRUE(t.GetNext());
 133   EXPECT_TRUE(t.token_is_delim());
 134   EXPECT_EQ(string(" "), t.token());
 135
 136   EXPECT_TRUE(t.GetNext());
 137   EXPECT_FALSE(t.token_is_delim());
 138   EXPECT_EQ(string("charset"), t.token());
 139
 140   EXPECT_TRUE(t.GetNext());
 141   EXPECT_TRUE(t.token_is_delim());
 142   EXPECT_EQ(string("="), t.token());
 143
 144   EXPECT_TRUE(t.GetNext());
 145   EXPECT_FALSE(t.token_is_delim());
 146   EXPECT_EQ(string("UTF-8"), t.token());
 147
 148   EXPECT_FALSE(t.GetNext());
 149   EXPECT_FALSE(t.token_is_delim());
 150 }
 151
 152 TEST(StringTokenizerTest, ParseQuotedString) {
 153   string input = "foo bar 'hello world' baz";
 154   StringTokenizer t(input, " ");
 155   t.set_quote_chars("'");
 156
 157   EXPECT_TRUE(t.GetNext());
 158   EXPECT_EQ(string("foo"), t.token());
 159
 160   EXPECT_TRUE(t.GetNext());
 161   EXPECT_EQ(string("bar"), t.token());
 162
 163   EXPECT_TRUE(t.GetNext());
 164   EXPECT_EQ(string("'hello world'"), t.token());
 165
 166   EXPECT_TRUE(t.GetNext());
 167   EXPECT_EQ(string("baz"), t.token());
 168
 169   EXPECT_FALSE(t.GetNext());
 170 }
 171
 172 TEST(StringTokenizerTest, ParseQuotedString_Malformed) {
 173   string input = "bar 'hello wo";
 174   StringTokenizer t(input, " ");
 175   t.set_quote_chars("'");
 176
 177   EXPECT_TRUE(t.GetNext());
 178   EXPECT_EQ(string("bar"), t.token());
 179
 180   EXPECT_TRUE(t.GetNext());
 181   EXPECT_EQ(string("'hello wo"), t.token());
 182
 183   EXPECT_FALSE(t.GetNext());
 184 }
 185
 186 TEST(StringTokenizerTest, ParseQuotedString_Multiple) {
 187   string input = "bar 'hel\"lo\" wo' baz\"";
 188   StringTokenizer t(input, " ");
 189   t.set_quote_chars("'\"");
 190
 191   EXPECT_TRUE(t.GetNext());
 192   EXPECT_EQ(string("bar"), t.token());
 193
 194   EXPECT_TRUE(t.GetNext());
 195   EXPECT_EQ(string("'hel\"lo\" wo'"), t.token());
 196
 197   EXPECT_TRUE(t.GetNext());
 198   EXPECT_EQ(string("baz\""), t.token());
 199
 200   EXPECT_FALSE(t.GetNext());
 201 }
 202
 203 TEST(StringTokenizerTest, ParseQuotedString_EscapedQuotes) {
 204   string input = "foo 'don\\'t do that'";
 205   StringTokenizer t(input, " ");
 206   t.set_quote_chars("'");
 207
 208   EXPECT_TRUE(t.GetNext());
 209   EXPECT_EQ(string("foo"), t.token());
 210
 211   EXPECT_TRUE(t.GetNext());
 212   EXPECT_EQ(string("'don\\'t do that'"), t.token());
 213
 214   EXPECT_FALSE(t.GetNext());
 215 }
 216
 217 TEST(StringTokenizerTest, ParseQuotedString_EscapedQuotes2) {
 218   string input = "foo='a, b', bar";
 219   StringTokenizer t(input, ", ");
 220   t.set_quote_chars("'");
 221
 222   EXPECT_TRUE(t.GetNext());
 223   EXPECT_EQ(string("foo='a, b'"), t.token());
 224
 225   EXPECT_TRUE(t.GetNext());
 226   EXPECT_EQ(string("bar"), t.token());
 227
 228   EXPECT_FALSE(t.GetNext());
 229 }