1 # Copyright (C) 2003-2007, 2009-2011 Nominum, Inc.
3 # Permission to use, copy, modify, and distribute this software and its
4 # documentation for any purpose with or without fee is hereby granted,
5 # provided that the above copyright notice and this permission notice
6 # appear in all copies.
8 # THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES
9 # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR
11 # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
14 # OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
21 Token
= dns
.tokenizer
.Token
23 class TokenizerTestCase(unittest
.TestCase
):
25 def testQuotedString1(self
):
26 tok
= dns
.tokenizer
.Tokenizer(r
'"foo"')
28 self
.failUnless(token
== Token(dns
.tokenizer
.QUOTED_STRING
, 'foo'))
30 def testQuotedString2(self
):
31 tok
= dns
.tokenizer
.Tokenizer(r
'""')
33 self
.failUnless(token
== Token(dns
.tokenizer
.QUOTED_STRING
, ''))
35 def testQuotedString3(self
):
36 tok
= dns
.tokenizer
.Tokenizer(r
'"\"foo\""')
38 self
.failUnless(token
== Token(dns
.tokenizer
.QUOTED_STRING
, '"foo"'))
40 def testQuotedString4(self
):
41 tok
= dns
.tokenizer
.Tokenizer(r
'"foo\010bar"')
43 self
.failUnless(token
== Token(dns
.tokenizer
.QUOTED_STRING
, 'foo\x0abar'))
45 def testQuotedString5(self
):
47 tok
= dns
.tokenizer
.Tokenizer(r
'"foo')
49 self
.failUnlessRaises(dns
.exception
.UnexpectedEnd
, bad
)
51 def testQuotedString6(self
):
53 tok
= dns
.tokenizer
.Tokenizer(r
'"foo\01')
55 self
.failUnlessRaises(dns
.exception
.SyntaxError, bad
)
57 def testQuotedString7(self
):
59 tok
= dns
.tokenizer
.Tokenizer('"foo\nbar"')
61 self
.failUnlessRaises(dns
.exception
.SyntaxError, bad
)
64 tok
= dns
.tokenizer
.Tokenizer('')
66 self
.failUnless(token
.is_eof())
69 tok
= dns
.tokenizer
.Tokenizer('')
72 self
.failUnless(token1
.is_eof() and token2
.is_eof())
75 tok
= dns
.tokenizer
.Tokenizer('\n')
78 self
.failUnless(token1
.is_eol() and token2
.is_eof())
81 tok
= dns
.tokenizer
.Tokenizer(' \n')
83 self
.failUnless(token1
.is_eol())
86 tok
= dns
.tokenizer
.Tokenizer(' \n')
87 token1
= tok
.get(want_leading
=True)
88 self
.failUnless(token1
.is_whitespace())
90 def testComment1(self
):
91 tok
= dns
.tokenizer
.Tokenizer(' ;foo\n')
93 self
.failUnless(token1
.is_eol())
95 def testComment2(self
):
96 tok
= dns
.tokenizer
.Tokenizer(' ;foo\n')
97 token1
= tok
.get(want_comment
= True)
99 self
.failUnless(token1
== Token(dns
.tokenizer
.COMMENT
, 'foo') and
102 def testComment3(self
):
103 tok
= dns
.tokenizer
.Tokenizer(' ;foo bar\n')
104 token1
= tok
.get(want_comment
= True)
106 self
.failUnless(token1
== Token(dns
.tokenizer
.COMMENT
, 'foo bar') and
109 def testMultiline1(self
):
110 tok
= dns
.tokenizer
.Tokenizer('( foo\n\n bar\n)')
111 tokens
= list(iter(tok
))
112 self
.failUnless(tokens
== [Token(dns
.tokenizer
.IDENTIFIER
, 'foo'),
113 Token(dns
.tokenizer
.IDENTIFIER
, 'bar')])
115 def testMultiline2(self
):
116 tok
= dns
.tokenizer
.Tokenizer('( foo\n\n bar\n)\n')
117 tokens
= list(iter(tok
))
118 self
.failUnless(tokens
== [Token(dns
.tokenizer
.IDENTIFIER
, 'foo'),
119 Token(dns
.tokenizer
.IDENTIFIER
, 'bar'),
120 Token(dns
.tokenizer
.EOL
, '\n')])
121 def testMultiline3(self
):
123 tok
= dns
.tokenizer
.Tokenizer('foo)')
124 tokens
= list(iter(tok
))
125 self
.failUnlessRaises(dns
.exception
.SyntaxError, bad
)
127 def testMultiline4(self
):
129 tok
= dns
.tokenizer
.Tokenizer('((foo)')
130 tokens
= list(iter(tok
))
131 self
.failUnlessRaises(dns
.exception
.SyntaxError, bad
)
133 def testUnget1(self
):
134 tok
= dns
.tokenizer
.Tokenizer('foo')
138 self
.failUnless(t1
== t2
and t1
.ttype
== dns
.tokenizer
.IDENTIFIER
and \
141 def testUnget2(self
):
143 tok
= dns
.tokenizer
.Tokenizer('foo')
147 self
.failUnlessRaises(dns
.tokenizer
.UngetBufferFull
, bad
)
149 def testGetEOL1(self
):
150 tok
= dns
.tokenizer
.Tokenizer('\n')
152 self
.failUnless(t
== '\n')
154 def testGetEOL2(self
):
155 tok
= dns
.tokenizer
.Tokenizer('')
157 self
.failUnless(t
== '')
159 def testEscapedDelimiter1(self
):
160 tok
= dns
.tokenizer
.Tokenizer(r
'ch\ ld')
162 self
.failUnless(t
.ttype
== dns
.tokenizer
.IDENTIFIER
and t
.value
== r
'ch\ ld')
164 def testEscapedDelimiter2(self
):
165 tok
= dns
.tokenizer
.Tokenizer(r
'ch\032ld')
167 self
.failUnless(t
.ttype
== dns
.tokenizer
.IDENTIFIER
and t
.value
== r
'ch\032ld')
169 def testEscapedDelimiter3(self
):
170 tok
= dns
.tokenizer
.Tokenizer(r
'ch\ild')
172 self
.failUnless(t
.ttype
== dns
.tokenizer
.IDENTIFIER
and t
.value
== r
'ch\ild')
174 def testEscapedDelimiter1u(self
):
175 tok
= dns
.tokenizer
.Tokenizer(r
'ch\ ld')
176 t
= tok
.get().unescape()
177 self
.failUnless(t
.ttype
== dns
.tokenizer
.IDENTIFIER
and t
.value
== r
'ch ld')
179 def testEscapedDelimiter2u(self
):
180 tok
= dns
.tokenizer
.Tokenizer(r
'ch\032ld')
181 t
= tok
.get().unescape()
182 self
.failUnless(t
.ttype
== dns
.tokenizer
.IDENTIFIER
and t
.value
== 'ch ld')
184 def testEscapedDelimiter3u(self
):
185 tok
= dns
.tokenizer
.Tokenizer(r
'ch\ild')
186 t
= tok
.get().unescape()
187 self
.failUnless(t
.ttype
== dns
.tokenizer
.IDENTIFIER
and t
.value
== r
'child')
189 if __name__
== '__main__':