1 --[[--------------------------------------------------------------------
3 Gazelle: a system for building fast, reusable parsers
7 Tests for the LL(*) lookahead generation. This is the among the
8 most complicated things the complier does, and it has a lot of edge
9 cases and failure modes, so this file is complicated and subtle.
11 Copyright (c) 2008 Joshua Haberman. See LICENSE for details.
13 --------------------------------------------------------------------]]--
17 require
"fa_algorithms"
18 require
"bootstrap/rtn"
21 function find_state(rule
, slotnum
)
26 for s
in each(rule
:states()) do
27 for edge_val
, dest_state
, properties
in s
:transitions() do
28 if properties
.slotnum
== slotnum
then
34 error(string.format("No such slotnum (%d) for rule %s", slotnum
, rule_str
))
37 function get_target_for_slotnum(state
, slotnum
)
41 for edge_val
, dest_state
, properties
in state
:transitions() do
42 if properties
.slotnum
== slotnum
then
43 return {edge_val
, dest_state
}
47 error(string.format("Slotnum %d not found for state %s", slotnum
, serialize(state
, 6, " ")))
50 function parse_gla(str
, rtn_state
)
51 local stream
= CharStream
:new(str
)
52 stream
:ignore("whitespace")
53 local gla
= fa
.GLA
:new()
54 gla
.rtn_state
= rtn_state
55 local states
= {[1]=gla
.start
}
56 while not stream
:eof() do
57 local statenum
= tonumber(stream
:consume_pattern("%d+"))
58 local state
= states
[statenum
]
60 error(string.format("GLA refers to state %d before it is used as a target",
63 while stream
:lookahead(1) ~= ";" do
65 local term
= stream
:consume_pattern("%w+")
67 local dest_state_num
= tonumber(stream
:consume_pattern("%d+"))
68 states
[dest_state_num
] = states
[dest_state_num
] or fa
.GLAState
:new()
69 local dest_state
= states
[dest_state_num
]
71 state
:add_transition(fa
.eof
, dest_state
)
73 state
:add_transition(term
, dest_state
)
75 if stream
:lookahead(1) == "(" then
77 local final_state_slotnum
= tonumber(stream
:consume_pattern("%d+"))
79 dest_state
.final
= get_target_for_slotnum(rtn_state
, final_state_slotnum
)
89 function assert_lookahead(grammar_str
, rule_str
, slotnum
, expected_gla_str
, k
)
90 local grammar
= parse_grammar(CharStream
:new(grammar_str
))
91 grammar
:assign_priorities()
92 grammar
:determinize_rtns()
93 grammar
:minimize_rtns()
95 local rule
= grammar
.rtns
:get(rule_str
)
96 local state
= find_state(rule
, slotnum
)
97 local expected_gla
= parse_gla(expected_gla_str
, state
)
99 compute_lookahead(grammar
, k
)
101 if not fa_isequal(expected_gla
, state
.gla
) then
102 local bad
= io
.open("bad.dot", "w")
103 bad
:write("digraph untitled {\n")
104 bad
:write(state
.gla
:to_dot())
108 local good
= io
.open("good.dot", "w")
109 good
:write("digraph untitled {\n")
110 good
:write(expected_gla
:to_dot())
114 os
.execute("dot -Tpng -o good.png good.dot")
115 os
.execute("dot -Tpng -o bad.png bad.dot")
117 error("GLAs were not equal: expected and actual are " ..
118 "in good.png and bad.png, respectively")
123 function TestSimple
:test1()
137 function TestSimple
:test_multiple_recursions()
153 function TestEpsilon
:test1()
168 function TestEpsilon
:test2()
183 function TestEpsilon
:test3()
199 function TestEpsilon
:test4()
216 TestMultipleNonterms
= {}
217 function TestMultipleNonterms
:test1()
234 function TestMultipleNonterms
:test22()
254 function TestLL1
:test1()
268 function TestLL2
:test1()
271 s -> "X" "Y" | "X" "Z";
282 function TestLL2
:test2()
297 function TestLL2
:test3()
313 function TestLL2
:test3()
316 s -> "X"? "Y" | "X" "Z";
328 function TestLL2
:test3()
331 s -> "X" s | "X" "Y";
341 function TestLL2
:test4()
356 function TestLL2
:test5()
370 function TestLL2
:test6()
374 a -> "X" "Y" | "X" "Z";
386 function TestLL3
:test1()
390 a -> ("P" | "Q") ("P" | "Q");
404 -- This is equivalent to the grammar on page 271 of The Definitive ANTLR Reference.
405 function TestLL3
:test3()
408 s -> "X" s "Y" | "X" "X" "Z";
419 function TestLL3
:test3()
422 s -> a a "Y" | a a "Z";
427 1 -X-> 2 -X-> 3 -Y-> 4(1);
434 function TestEOF
:test1()
448 function TestEOF
:test2()
449 -- this is the example used by Terence Parr in his discussion of ANTLR 3.0's
450 -- lookahead analysis: http://www.antlr.org/blog/antlr3/lookahead.tml
467 function TestEOF
:test3()
481 -- This is really a "follow" test
482 function TestEOF
:test4()
499 function TestLLStar
:test1()
514 -- Test lookahead that we can only compute correctly if we apply the
515 -- tail-recursion optimization.
516 function TestLLStar
:test2()
532 function TestFollow
:test1()
547 function TestFollow
:test2()
561 function assert_fails_with_error(grammar_str
, error_string
)
562 local grammar
= parse_grammar(CharStream
:new(grammar_str
))
563 grammar
:assign_priorities()
564 grammar
:determinize_rtns()
565 grammar
:minimize_rtns()
567 local success
, message
= pcall(compute_lookahead
, grammar
)
569 error("Failed to fail!")
570 elseif not message
:find(error_string
) then
571 error("Failed with wrong message! Message was supposed to contain "
572 .. error_string
.. ", but it was: " .. message
)
576 function assert_left_recursive(grammar_str
)
577 assert_fails_with_error(grammar_str
, "it is left%-recursive")
580 function assert_nonregular(grammar_str
)
581 assert_fails_with_error(grammar_str
, "one lookahead language was nonregular, others were not all fixed")
584 function assert_ambiguous(grammar_str
)
585 assert_fails_with_error(grammar_str
, "Ambiguous grammar")
588 function assert_no_nonrecursive_alt(grammar_str
)
589 assert_fails_with_error(grammar_str
, "no non%-recursive alternative")
592 function assert_not_ll(grammar_str
)
593 assert_fails_with_error(grammar_str
, "It is not Strong%-LL or full%-LL")
596 function assert_never_taken(grammar_str
)
597 assert_fails_with_error(grammar_str
, "will never be taken")
600 function assert_resolution_not_supported(grammar_str
)
601 assert_fails_with_error(grammar_str
, "cannot support this resolution of the ambiguity")
604 TestDetectNonLLStar
= {}
605 function TestDetectNonLLStar
:test_left_recursive()
606 assert_left_recursive(
613 function TestDetectNonLLStar
:test_left_recursive2()
614 assert_left_recursive(
622 function TestDetectNonLLStar
:test_left_recursive3()
623 assert_left_recursive(
630 -- Hmm, technically this test is broken because this is left-recursion
631 -- that is getting incorrectly reported as ambiguity instead. But
632 -- fixing this one isn't a high priority right now.
633 -- function TestDetectNonLLStar:test_left_recursive4()
634 -- assert_left_recursive(
643 function TestDetectNonLLStar
:test_nonregular()
647 e -> "(" e ")" | "ID";
652 function TestDetectNonLLStar
:test_fails_heuristic_but_is_ll()
655 s -> "X"* "Y" "Y" "Z"| "X" c;
656 c -> "Y" c "Y" | "Q";
661 s -> "X"* "Y" "Y" "Z"| "X" c;
662 c -> "Y" c "Y" | "Q";
667 1 -X-> 3 -Y-> 4 -Y-> 5 -Y-> 6(5);
678 function TestDetectNonLLStar
:test_not_full_ll_1()
688 function TestDetectNonLLStar
:test_not_full_ll_1()
696 function TestDetectNonLLStar
:test_not_full_ll_2()
699 s -> "if" e "then" s ("else" s)? | e;
707 function TestAmbiguity
:test1()
717 function TestAmbiguity
:test2()
727 function TestAmbiguity
:test3()
737 function TestAmbiguity
:test4()
746 function TestAmbiguity
:test5()
755 function TestAmbiguity
:test6()
764 function TestAmbiguity
:test7()
772 function TestAmbiguity
:test8()
780 function TestAmbiguity
:test9()
788 function TestAmbiguity
:test10()
798 function TestAmbiguity
:test11()
807 function TestAmbiguity
:test12()
816 -- These tests are currently failing because I have not implemented this
818 TestNoNonRecursiveAlt
= {}
819 function TestNoNonRecursiveAlt
:test1()
820 assert_no_nonrecursive_alt(
827 function TestNoNonRecursiveAlt
:test2()
828 assert_no_nonrecursive_alt(
835 function TestNoNonRecursiveAlt
:test3()
836 assert_no_nonrecursive_alt(
844 function TestNoNonRecursiveAlt
:test4()
845 assert_no_nonrecursive_alt(
854 function TestNoNonRecursiveAlt
:test4()
855 assert_no_nonrecursive_alt(
863 TestAmbiguityResolution
= {}
864 function TestAmbiguityResolution
:test1()
867 s -> "X" "Y" / "X"+ "Y";
877 function TestAmbiguityResolution
:test2()
880 s -> "if" e "then" s ("else" s)?+ | e;
891 function TestAmbiguityResolution
:test_never_taken1()
894 s -> "X" "Y" / "X" "Y";
899 -- This test is currently failing -- it's throwing the wrong error message.
900 -- Instead of warning you that one transition is never taken, it just says
901 -- that Gazelle can't support this resolution of the ambiguity.
902 function TestAmbiguityResolution
:test_never_taken2()
912 function TestAmbiguityResolution
:test_never_taken3()
915 s -> "X"* "Y" / "X" "Y";
920 function TestAmbiguityResolution
:test_resolution_not_supported1()
921 assert_resolution_not_supported(
923 s -> "S" / "S" s "S";
928 function TestDetectNonLLStar
:test_resolution_not_supported2()
929 assert_resolution_not_supported(
931 s -> "if" e "then" s ("else" s)?- | e;
938 LuaUnit
:run(unpack(arg
))