lexer/lexer.cpp

   1 #include <fridh/lexer.hpp>
   2 #include <ail/array.hpp>
   3 #include <ail/string.hpp>
   4 #include <boost/foreach.hpp>
   5
   6 namespace fridh
   7 {
   8         boost::mutex table_mutex;
   9
  10         line_of_code::line_of_code():
  11                 indentation_level(0)
  12         {
  13         }
  14
  15         operator_lexeme::operator_lexeme(lexeme_type::type lexeme, std::string const & string):
  16                 lexeme(lexeme),
  17                 string(string)
  18         {
  19         }
  20
  21         bool operator_lexeme::operator<(operator_lexeme const & other) const
  22         {
  23                 return string.length() > other.string.length();
  24         }
  25
  26         lexer::lexer(std::string const & input, lines_of_code & lines):
  27                 input(input),
  28                 lines(lines)
  29         {
  30         }
  31
  32         bool lexer::parse_operator(line_of_code & output)
  33         {
  34                 std::size_t remaining_characters = end - i;
  35
  36                 BOOST_FOREACH(operator_lexeme & current_lexeme, operator_lexeme_data)
  37                 {
  38                         std::size_t operator_length = current_lexeme.string.size();
  39                         if(remaining_characters < operator_length)
  40                                 return false;
  41
  42                         std::string substring = input.substr(i, operator_length);
  43
  44                         if(substring == current_lexeme.string)
  45                         {
  46                                 output.lexemes.push_back(current_lexeme.lexeme);
  47                                 i += operator_length;
  48                                 return true;
  49                         }
  50                 }
  51                 return false;
  52         }
  53
  54         void lexer::lexer_error(std::string const & message, uword error_line)
  55         {
  56                 if(error_line == 0)
  57                         error_line = line;
  58                 throw ail::exception("Lexer error: Line " + ail::number_to_string<uword>(error_line) + ": " + message);
  59         }
  60
  61         void lexer::number_parsing_error(std::string const & message)
  62         {
  63                 lexer_error(message);
  64         }
  65
  66         bool lexer::is_name_char(char input)
  67         {
  68                 return ail::is_alpha(input) || ail::is_digit(input) || input == '_';
  69         }
  70
  71         void lexer::parse_name(line_of_code & output)
  72         {
  73                 std::size_t start = i;
  74                 for(i++; i < end && is_name_char(input[i]); i++);
  75                 std::string name = input.substr(start, i - start);
  76
  77                 lexeme current_lexeme;
  78                 if(name == "true")
  79                         current_lexeme = lexeme(true);
  80                 else if(name == "false")
  81                         current_lexeme = lexeme(false);
  82                 else
  83                 {
  84                         current_lexeme = lexeme(name);
  85                         current_lexeme.type = lexeme_type::name;
  86                 }
  87
  88                 output.lexemes.push_back(current_lexeme);
  89         }
  90
  91         bool lexer::string_match(std::string const & target)
  92         {
  93                 if(end - i < target.size())
  94                         return false;
  95
  96                 return input.substr(i, target.size()) == target;
  97         }
  98
  99         void lexer::process_newline()
 100         {
 101                 if(!current_line.lexemes.empty())
 102                 {
 103                         current_line.line = line;
 104                         lines.push_back(current_line);
 105                 }
 106                 std::string line_string = input.substr(line_offset, i - line_offset);
 107                 current_line = line_of_code();
 108                 line++;
 109                 i++;
 110                 line_offset = i;
 111         }
 112
 113         void lexer::parse_lexemes()
 114         {
 115                 initialise_tables();
 116
 117                 line = 1;
 118
 119                 line_offset = 0;
 120
 121                 for(i = 0, end = input.size(); i < end;)
 122                 {
 123                         if(parse_operator(current_line))
 124                                 continue;
 125
 126                         char const tab = '\t';
 127
 128                         char byte = input[i];
 129
 130                         switch(byte)
 131                         {
 132                                 case tab:
 133                                         if(current_line.indentation_level > 0)
 134                                                 lexer_error("Tabs are only permitted in the beginning of a line (offset " + ail::number_to_string(i - line_offset + 1) + ")");
 135                                         for(i++, current_line.indentation_level = 1; i < end && input[i] == tab; i++, current_line.indentation_level++);
 136                                         continue;
 137
 138                                 case ' ':
 139                                 case '\r':
 140                                         i++;
 141                                         continue;
 142
 143                                 case '\n':
 144                                 {
 145                                         process_newline();
 146                                         continue;
 147                                 }
 148
 149                                 case '\'':
 150                                 case '"':
 151                                 {
 152                                         std::string string;
 153                                         parse_string(current_line);
 154                                         continue;
 155                                 }
 156
 157                                 case ';':
 158                                         parse_comment();
 159                                         continue;
 160                         }
 161
 162                         if(parse_number(current_line))
 163                                 continue;
 164
 165                         parse_name(current_line);
 166                 }
 167
 168                 if(!current_line.lexemes.empty())
 169                 {
 170                         current_line.line = line;
 171                         lines.push_back(current_line);
 172                 }
 173         }
 174
 175         std::string visualise_lexemes(lines_of_code & lines)
 176         {
 177                 std::string output;
 178
 179                 BOOST_FOREACH(line_of_code & current_line, lines)
 180                 {
 181                         std::string number_string = ail::number_to_string(current_line.line);
 182                         for(word i = 0, end = 5 - number_string.size(); i < end; i++)
 183                                 output += " ";
 184                         output += number_string;
 185                         output += ": ";
 186                         for(uword indentation = 0; indentation < current_line.indentation_level; indentation++)
 187                                 output += "    ";
 188                         bool first = true;
 189                         BOOST_FOREACH(lexeme & current_lexeme, current_line.lexemes)
 190                         {
 191                                 if(first)
 192                                         first = false;
 193                                 else
 194                                         output += " ";
 195                                 output += "[" + current_lexeme.to_string() + "]";
 196                         }
 197                         output += "\n";
 198                 }
 199
 200                 return output;
 201         }
 202
 203         bool lexer::parse(std::string & error)
 204         {
 205                 try
 206                 {
 207                         parse_lexemes();
 208                         return true;
 209                 }
 210                 catch(ail::exception & exception)
 211                 {
 212                         error = exception.get_message();
 213                         return false;
 214                 }
 215         }
 216 }