3 #include "lyxlex_pimpl.h"
6 #include "support/lyxalgo.h"
7 #include "support/filetools.h"
8 #include "support/lstrings.h"
12 using namespace lyx::support;
19 using std::lower_bound;
25 // used by lower_bound, sort and sorted
27 int operator()(keyword_item const & a, keyword_item const & b) const {
28 // we use the ascii version, because in turkish, 'i'
29 // is not the lowercase version of 'I', and thus
30 // turkish locale breaks parsing of tags.
31 return compare_ascii_no_case(a.tag, b.tag) < 0;
34 // } // end of anon namespace
37 LyXLex::Pimpl::Pimpl(keyword_item * tab, int num)
38 : is(&fb__), table(tab), no_items(num),
39 status(0), lineno(0), commentChar('#')
45 string const LyXLex::Pimpl::getString() const
47 return string(buff.begin(), buff.end());
51 void LyXLex::Pimpl::printError(string const & message) const
53 string const tmpmsg = subst(message, "$$Token", getString());
54 lyxerr << "LyX: " << tmpmsg << " [around line " << lineno
55 << " of file " << MakeDisplayPath(name) << ']' << endl;
59 void LyXLex::Pimpl::printTable(ostream & os)
61 os << "\nNumber of tags: " << no_items << '\n';
62 for (int i= 0; i < no_items; ++i)
64 << "]: tag: `" << table[i].tag
65 << "' code:" << table[i].code << '\n';
70 void LyXLex::Pimpl::verifyTable()
72 // Check if the table is sorted and if not, sort it.
74 && !lyx::sorted(table, table + no_items, compare_tags())) {
75 lyxerr << "The table passed to LyXLex is not sorted!\n"
76 << "Tell the developers to fix it!" << endl;
77 // We sort it anyway to avoid problems.
78 lyxerr << "\nUnsorted:\n";
81 sort(table, table + no_items, compare_tags());
82 lyxerr << "\nSorted:\n";
88 void LyXLex::Pimpl::pushTable(keyword_item * tab, int num)
90 pushed_table tmppu(table, no_items);
100 void LyXLex::Pimpl::popTable()
102 if (pushed.empty()) {
103 lyxerr << "LyXLex error: nothing to pop!" << endl;
107 pushed_table tmp = pushed.top();
109 table = tmp.table_elem;
110 no_items = tmp.table_siz;
114 bool LyXLex::Pimpl::setFile(string const & filename)
117 // Check the format of the file.
118 string const format = getExtFromContents(filename);
120 if (format == "gzip" || format == "zip" || format == "compress") {
121 lyxerr << "lyxlex: compressed" << endl;
123 // The check only outputs a debug message, because it triggers
124 // a bug in compaq cxx 6.2, where is_open() returns 'true' for
125 // a fresh new filebuf. (JMarc)
126 if (gz__.is_open() || is.tellg() > 0)
127 lyxerr[Debug::LYXLEX] << "Error in LyXLex::setFile: "
128 "file or stream already set." << endl;
129 gz__.open(filename.c_str(), ios::in);
133 return gz__.is_open() && is.good();
135 lyxerr << "lyxlex: UNcompressed" << endl;
137 // The check only outputs a debug message, because it triggers
138 // a bug in compaq cxx 6.2, where is_open() returns 'true' for
139 // a fresh new filebuf. (JMarc)
140 if (fb__.is_open() || is.tellg() > 0)
141 lyxerr[Debug::LYXLEX] << "Error in LyXLex::setFile: "
142 "file or stream already set." << endl;
143 fb__.open(filename.c_str(), ios::in);
147 return fb__.is_open() && is.good();
152 void LyXLex::Pimpl::setStream(istream & i)
154 if (fb__.is_open() || is.tellg() > 0)
155 lyxerr[Debug::LYXLEX] << "Error in LyXLex::setStream: "
156 "file or stream already set." << endl;
162 void LyXLex::Pimpl::setCommentChar(char c)
168 bool LyXLex::Pimpl::next(bool esc /* = false */)
170 if (!pushTok.empty()) {
171 // There can have been a whole line pushed so
172 // we extract the first word and leaves the rest
174 if (pushTok.find(' ') != string::npos && pushTok[0] == '\\') {
176 pushTok = split(pushTok, tmp, ' ');
177 buff.assign(tmp.begin(), tmp.end());
180 buff.assign(pushTok.begin(), pushTok.end());
186 unsigned char c = 0; // getc() returns an int
189 while (is && !status) {
192 if (c == commentChar) {
193 // Read rest of line (fast :-)
195 // That is not fast... (Lgb)
199 lyxerr[Debug::LYXLEX] << "Comment read: `" << c
200 << dummy << '\'' << endl;
202 // unfortunately ignore is buggy (Lgb)
203 is.ignore(100, '\n');
217 } while (c != '\"' && c != '\n' && is);
220 printError("Missing quote");
231 continue; /* Skip ','s */
233 // using relational operators with chars other
234 // than == and != is not safe. And if it is done
235 // the type _have_ to be unsigned. It usually a
236 // lot better to use the functions from cctype
244 } while (c > ' ' && c != ',' && is);
249 if (c == '\r' && is) {
250 // The Windows support has lead to the
251 // possibility of "\r\n" at the end of
252 // a line. This will stop LyX choking
253 // when it expected to find a '\n'
262 if (status) return true;
264 status = is.eof() ? LEX_FEOF: LEX_UNDEF;
268 unsigned char c = 0; // getc() returns an int
272 while (is && !status) {
277 if (c == ',') continue;
285 // escape the next char
292 } while (c > ' ' && c != ',' && is);
298 if (c == commentChar) {
299 // Read rest of line (fast :-)
301 // That is still not fast... (Lgb)
305 lyxerr[Debug::LYXLEX] << "Comment read: `" << c
306 << dummy << '\'' << endl;
308 // but ignore is also still buggy (Lgb)
309 // This is fast (Lgb)
310 is.ignore(100, '\n');
320 bool escaped = false;
325 if (c == '\r') continue;
327 // escape the next char
330 if (c == '\"' || c == '\\')
333 buff.push_back('\\');
337 if (!escaped && c == '\"') break;
338 } while (c != '\n' && is);
341 printError("Missing quote");
356 // escape the next char
364 } while (c > ' ' && c != ',' && is);
373 if (status) return true;
375 status = is.eof() ? LEX_FEOF : LEX_UNDEF;
382 int LyXLex::Pimpl::search_kw(char const * const tag) const
384 keyword_item search_tag = { tag, 0 };
386 lower_bound(table, table + no_items,
387 search_tag, compare_tags());
388 // use the compare_ascii_no_case instead of compare_no_case,
389 // because in turkish, 'i' is not the lowercase version of 'I',
390 // and thus turkish locale breaks parsing of tags.
391 if (res != table + no_items
392 && !compare_ascii_no_case(res->tag, tag))
398 int LyXLex::Pimpl::lex()
400 //NOTE: possible bug.
401 if (next() && status == LEX_TOKEN) {
402 return search_kw(getString().c_str());
408 bool LyXLex::Pimpl::eatLine()
412 unsigned char c = '\0';
414 while (is && c != '\n') {
417 //lyxerr[Debug::LYXLEX] << "LyXLex::EatLine read char: `"
418 // << c << '\'' << endl;
434 bool LyXLex::Pimpl::nextToken()
436 if (!pushTok.empty()) {
437 // There can have been a whole line pushed so
438 // we extract the first word and leaves the rest
440 if (pushTok.find(' ') != string::npos && pushTok[0] == '\\') {
442 pushTok = split(pushTok, tmp, ' ');
443 buff.assign(tmp.begin(), tmp.end());
446 buff.assign(pushTok.begin(), pushTok.end());
453 while (is && !status) {
458 if (c >= ' ' && is) {
461 if (c == '\\') { // first char == '\\'
466 } while (c > ' ' && c != '\\' && is);
472 } while (c >= ' ' && c != '\\' && is);
475 if (c == '\\') is.putback(c); // put it back
483 if (status) return true;
485 status = is.eof() ? LEX_FEOF: LEX_UNDEF;
491 void LyXLex::Pimpl::pushToken(string const & pt)