1 /* -*- Mode: C; indent-tabs-mode: t; c-basic-offset: 4; tab-width: 4 -*- */
3 * anjuta-token-stream.c
4 * Copyright (C) Sébastien Granjoux 2009 <seb.sfo@free.fr>
6 * This program is free software: you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License as published by the
8 * Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
14 * See the GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License along
17 * with this program. If not, see <http://www.gnu.org/licenses/>.
20 #include "anjuta-token-stream.h"
22 #include "anjuta-debug.h"
24 #include <glib-object.h>
30 * SECTION:anjuta-token-stream
31 * @title: Anjuta token stream
32 * @short_description: Anjuta token stream
34 * @stability: Unstable
35 * @include: libanjuta/anjuta-token-stream.h
37 * A #AnjutaTokenStream object reads and writes a list of tokens. It uses two
38 * list. The first list is assigned when the object is created. Each token is
39 * read as characters discarding the separation between tokens. The second list
40 * is written using the data of the first list, so no new memory is allocated,
41 * in order to recreate a new list of tokens.
43 * This is used when the lexer needs several passes. At the beginning the file
44 * is read as a single token containing the whole file content. The first pass
45 * split this content into tokens. Additional passes are done on some parts of
46 * the token list to get a more precise splitting.
48 * It is important to not allocate new memory and keep the same character
49 * pointers in the additional passes because the token list does not own the
50 * memory. The address of each character is used to find the position of the
51 * changed data in the file.
53 * Several objects can be linked together to create a stack. It is used for
54 * included file or variable expansion.
58 *---------------------------------------------------------------------------*/
60 struct _AnjutaTokenStream
66 /* Read position in input stream */
70 /* Write position in input stream */
78 AnjutaTokenStream
*parent
;
80 /* Current directory */
81 GFile
*current_directory
;
91 *---------------------------------------------------------------------------*/
94 *---------------------------------------------------------------------------*/
97 *---------------------------------------------------------------------------*/
100 * anjuta_token_stream_append_token:
101 * @stream: a #AnjutaTokenStream object.
102 * @token: a #AnjutaToken object.
104 * Append an already existing token in the output stream.
107 anjuta_token_stream_append_token (AnjutaTokenStream
*stream
, AnjutaToken
*token
)
109 anjuta_token_append_child (stream
->root
, token
);
113 * anjuta_token_stream_tokenize:
114 * @stream: a #AnjutaTokenStream object.
115 * @type: a token type.
116 * @length: the token length in character.
118 * Create a token of type from the last length characters previously read and
119 * append it in the output stream. The characters are not copied in the output
120 * stream, the new token uses the same characters.
122 * Return value: The created token.
125 anjuta_token_stream_tokenize (AnjutaTokenStream
*stream
, gint type
, gsize length
)
130 frag
= anjuta_token_new_static (type
, NULL
);
132 for (end
= stream
->start
; end
!= NULL
;)
134 if ((anjuta_token_get_type (end
) < ANJUTA_TOKEN_PARSED
) || (anjuta_token_get_length (end
) == 0))
136 gint toklen
= anjuta_token_get_length (end
);
137 AnjutaToken
*copy
= anjuta_token_cut (end
, stream
->begin
, length
);
139 if (toklen
>= (length
+ stream
->begin
))
142 if (end
== stream
->start
)
144 /* Get whole token */
145 anjuta_token_free (frag
);
146 anjuta_token_set_type (copy
, type
);
151 /* Get several token */
152 anjuta_token_insert_after (frag
, copy
);
153 anjuta_token_merge (frag
, copy
);
156 if (toklen
== (length
+ stream
->begin
))
158 stream
->start
= anjuta_token_next (end
);
164 stream
->begin
+= length
;
170 anjuta_token_insert_after (frag
, copy
);
171 anjuta_token_merge (frag
, copy
);
172 length
-= toklen
- stream
->begin
;
173 end
= anjuta_token_next (end
);
179 end
= anjuta_token_next (end
);
184 anjuta_token_stream_append_token (stream
, frag
);
190 * anjuta_token_stream_read:
191 * @stream: a #AnjutaTokenStream object.
192 * @buffer: a character buffer to fill with token data.
193 * @max_size: the size of the buffer.
195 * Read token from the input stream and write the content as a C string in the
196 * buffer passed as argument.
198 * Return value: The number of characters written in the buffer.
201 anjuta_token_stream_read (AnjutaTokenStream
*stream
, gchar
*buffer
, gsize max_size
)
205 if (stream
->token
!= NULL
)
207 gsize length
= anjuta_token_get_length (stream
->token
);
209 if ((anjuta_token_get_type (stream
->token
) >= ANJUTA_TOKEN_PARSED
) || (stream
->pos
>= length
))
214 if (stream
->token
== stream
->last
) return 0;
216 if (anjuta_token_get_type (stream
->token
) >= ANJUTA_TOKEN_PARSED
)
218 stream
->token
= anjuta_token_next (stream
->token
);
222 stream
->token
= anjuta_token_next (stream
->token
);
225 if ((stream
->token
== NULL
) || (anjuta_token_get_type (stream
->token
) == ANJUTA_TOKEN_EOV
))
230 else if ((anjuta_token_get_length (stream
->token
) != 0) && (anjuta_token_get_type (stream
->token
) < ANJUTA_TOKEN_PARSED
))
234 length
= anjuta_token_get_length (stream
->token
);
240 if (stream
->pos
< length
)
242 const gchar
*start
= anjuta_token_get_string (stream
->token
);
244 length
-= stream
->pos
;
246 if (length
> max_size
) length
= max_size
;
247 memcpy (buffer
, start
+ stream
->pos
, length
);
248 stream
->pos
+= length
;
257 * anjuta_token_stream_get_root:
258 * @stream: a #AnjutaTokenStream object.
260 * Return the root token for the output stream.
262 * Return value: (transfer none): The output root token.
265 anjuta_token_stream_get_root (AnjutaTokenStream
*stream
)
267 g_return_val_if_fail (stream
!= NULL
, NULL
);
273 * anjuta_token_stream_get_current_directory:
274 * @stream: a #AnjutaTokenStream object.
276 * Return the current directory.
278 * Return value: (transfer none): The current directory.
281 anjuta_token_stream_get_current_directory (AnjutaTokenStream
*stream
)
283 g_return_val_if_fail (stream
!= NULL
, NULL
);
285 return stream
->current_directory
;
290 * anjuta_token_stream_get_current_file:
291 * @stream: a #AnjutaTokenStream object.
293 * Return the current file.
295 * Return value: (transfer none): The current file.
298 anjuta_token_stream_get_current_file (AnjutaTokenStream
*stream
)
300 g_return_val_if_fail (stream
!= NULL
, NULL
);
302 return stream
->current_file
;
305 /* Constructor & Destructor
306 *---------------------------------------------------------------------------*/
309 * anjuta_token_stream_push:
310 * @parent: (allow-none): a parent #AnjutaTokenStream object or %NULL.
311 * @root: (allow-none): a token or %NULL
312 * @content: a token list.
313 * @file: (allow-none): a #GFile of the file.
315 * Create a new stream from a list of tokens. If a parent stream is passed,
316 * the new stream keep a link on it, so we can return it when the new stream
319 * Return value: The newly created stream.
322 anjuta_token_stream_push (AnjutaTokenStream
*parent
, AnjutaToken
*root
, AnjutaToken
*content
, GFile
*file
)
324 AnjutaTokenStream
*child
;
325 AnjutaTokenStream
*stream
;
327 /* Check if content is not already parsed to avoid endless parsing loop */
328 for (stream
= parent
; stream
!= NULL
; stream
= stream
->parent
)
330 if (stream
->content
== content
) return NULL
;
333 /* Create new stream */
334 child
= g_new (AnjutaTokenStream
, 1);
335 child
->first
= content
;
338 child
->parent
= parent
;
339 child
->content
= content
;
340 child
->token
= content
;
341 child
->start
= child
->token
;
342 child
->last
= content
== NULL
? NULL
: anjuta_token_last (content
);
344 child
->root
= root
== NULL
? anjuta_token_new_static (ANJUTA_TOKEN_FILE
, NULL
) : root
;
347 child
->current_directory
= parent
== NULL
? NULL
: (parent
->current_directory
== NULL
? NULL
: g_object_ref (parent
->current_directory
));
348 child
->current_file
= NULL
;
352 child
->current_directory
= g_file_get_parent (file
);
353 child
->current_file
= g_object_ref (file
);
360 * anjuta_token_stream_pop:
361 * @stream: a #AnjutaTokenStream object.
363 * Destroy the stream object and return the parent stream if it exists.
365 * Return value: (transfer none) (allow-none): The parent stream or %NULL if there is no parent.
368 anjuta_token_stream_pop (AnjutaTokenStream
*stream
)
370 AnjutaTokenStream
*parent
;
372 g_return_val_if_fail (stream
!= NULL
, NULL
);
374 if (stream
->current_directory
) g_object_unref (stream
->current_directory
);
375 if (stream
->current_file
) g_object_unref (stream
->current_file
);
376 parent
= stream
->parent
;
383 * anjuta_token_stream_get_parent:
384 * @stream: a #AnjutaTokenStream object.
386 * Return the parent stream
388 * Return value: (transfer none) (allow-none): The parent stream or %NULL if
389 * there is no parent.
392 anjuta_token_stream_get_parent (AnjutaTokenStream
*stream
)
394 g_return_val_if_fail (stream
!= NULL
, NULL
);
396 return stream
->parent
;