libanjuta/anjuta-token.c

   1 /* -*- Mode: C; indent-tabs-mode: t; c-basic-offset: 4; tab-width: 4 -*- */
   2 /*
   3  * anjuta-token.c
   4  * Copyright (C) Sébastien Granjoux 2009 <seb.sfo@free.fr>
   5  *
   6  * This program is free software: you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License as published by the
   8  * Free Software Foundation, either version 3 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful, but
  12  * WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  14  * See the GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License along
  17  * with this program.  If not, see <http://www.gnu.org/licenses/>.
  18  */
  19
  20 #include "anjuta-token.h"
  21
  22 #include "anjuta-debug.h"
  23
  24 #include <glib-object.h>
  25
  26 #include <stdio.h>
  27 #include <string.h>
  28 #include <stdlib.h>
  29
  30 /**
  31  * SECTION:anjuta-token
  32  * @title: Anjuta token
  33  * @short_description: Anjuta token
  34  * @see_also:
  35  * @stability: Unstable
  36  * @include: libanjuta/anjuta-token.h
  37  *
  38  * A #AnjutaToken represents a token. It is a sequence of characters associated
  39  * with a type representing its meaning. By example, a token can represent
  40  * a keyword, a comment, a variable...
  41  *
  42  * The token can own the string or has only a pointer on some data allocated
  43  * somewhere else with a length.
  44  *
  45  * A token is linked with other tokens using three double linked lists.
  46  *
  47  * The first list using next and prev fields is used to keep the token in the
  48  * order where there are in the file. The first character of the first token is
  49  * the first character in the file.
  50  *
  51  * A second list is used to represent included
  52  * files. Such file is represented by a special token in the first list which
  53  * has a pointer, named children to a token list. Each token in this secondary
  54  * list has a pointer to its parent, it means the token representing the file
  55  * where is the token. It looks like a tree. In fact, every file is represented
  56  * by a special token, so the root node is normally a file token and has as
  57  * children all the token representing the file content. This parent/child list
  58  * is used for expanded variable too.
  59  *
  60  * A third list is used to group several tokens. A token can have a pointer to
  61  * another last token. It means that this token is a group starting from this
  62  * token to the one indicated by the last field. In addition each token in this
  63  * group has a pointer on the first token of the group. This grouping is
  64  * independent of the parent/child list. So a group can start in one file and
  65  * end in another included file. The grouping can be nested too. Typically
  66  * we can have a group representing a command, a sub group representing the
  67  * arguments and then one sub group for each argument.
  68  */
  69
  70 /*
  71  * A token with its pointer can be put in one of the following case:
  72  *
  73  * * Simple token:
  74  *              children = NULL && last = NULL
  75  *              This is the case of a simple token without any children.
  76  *
  77  * * Composite token:
  78  *              children = NULL && last != NULL
  79  *              This is the case of a token grouping several other tokens, representing
  80  *              by example an item in a list.
  81  *
  82  * * Parent token:
  83  *              children != NULL && last == NULL
  84  *              This is the case of a token having children, by example a variable where
  85  *              the children represent the content. Most of the time the value of the
  86  *              parent token is ignored.
  87  *
  88  * * Composite parent token:
  89  *              children != NULL && last != NULL && (last in list)
  90  *              This case represents a variable which is split into several tokens. The
  91  *              children represents the content of the variable. The token up to last
  92  *              corresponds to the variable name. After getting the variable content, the
  93  *              next token is after the last token of the variable.
  94  *
  95  * * Composite parent token:
  96  *              children != NULL && last != NULL && (last in children)
  97  *              This case would represent a variable which is split into several token
  98  *              with one inside the children of the variable. This is not possible as
  99  *              the complete name of the variable is needed to get the children.
 100  *
 101  * * Composite parent token:
 102  *              children != NULL && last != NULL && (last in parent)
 103  *              This case represents a variable split into several token where the
 104  *              last element is a sibling of one parent of the token. I think this case
 105  *              is possible.
 106  *
 107  */
 108
 109 typedef struct _AnjutaTokenData AnjutaTokenData;
 110
 111 struct _AnjutaTokenData
 112 {
 113         AnjutaTokenType type;
 114         gint flags;
 115         gchar *pos;
 116         gsize length;
 117 };
 118
 119 struct _AnjutaToken
 120 {
 121         AnjutaToken     *next;
 122         AnjutaToken     *prev;
 123         AnjutaToken     *parent;
 124         AnjutaToken *last;
 125         AnjutaToken *group;
 126         AnjutaToken     *children;
 127         AnjutaTokenData data;
 128 };
 129
 130 /* Helpers functions
 131  *---------------------------------------------------------------------------*/
 132
 133 /* Private functions
 134  *---------------------------------------------------------------------------*/
 135
 136 static AnjutaToken *
 137 anjuta_token_next_child (AnjutaToken *child, AnjutaToken **last)
 138 {
 139         if (child == NULL) return child;
 140
 141         if (child->children != NULL)
 142         {
 143                 child = child->children;
 144         }
 145         else
 146         {
 147                 for (;;)
 148                 {
 149                         if ((*last == NULL) || (child == *last))
 150                         {
 151                                 if (child->last == NULL)
 152                                 {
 153                                         child = NULL;
 154                                         break;
 155                                 }
 156                                 *last = child->last;
 157                         }
 158                         if (child->next != NULL)
 159                         {
 160                                 child = child->next;
 161                                 break;
 162                         }
 163                         child = child->parent;
 164                 }
 165         }
 166
 167         return child;
 168 }
 169
 170 AnjutaToken *
 171 anjuta_token_next_after_children (AnjutaToken *token)
 172 {
 173         while (token->next == NULL)
 174         {
 175                 token = token->parent;
 176                 if (token == NULL) return NULL;
 177         };
 178
 179         return token->next;
 180 }
 181
 182 static AnjutaToken *
 183 anjuta_token_copy (AnjutaToken *token)
 184 {
 185         AnjutaToken *copy = NULL;
 186
 187         if (token != NULL)
 188         {
 189                 copy = g_slice_new0 (AnjutaToken);
 190                 copy->data.type = token->data.type;
 191                 copy->data.flags = token->data.flags;
 192                 if ((copy->data.flags & ANJUTA_TOKEN_STATIC) || (token->data.pos == NULL))
 193                 {
 194                         copy->data.pos = token->data.pos;
 195                 }
 196                 else
 197                 {
 198                         copy->data.pos = g_strdup (token->data.pos);
 199                 }
 200                 copy->data.length = token->data.length;
 201         }
 202
 203         return copy;
 204 }
 205
 206 /**
 207  * anjuta_token_unlink_token:
 208  * @token: a #AnjutaToken object.
 209  *
 210  * Unlink a single token, not the complete item, from the token tree.
 211  *
 212  * Return value: the removed token tree
 213  */
 214 static AnjutaToken *
 215 anjuta_token_unlink_token (AnjutaToken *token)
 216 {
 217
 218         if (token->prev != NULL)
 219         {
 220                 token->prev->next = token->next;
 221         }
 222         else if ((token->parent != NULL) && (token->parent->children == token))
 223         {
 224                 token->parent->children = token->next;
 225         }
 226         token->parent = NULL;
 227
 228         if ((token->group != NULL) && (token->group->last == token))
 229         {
 230                 AnjutaToken *prev;
 231
 232                 for (prev = token->prev; prev != NULL; prev = prev->prev)
 233                 {
 234                         if (prev->group == token->group)
 235                         {
 236                                 /* Find previous token in the same group */
 237                                 token->group->last = prev;
 238                                 break;
 239                         }
 240                         else if (prev == token->group)
 241                         {
 242                                 /* No more token in group */
 243                                 token->group->last = NULL;
 244                                 break;
 245                         }
 246                 }
 247         }
 248
 249         if (token->next != NULL)
 250         {
 251                 token->next->prev = token->prev;
 252                 token->next = NULL;
 253         }
 254         token->prev = NULL;
 255
 256         return token;
 257 }
 258
 259 /**
 260  * anjuta_token_insert_token_before:
 261  * @sibling: a #AnjutaToken object.
 262  * @token: a #AnjutaToken object.
 263  *
 264  * Insert token before sibling.
 265  *
 266  * Return value: inserted token
 267  */
 268 static AnjutaToken *
 269 anjuta_token_insert_token_before (AnjutaToken *sibling, AnjutaToken *token)
 270 {
 271         token->prev = sibling->prev;
 272         token->next = sibling;
 273
 274         if (token->prev != NULL)
 275         {
 276                 token->prev->next = token;
 277         }
 278         sibling->prev = token;
 279
 280         if ((sibling->parent != NULL) && (sibling->parent->children == sibling))
 281         {
 282                 sibling->parent->children = token;
 283         }
 284         token->parent = sibling->parent;
 285
 286         return token;
 287 }
 288
 289 static void
 290 anjuta_token_evaluate_token (AnjutaToken *token, GString *value, gboolean raw)
 291 {
 292         if ((token != NULL) && (token->data.length != 0))
 293         {
 294                 if (!raw)
 295                 {
 296                         switch (anjuta_token_get_type (token))
 297                         {
 298                         case ANJUTA_TOKEN_COMMENT:
 299                         case ANJUTA_TOKEN_OPEN_QUOTE:
 300                         case ANJUTA_TOKEN_CLOSE_QUOTE:
 301                         case ANJUTA_TOKEN_ESCAPE:
 302                         case ANJUTA_TOKEN_MACRO:
 303                         case ANJUTA_TOKEN_EOV:
 304                                 return;
 305                         default:
 306                                 break;
 307                         }
 308                 }
 309                 g_string_append_len (value, anjuta_token_get_string (token), anjuta_token_get_length (token));
 310         }
 311 }
 312
 313 static void
 314 anjuta_token_show (AnjutaToken *token, gint indent, gchar parent)
 315 {
 316         static gchar type[] = "\0";
 317         const gchar *string;
 318         gsize length;
 319
 320         type[0] = parent;
 321         fprintf (stderr, "%*s%s %p", indent, "", type, token);
 322         fprintf (stderr, ": %d ",
 323             anjuta_token_get_type (token));
 324         string = anjuta_token_get_string (token);
 325         length = anjuta_token_get_length (token);
 326         if (string == NULL)
 327         {
 328                 /* Value doesn't contain a newline */
 329                 fprintf (stderr, "(%lu)", length);
 330         }
 331         else
 332         {
 333                 const gchar *newline;
 334
 335                 newline = g_strrstr_len (string, length, "\n");
 336                 if (newline == NULL)
 337                 {
 338                         /* Value doesn't contain a newline */
 339                         fprintf (stderr, "\"%.*s\"",
 340                                 length,
 341                         string);
 342                 }
 343                 else
 344                 {
 345                         /* Value contains a newline, take care of indentation */
 346                         newline++;
 347                         fprintf (stderr, "\"%.*s",
 348                         newline - string,
 349                         string);
 350                         for (;;)
 351                         {
 352                                 length -= newline - string;
 353                                 string = newline;
 354
 355                                 newline = g_strrstr_len (string, length, "\n");
 356                                 if (newline == NULL) break;
 357
 358                                 newline++;
 359                                 fprintf (stderr, "%*s  %.*s",
 360                                     indent, "",
 361                                         newline - string,
 362                                 string);
 363                         }
 364                         fprintf (stderr, "%*s  %.*s\"",
 365                                     indent, "",
 366                                         length,
 367                                 string);
 368                 }
 369         }
 370         fprintf (stderr, " %p/%p (%p/%p) %s\n",
 371                 token->last, token->children,
 372                 token->group, token->parent,
 373                 anjuta_token_get_flags (token) & ANJUTA_TOKEN_REMOVED ? " (removed)" : "");
 374 }
 375
 376 static AnjutaToken*
 377 anjuta_token_dump_child (AnjutaToken *token, gint indent, gchar type)
 378 {
 379         AnjutaToken *last;
 380         AnjutaToken *child;
 381
 382         anjuta_token_show (token, indent, type);
 383         indent += 4;
 384
 385         last = token;
 386         if (token->last != NULL)
 387         {
 388                 do
 389                 {
 390                         child = last->next;
 391                         if (child == NULL) break;
 392                         last = anjuta_token_dump_child (child, indent, '+');
 393                 }
 394                 while (child != token->last);
 395         }
 396
 397         if (token->children != NULL)
 398         {
 399                 for (child = token->children; child != NULL; child = child->next)
 400                 {
 401                         child = anjuta_token_dump_child (child, indent, '*');
 402                 }
 403         }
 404
 405         return last;
 406 }
 407
 408 static gboolean
 409 anjuta_token_check_child (AnjutaToken *token, AnjutaToken *parent)
 410 {
 411         if (token->parent != parent)
 412         {
 413                 anjuta_token_show (token, 0, 0);
 414                 fprintf(stderr, "Error: Children has %p as parent instead of %p\n", token->parent, parent);
 415                 return FALSE;
 416         }
 417
 418         return anjuta_token_check (token);
 419 }
 420
 421 /* Get and set functions
 422  *---------------------------------------------------------------------------*/
 423
 424 void
 425 anjuta_token_set_type (AnjutaToken *token, gint type)
 426 {
 427         token->data.type = type;
 428 }
 429
 430 gint
 431 anjuta_token_get_type (AnjutaToken *token)
 432 {
 433         return token->data.type;
 434 }
 435
 436 void
 437 anjuta_token_set_flags (AnjutaToken *token, gint flags)
 438 {
 439         AnjutaToken *child;
 440         AnjutaToken *last = token->last;
 441
 442         for (child = token; child != NULL; child = anjuta_token_next_child (child, &last))
 443         {
 444                 child->data.flags |= flags;
 445         }
 446 }
 447
 448 void
 449 anjuta_token_clear_flags (AnjutaToken *token, gint flags)
 450 {
 451         token->data.flags &= ~flags;
 452 }
 453
 454 gint
 455 anjuta_token_get_flags (AnjutaToken *token)
 456 {
 457         return token->data.flags;
 458 }
 459
 460 void
 461 anjuta_token_set_string (AnjutaToken *token, const gchar *data, gsize length)
 462 {
 463         if (!(token->data.flags & ANJUTA_TOKEN_STATIC))
 464         {
 465                 g_free (token->data.pos);
 466                 token->data.flags |= ANJUTA_TOKEN_STATIC;
 467         }
 468         token->data.pos = (gchar *)data;
 469         token->data.length = length;
 470 }
 471
 472 const gchar *
 473 anjuta_token_get_string (AnjutaToken *token)
 474 {
 475         return token->data.pos;
 476 }
 477
 478 void
 479 anjuta_token_set_length (AnjutaToken *token, gsize length)
 480 {
 481         token->data.length = length;
 482 }
 483
 484 gsize
 485 anjuta_token_get_length (AnjutaToken *token)
 486 {
 487         return token->data.length;
 488 }
 489
 490 /* Basic move functions
 491  *---------------------------------------------------------------------------*/
 492
 493 AnjutaToken *
 494 anjuta_token_next (AnjutaToken *token)
 495 {
 496         if (token->children != NULL)
 497         {
 498                 return token->children;
 499         }
 500         else if (token->next != NULL)
 501         {
 502                 return token->next;
 503         }
 504         else if (token->parent != NULL)
 505         {
 506                 return anjuta_token_next_after_children (token->parent);
 507         }
 508         else
 509         {
 510                 return NULL;
 511         }
 512 }
 513
 514 AnjutaToken *
 515 anjuta_token_previous (AnjutaToken *token)
 516 {
 517         if (token->prev != NULL)
 518         {
 519                 return token->prev;
 520         }
 521         else
 522         {
 523                 return token->parent;
 524         }
 525 }
 526
 527 AnjutaToken *
 528 anjuta_token_last (AnjutaToken *token)
 529 {
 530         AnjutaToken *last;
 531
 532         for (last = token; last->last != NULL; last = last->last);
 533         if (last->children != NULL)
 534         {
 535                 for (last = last->children; last->next != NULL; last = last->next);
 536         }
 537
 538         return last;
 539 }
 540
 541 AnjutaToken *
 542 anjuta_token_parent (AnjutaToken *token)
 543 {
 544         return token->parent;
 545 }
 546
 547 AnjutaToken *
 548 anjuta_token_list (AnjutaToken *token)
 549 {
 550         return token->group;
 551 }
 552
 553 /* Item move functions
 554  *---------------------------------------------------------------------------*/
 555
 556 AnjutaToken *
 557 anjuta_token_last_item (AnjutaToken *list)
 558 {
 559         return list->last;
 560 }
 561
 562 AnjutaToken *
 563 anjuta_token_first_item (AnjutaToken *list)
 564 {
 565         AnjutaToken *first = NULL;
 566
 567         if (list != NULL)
 568         {
 569                 if (list->children != NULL)
 570                 {
 571                         first = list->children;
 572                 }
 573                 else if (list->last != NULL)
 574                 {
 575                         first = list->next;
 576                 }
 577         }
 578
 579         return first;
 580 }
 581
 582 AnjutaToken *
 583 anjuta_token_next_item (AnjutaToken *item)
 584 {
 585         AnjutaToken *next;
 586
 587         if (item != NULL)
 588         {
 589                 do
 590                 {
 591                         next = NULL;
 592                         if ((item->group == NULL) || (item->group->last != item))
 593                         {
 594                                 AnjutaToken *last;
 595                                 for (last = item; last->last != NULL; last = last->last);
 596                                 next = anjuta_token_next (last);
 597                                 if ((next != NULL) && (next->group != item->group)) next = NULL;
 598                         }
 599                         item = next;
 600                 }
 601                 /* Loop if the current item has been deleted */
 602                 while ((next != NULL) && (anjuta_token_get_flags (next) & ANJUTA_TOKEN_REMOVED));
 603         }
 604
 605         return next;
 606 }
 607
 608 AnjutaToken *
 609 anjuta_token_previous_item (AnjutaToken *item)
 610 {
 611         AnjutaToken *prev = NULL;
 612
 613
 614         if (item != NULL)
 615         {
 616                 do
 617                 {
 618                         for (prev = item->prev; (prev != NULL) && (prev->group != item->group); prev = prev->group);
 619                         item = prev;
 620                 }
 621                 /* Loop if the current item has been deleted */
 622                 while ((prev != NULL) && (anjuta_token_get_flags (prev) & ANJUTA_TOKEN_REMOVED));
 623         }
 624
 625         return prev;
 626 }
 627
 628 /* Add/Insert/Remove tokens
 629  *---------------------------------------------------------------------------*/
 630
 631 /**
 632  * anjuta_token_append_child:
 633  * @parent: a #AnjutaToken object used as parent.
 634  * @children: a #AnjutaToken object.
 635  *
 636  * Insert all tokens in children as the last children of the given parent.
 637  *
 638  * Return value: The first token append.
 639  */
 640 AnjutaToken *
 641 anjuta_token_append_child (AnjutaToken *parent, AnjutaToken *children)
 642 {
 643         AnjutaToken *token;
 644         AnjutaToken *last;
 645         AnjutaToken *old_group;
 646         AnjutaToken *old_parent;
 647
 648         g_return_val_if_fail (parent != NULL, NULL);
 649         g_return_val_if_fail (children != NULL, NULL);
 650
 651         old_group = children->group;
 652         old_parent = children->parent;
 653
 654         if (parent->children == NULL)
 655         {
 656                 parent->children = children;
 657
 658                 children->prev = NULL;
 659         }
 660         else
 661         {
 662                 /* Find last children */
 663                 for (last = parent->children; last->next != NULL;)
 664                 {
 665                         if ((last->last != NULL) && (last->last->parent == last->parent))
 666                         {
 667                                 last = last->last;
 668                         }
 669                         else
 670                         {
 671                                 last = last->next;
 672                         }
 673                 }
 674
 675                 last->next = children;
 676                 children->prev = last;
 677         }
 678
 679         /* Update each token */
 680         for (token = children;;)
 681         {
 682                 if (token->parent == old_parent) token->parent = parent;
 683                 if (token->group == old_group) token->group = parent->group;
 684
 685                 if (token->children != NULL)
 686                 {
 687                         token = token->children;
 688                 }
 689                 else if (token->next != NULL)
 690                 {
 691                         token = token->next;
 692                 }
 693                 else
 694                 {
 695                         while (token->parent != parent)
 696                         {
 697                                 token = token->parent;
 698                                 if (token->next != NULL) break;
 699                         }
 700                         if (token->next == NULL) break;
 701                         token = token->next;
 702                 }
 703         }
 704
 705         return children;
 706 }
 707
 708 /**
 709  * anjuta_token_prepend_child:
 710  * @parent: a #AnjutaToken object used as parent.
 711  * @children: a #AnjutaToken object.
 712  *
 713  * Insert all tokens in children as the first children of the given parent.
 714  *
 715  * Return value: The first token append.
 716  */
 717 AnjutaToken *
 718 anjuta_token_prepend_child (AnjutaToken *parent, AnjutaToken *children)
 719 {
 720         AnjutaToken *child;
 721         AnjutaToken *last = NULL;
 722
 723         g_return_val_if_fail (parent != NULL, NULL);
 724         g_return_val_if_fail (children != NULL, NULL);
 725
 726         /* Update each token */
 727         for (child = children;;)
 728         {
 729                 AnjutaToken *next;
 730
 731                 if (child->parent == children->parent) child->parent = parent;
 732                 if (child->group == children->group) child->group = parent->group;
 733
 734                 next = anjuta_token_next_child (child, &last);
 735                 if (next == NULL) break;
 736                 child = next;
 737         }
 738
 739         child->next = parent->children;
 740         if (child->next) child->next->prev = child;
 741         parent->children = children;
 742
 743         return children;
 744 }
 745
 746 /**
 747  * anjuta_token_prepend_items:
 748  * @list: a #AnjutaToken object used as list.
 749  * @item: a #AnjutaToken object.
 750  *
 751  * Insert all tokens in item as item of the given list.
 752  *
 753  * Return value: The first token append.
 754  */
 755 AnjutaToken *
 756 anjuta_token_prepend_items (AnjutaToken *list, AnjutaToken *item)
 757 {
 758         AnjutaToken *token;
 759         AnjutaToken *old_group;
 760         AnjutaToken *old_parent;
 761
 762         g_return_val_if_fail (list != NULL, NULL);
 763         g_return_val_if_fail (item != NULL, NULL);
 764
 765         old_group = item->group;
 766         old_parent = item->parent;
 767
 768         /* Update each token */
 769         for (token = item;;)
 770         {
 771                 if (token->parent == old_parent) token->parent = list->parent;
 772                 if (token->group == old_group) token->group = list;
 773
 774                 if (token->children != NULL)
 775                 {
 776                         token = token->children;
 777                 }
 778                 else if (token->next != NULL)
 779                 {
 780                         token = token->next;
 781                 }
 782                 else
 783                 {
 784                         while (token->parent != list->parent)
 785                         {
 786                                 token = token->parent;
 787                                 if (token->next != NULL) break;
 788                         }
 789                         if (token->next == NULL) break;
 790                         token = token->next;
 791                 }
 792         }
 793
 794         token->next = list->next;
 795         if (token->next) token->next->prev = token;
 796
 797         list->next = item;
 798         item->prev = list;
 799
 800         if (list->last == NULL)
 801         {
 802                 while (token->group != list) token = token->group;
 803                 list->last = token;
 804         }
 805
 806         return item;
 807 }
 808
 809 /**
 810  * anjuta_token_insert_after:
 811  * @sibling: a #AnjutaToken object.
 812  * @item: a #AnjutaToken object.
 813  *
 814  * Insert all tokens after sibling.
 815  *
 816  * Return value: The first token inserted.
 817  */
 818 AnjutaToken *
 819 anjuta_token_insert_after (AnjutaToken *sibling, AnjutaToken *list)
 820 {
 821         AnjutaToken *last;
 822         AnjutaToken *token;
 823         AnjutaToken *old_group;
 824         AnjutaToken *old_parent;
 825
 826         g_return_val_if_fail (sibling != NULL, NULL);
 827         g_return_val_if_fail (list != NULL, NULL);
 828
 829         old_group = list->group;
 830         old_parent = list->parent;
 831
 832         /* Update each token */
 833         for (token = list;;)
 834         {
 835                 if (token->parent == old_parent) token->parent = sibling->parent;
 836                 if (token->group == old_group) token->group = sibling->group;
 837
 838                 if (token->children != NULL)
 839                 {
 840                         token = token->children;
 841                 }
 842                 else if (token->next != NULL)
 843                 {
 844                         token = token->next;
 845                 }
 846                 else
 847                 {
 848                         while (token->parent != sibling->parent)
 849                         {
 850                                 token = token->parent;
 851                                 if (token->next != NULL) break;
 852                         }
 853                         if (token->next == NULL) break;
 854                         token = token->next;
 855                 }
 856         }
 857
 858         for (last = sibling; last->last != NULL; last = last->last);
 859
 860         token->next = last->next;
 861         if (token->next) token->next->prev = token;
 862
 863         last->next = list;
 864         list->prev = last;
 865
 866         if ((sibling->group != NULL) && (sibling->group->last == sibling))
 867         {
 868                 while (token->group != sibling->group) token = token->group;
 869                 sibling->group->last = token;
 870         }
 871
 872         return list;
 873 }
 874
 875 /**
 876  * anjuta_token_insert_before:
 877  * @sibling: a #AnjutaToken object.
 878  * @item: a #AnjutaToken object.
 879  *
 880  * Insert all tokens before sibling.
 881  *
 882  * Return value: The first token inserted.
 883  */
 884 AnjutaToken *
 885 anjuta_token_insert_before (AnjutaToken *sibling, AnjutaToken *list)
 886 {
 887         AnjutaToken *last;
 888         AnjutaToken *token;
 889         AnjutaToken *old_group;
 890         AnjutaToken *old_parent;
 891
 892         g_return_val_if_fail (sibling != NULL, NULL);
 893         g_return_val_if_fail (list != NULL, NULL);
 894
 895         old_group = list->group;
 896         old_parent = list->parent;
 897
 898         /* Update each token */
 899         for (token = list;;)
 900         {
 901                 if (token->parent == old_parent) token->parent = sibling->parent;
 902                 if (token->group == old_group) token->group = sibling->group;
 903
 904                 if (token->children != NULL)
 905                 {
 906                         token = token->children;
 907                 }
 908                 else if (token->next != NULL)
 909                 {
 910                         token = token->next;
 911                 }
 912                 else
 913                 {
 914                         while (token->parent != sibling->parent)
 915                         {
 916                                 token = token->parent;
 917                                 if (token->next != NULL) break;
 918                         }
 919                         if (token->next == NULL) break;
 920                         token = token->next;
 921                 }
 922         }
 923
 924         for (last = sibling; last->last != NULL; last = last->last);
 925
 926         token->next = sibling;
 927         list->prev = sibling->prev;
 928         sibling->prev = token;
 929
 930         if (list->prev) list->prev->next = list;
 931
 932         if ((list->parent != NULL) && (list->parent->children == sibling)) list->parent->children = list;
 933
 934         return list;
 935 }
 936
 937 /**
 938  * anjuta_token_delete_parent:
 939  * @parent: a #AnjutaToken object used as parent.
 940  *
 941  * Delete only the parent token.
 942  *
 943  * Return value: the first children
 944  */
 945 AnjutaToken *
 946 anjuta_token_delete_parent (AnjutaToken *parent)
 947 {
 948         AnjutaToken *token;
 949
 950         g_return_val_if_fail (parent != NULL, NULL);
 951
 952         if (parent->children == NULL) return NULL;
 953
 954         /* Update each token */
 955         for (token = parent->children;;)
 956         {
 957                 if (token->parent == parent) token->parent = parent->parent;
 958
 959                 if (token->children != NULL)
 960                 {
 961                         token = token->children;
 962                 }
 963                 else if (token->next != NULL)
 964                 {
 965                         token = token->next;
 966                 }
 967                 else
 968                 {
 969                         while (token->parent != parent->parent)
 970                         {
 971                                 token = token->parent;
 972                                 if (token->next != NULL) break;
 973                         }
 974                         if (token->next == NULL) break;
 975                         token = token->next;
 976                 }
 977         }
 978
 979         token->next = parent->next;
 980         if (token->next) token->next->prev = token;
 981
 982         parent->next = parent->children;
 983         parent->children->prev = parent;
 984         parent->children = NULL;
 985
 986         return anjuta_token_free (parent);
 987 }
 988
 989 /* Merge function
 990  *---------------------------------------------------------------------------*/
 991
 992 /* anjuta_token_merge can be used with first or end being a floating token and
 993  * on already grouped tokens to change the group organisation */
 994 AnjutaToken *
 995 anjuta_token_merge (AnjutaToken *first, AnjutaToken *end)
 996 {
 997         AnjutaToken *next;
 998
 999         if ((first == end) || (end == NULL)) return first;
1000
1001         /* Insert first or end in the same sequence if it is not already the case */
1002         for (next = first; next != end; next = anjuta_token_next (next))
1003         {
1004                 if (next == NULL)
1005                 {
1006                         if (first->parent == NULL)
1007                         {
1008                                 anjuta_token_insert_before (end, first);
1009                         }
1010                         else
1011                         {
1012                                 anjuta_token_insert_after (first, end);
1013                         }
1014                         break;
1015                 }
1016         }
1017         first->last = end;
1018         if ((end->group != NULL) && (end->group != first) && (end->group->last == end)) end->group->last = first;
1019         end->group = first;
1020
1021         return first;
1022 }
1023
1024 AnjutaToken *
1025 anjuta_token_merge_own_children (AnjutaToken *group)
1026 {
1027         AnjutaToken *token;
1028         AnjutaToken *next = NULL;
1029
1030         if (group->last != NULL) return group;
1031
1032         if (group->last->last != NULL) group->last = group->last->last;
1033
1034         for (token = anjuta_token_next (group); (token != NULL) && (token != group->last); token = anjuta_token_next (token))
1035         {
1036                 if (next == NULL)
1037                 {
1038                         if (token->last != NULL)
1039                         {
1040                                 next = token->last;
1041                                 //token->last = NULL;
1042                         }
1043                         token->group = group;
1044                 }
1045                 else if (next == token)
1046                 {
1047                         next = NULL;
1048                 }
1049         }
1050
1051         return group;
1052 }
1053
1054 AnjutaToken *
1055 anjuta_token_merge_children (AnjutaToken *first, AnjutaToken *end)
1056 {
1057         if (first == NULL)
1058         {
1059                 return end;
1060         }
1061         if ((first == end) || (end == NULL))
1062         {
1063                 return first;
1064         }
1065
1066         if (first->parent == NULL)
1067         {
1068                 first->parent = end->parent;
1069         }
1070         if (first->next == NULL)
1071         {
1072                 anjuta_token_insert_before (end, first);
1073         }
1074         anjuta_token_unlink_token (end);
1075         if (end->last != NULL)
1076         {
1077                 AnjutaToken *child;
1078
1079                 first->last = end->last;
1080                 for (child = anjuta_token_next (first); child != first->last; child = anjuta_token_next (child))
1081                 {
1082                         if (child->group == end) child->group = first;
1083                 }
1084                 first->last->group = first;
1085         }
1086         end->group = first;
1087         anjuta_token_free (end);
1088
1089         return first;
1090 }
1091
1092 /**
1093  * anjuta_token_merge_previous:
1094  * @list: a #AnjutaToken object representing a list
1095  * @first: a #AnjutaToken object for the new beginning of the list
1096  *
1097  * If the list token is not already linked with first, it is inserted
1098  * just before first.
1099  * If the list token is already linked, it must be in the same list after
1100  * first token. It it possible to have several tokens beweent list and
1101  * first.
1102  *
1103  * Return value: the new list
1104  */
1105 AnjutaToken *
1106 anjuta_token_merge_previous (AnjutaToken *list, AnjutaToken *first)
1107 {
1108         AnjutaToken *token;
1109
1110         if ((first == NULL) || (list == first)) return list;
1111
1112         /* Change group of all tokens from end to first
1113          * if the list is already linked */
1114         if ((list->prev != NULL) || (list->parent != NULL))
1115         {
1116                 for (token = first; token != NULL; token = anjuta_token_next_item (token))
1117                 {
1118                         token->group = list;
1119                 }
1120         }
1121
1122         token = anjuta_token_next (list);
1123         anjuta_token_unlink_token (list);
1124         anjuta_token_insert_token_before (first, list);
1125
1126         return list;
1127 }
1128
1129 AnjutaToken *anjuta_token_split (AnjutaToken *token, guint size)
1130 {
1131         if (token->data.length > size)
1132         {
1133                 AnjutaToken *copy;
1134
1135                 copy = anjuta_token_copy (token);
1136                 anjuta_token_insert_before (token, copy);
1137
1138                 copy->data.length = size;
1139                 if (token->data.flags & ANJUTA_TOKEN_STATIC)
1140                 {
1141                         token->data.pos += size;
1142                         token->data.length -= size;
1143                 }
1144                 else
1145                 {
1146                         memcpy(token->data.pos, token->data.pos + size, token->data.length - size);
1147                 }
1148
1149                 return copy;
1150         }
1151         else
1152         {
1153                 return token;
1154         }
1155 }
1156
1157 AnjutaToken *
1158 anjuta_token_cut (AnjutaToken *token, guint pos, guint size)
1159 {
1160         AnjutaToken *copy;
1161
1162         copy = anjuta_token_copy (token);
1163
1164         if (pos >= token->data.length)
1165         {
1166                 if (!(copy->data.flags & ANJUTA_TOKEN_STATIC))
1167                 {
1168                         g_free (copy->data.pos);
1169                 }
1170                 copy->data.pos = NULL;
1171                 copy->data.length = 0;
1172         }
1173         if ((pos + size) > token->data.length)
1174         {
1175                 size = token->data.length - pos;
1176         }
1177
1178         if (copy->data.flags & ANJUTA_TOKEN_STATIC)
1179         {
1180                 copy->data.pos += pos;
1181         }
1182         else
1183         {
1184                 memcpy(copy->data.pos, copy->data.pos + pos, size);
1185         }
1186         copy->data.length = size;
1187
1188         return copy;
1189 }
1190
1191 static void
1192 concat_token (AnjutaToken *token, gpointer user_data)
1193 {
1194         AnjutaToken *first = (AnjutaToken *)user_data;
1195
1196         if (anjuta_token_get_length (token) > 0)
1197         {
1198                 if (anjuta_token_get_string (first) == NULL)
1199                 {
1200                         anjuta_token_set_string (first, anjuta_token_get_string (token), anjuta_token_get_length (token));
1201                 }
1202                 else if ((first != NULL) && ((anjuta_token_get_string(first) + anjuta_token_get_length (first)) == anjuta_token_get_string (token)))
1203                 {
1204                         anjuta_token_set_string (first, anjuta_token_get_string (first), anjuta_token_get_length (first) + anjuta_token_get_length (token));
1205                 }
1206                 else
1207                 {
1208                         AnjutaToken *new;
1209
1210                         new = anjuta_token_new_static_len (ANJUTA_TOKEN_CONTENT, anjuta_token_get_string (token), anjuta_token_get_length (token));
1211                         anjuta_token_insert_after (first, new);
1212                         anjuta_token_merge (first, new);
1213                 }
1214         }
1215 }
1216
1217
1218 AnjutaToken *
1219 anjuta_token_concat(AnjutaToken *token)
1220 {
1221         AnjutaToken *new;
1222
1223         new = anjuta_token_new_static (ANJUTA_TOKEN_CONTENT, NULL);
1224         anjuta_token_foreach_token (token, concat_token, new);
1225
1226         anjuta_token_insert_token_before (token, new);
1227         anjuta_token_free (token);
1228
1229         return new;
1230 }
1231
1232 /* Token foreach
1233  *---------------------------------------------------------------------------*/
1234
1235 void
1236 anjuta_token_foreach_token (AnjutaToken *token, AnjutaTokenForeachFunc func, gpointer user_data)
1237 {
1238         if (token != NULL)
1239         {
1240                 AnjutaToken *last_token;
1241                 gint child = 0;
1242
1243                 last_token = token->last == NULL ? token : token->last;
1244                 while (token != NULL)
1245                 {
1246                         if (child == 0) func (token, user_data);
1247
1248                         /* Check if we have found the last token */
1249                         if (token == last_token)
1250                         {
1251                                 /* Find last token */
1252                                 if (token->last == NULL)
1253                                 {
1254                                         break;
1255                                 }
1256                                 /* Last token still include additional tokens */
1257                                 last_token = token->last;
1258                         }
1259
1260                         if (token->children != NULL)
1261                         {
1262                                 /* Check children, only for last token */
1263                                 child++;
1264                                 token = token->children;
1265                         }
1266                         else if (token->next != NULL)
1267                         {
1268                                 /* Get next sibling */
1269                                 token = token->next;
1270                         }
1271                         else
1272                         {
1273                                 /* Get parent */
1274                                 for (;;)
1275                                 {
1276                                         child--;
1277                                         token = token->parent;
1278                                         if (token == NULL) break;
1279                                         if (token->next != NULL)
1280                                         {
1281                                                 token = token->next;
1282                                                 break;
1283                                         }
1284                                 }
1285                         }
1286                 }
1287         }
1288
1289         return;
1290 }
1291
1292 void
1293 anjuta_token_foreach_content (AnjutaToken *token, AnjutaTokenForeachFunc func, gpointer user_data)
1294 {
1295         if (token != NULL)
1296         {
1297                 AnjutaToken *last_parent;       /* If not NULL, token belong to a parent and
1298                                                                          * are not taken into account */
1299                 AnjutaToken *last_token;
1300                 gboolean expand = TRUE;
1301
1302                 last_parent = NULL;
1303                 last_token = token->last == NULL ? token : token->last;
1304                 for (;;)
1305                 {
1306                         if (expand && (token->children != NULL))
1307                         {
1308                                 /* Children of the last token does not belong to the group */
1309                                 if (token == last_token)
1310                                 {
1311                                         /* Find last token */
1312                                         if (token->last == NULL)
1313                                         {
1314                                                 break;
1315                                         }
1316                                         /* Last token still include additional tokens */
1317                                         last_token = token->last;
1318                                 }
1319
1320                                 /* Enumerate children */
1321                                 token = token->children;
1322                         }
1323                         else
1324                         {
1325                                 if (token->children == NULL)
1326                                 {
1327                                         /* Take into account only the content of token having no children */
1328                                         if (last_parent == NULL)
1329                                         {
1330                                                 /* Take into account only the content of group having no children */
1331                                                 func (token, user_data);
1332                                         }
1333                                 }
1334
1335                                 /* Check if we have found the last token */
1336                                 if (token == last_token)
1337                                 {
1338                                         /* Find last token */
1339                                         if (token->last == NULL)
1340                                         {
1341                                                 break;
1342                                         }
1343                                         /* Last token still include additional tokens */
1344                                         last_token = token->last;
1345                                 }
1346
1347                                 if (token == last_parent)
1348                                 {
1349                                         /* Find last parent */
1350                                         if (token->last == NULL)
1351                                         {
1352                                                 /* Found complete group having children */
1353                                                 last_parent = NULL;
1354                                         }
1355                                         else
1356                                         {
1357                                                 /* Parent group has additional token */
1358                                                 last_parent = token->last;
1359                                         }
1360                                 }
1361
1362                                 if (token->next != NULL)
1363                                 {
1364                                         /* Get next sibling */
1365                                         token = token->next;
1366                                         expand = TRUE;
1367                                 }
1368                                 else
1369                                 {
1370                                         /* Get parent */
1371                                         token = token->parent;
1372                                         if (token != NULL) last_parent = token->last;
1373                                         expand = FALSE;
1374                                 }
1375                         }
1376                 }
1377         }
1378
1379         return;
1380 }
1381
1382 void
1383 anjuta_token_foreach_container (AnjutaToken *token, AnjutaTokenForeachFunc func, gpointer user_data)
1384 {
1385         if (token != NULL)
1386         {
1387                 AnjutaToken *parent;    /* If not NULL, token belong to a parent and
1388                                                                          * are not taken into account */
1389                 AnjutaToken *last_token;
1390                 gboolean expand = TRUE;
1391
1392                 parent = NULL;
1393                 last_token = token->last == NULL ? token : token->last;
1394                 for (;;)
1395                 {
1396                         if (expand && (token->children != NULL))
1397                         {
1398                                 if (parent == NULL) parent = token;
1399                                 /* Enumerate children */
1400                                 token = token->children;
1401                         }
1402                         else
1403                         {
1404                                 /* Take into account only the content of parent token */
1405                                 if (parent == NULL)
1406                                 {
1407                                         /* Take into account only the content of group having no children */
1408                                         func (token, user_data);
1409                                 }
1410
1411                                 /* Check if we have found the last token */
1412                                 if (token == last_token)
1413                                 {
1414                                         /* Find last token */
1415                                         if (token->last == NULL)
1416                                         {
1417                                                 break;
1418                                         }
1419                                         /* Last token still include additional tokens */
1420                                         last_token = token->last;
1421                                 }
1422
1423                                 if (token->next != NULL)
1424                                 {
1425                                         /* Get next sibling */
1426                                         token = token->next;
1427                                         expand = TRUE;
1428                                 }
1429                                 else
1430                                 {
1431                                         /* Get parent */
1432                                         token = token->parent;
1433                                         if (token == parent) parent = NULL;
1434                                         expand = FALSE;
1435                                 }
1436                         }
1437                 }
1438         }
1439
1440         return;
1441 }
1442
1443 AnjutaToken *
1444 anjuta_token_foreach_post_order (AnjutaToken *token, AnjutaTokenForeachFunc func, gpointer user_data)
1445 {
1446         if (token != NULL)
1447         {
1448                 AnjutaToken *last_parent;       /* If not NULL, token belong to a parent */
1449                 AnjutaToken *last_token;
1450                 AnjutaToken buffer;                     /* Temporary token allowing func to destroy
1451                                                                          * the current token */
1452                 gboolean expand = TRUE;
1453
1454                 last_parent = NULL;
1455                 last_token = token->last == NULL ? token : token->last;
1456                 while (token != NULL)
1457                 {
1458                         if (expand && (token->children != NULL))
1459                         {
1460                                 /* Check if we have found the last token */
1461                                 if (token == last_token)
1462                                 {
1463                                         /* Find last token */
1464                                         if (token->last == NULL)
1465                                         {
1466                                                 break;
1467                                         }
1468                                         /* Last token still include additional tokens */
1469                                         last_token = token->last;
1470                                 }
1471
1472                                 /* Enumerate children */
1473                                 token = token->children;
1474                         }
1475                         else
1476                         {
1477                                 /* Save token data in case it is destroyed */
1478                                 memcpy (&buffer, token, sizeof (buffer));
1479                                 /* Take into account all token */
1480                                 func (token, user_data);
1481
1482                                 /* Check if we have found the last token */
1483                                 if (token == last_token)
1484                                 {
1485                                         /* Find last token */
1486                                         if (buffer.last == NULL)
1487                                         {
1488                                                 token = &buffer;
1489                                                 break;
1490                                         }
1491                                         /* Last token still include additional tokens */
1492                                         last_token = buffer.last;
1493                                 }
1494
1495                                 if (token == last_parent)
1496                                 {
1497                                         /* Find last parent */
1498                                         if (buffer.last == NULL)
1499                                         {
1500                                                 /* Found complete group having children */
1501                                                 last_parent = NULL;
1502                                         }
1503                                         else
1504                                         {
1505                                                 /* Parent group has additional token */
1506                                                 last_parent = buffer.last;
1507                                         }
1508                                 }
1509
1510                                 if (buffer.next != NULL)
1511                                 {
1512                                         /* Get next sibling */
1513                                         token = buffer.next;
1514                                         expand = TRUE;
1515                                 }
1516                                 else
1517                                 {
1518                                         /* Get parent */
1519                                         token = buffer.parent;
1520                                         if (token != NULL) last_parent = token->last;
1521                                         expand = FALSE;
1522                                 }
1523                         }
1524                 }
1525
1526                 while ((token != NULL) && (token->next == NULL))
1527                 {
1528                         token = token->parent;
1529                 };
1530
1531                 if (token != NULL) token = token->next;
1532         }
1533
1534         return token;
1535 }
1536
1537 /* Token evaluation
1538  *---------------------------------------------------------------------------*/
1539
1540 static void
1541 evaluate_raw_token (AnjutaToken *token, gpointer user_data)
1542 {
1543         GString *value = (GString *)user_data;
1544
1545         anjuta_token_evaluate_token (token, value, TRUE);
1546 }
1547
1548 static void
1549 evaluate_token (AnjutaToken *token, gpointer user_data)
1550 {
1551         GString *value = (GString *)user_data;
1552
1553         anjuta_token_evaluate_token (token, value, FALSE);
1554 }
1555
1556
1557 gchar *
1558 anjuta_token_evaluate (AnjutaToken *token)
1559 {
1560         GString *value = g_string_new (NULL);
1561
1562         anjuta_token_foreach_content (token, evaluate_token, value);
1563
1564         /* Return NULL and free data for an empty string */
1565         return g_string_free (value, *(value->str) == '\0');
1566 }
1567
1568 /* Does not evaluate content if token is a variable */
1569 gchar *
1570 anjuta_token_evaluate_name (AnjutaToken *token)
1571 {
1572         GString *value = g_string_new (NULL);
1573
1574         anjuta_token_foreach_container (token, evaluate_token, value);
1575
1576         /* Return NULL and free data for an empty string */
1577         return g_string_free (value, *(value->str) == '\0');
1578 }
1579
1580 gboolean
1581 anjuta_token_is_empty (AnjutaToken *token)
1582 {
1583         return (token == NULL) || ((token->data.length == 0) && (token->last == NULL) && (token->children == NULL));
1584 }
1585
1586
1587 /* Other functions
1588  *---------------------------------------------------------------------------*/
1589
1590 gboolean
1591 anjuta_token_compare (AnjutaToken *toka, AnjutaToken *tokb)
1592 {
1593         if (tokb->data.type)
1594         {
1595                 if (tokb->data.type != toka->data.type) return FALSE;
1596         }
1597
1598         if (tokb->data.type != ANJUTA_TOKEN_NONE)
1599         {
1600                 if (tokb->data.length != 0)
1601                 {
1602                         if (toka->data.length != tokb->data.length) return FALSE;
1603
1604                         if ((toka->data.flags & ANJUTA_TOKEN_CASE_INSENSITIVE)  && (tokb->data.flags & ANJUTA_TOKEN_CASE_INSENSITIVE))
1605                         {
1606                                 if (g_ascii_strncasecmp (toka->data.pos, tokb->data.pos, toka->data.length) != 0) return FALSE;
1607                         }
1608                         else
1609                         {
1610                                 if (strncmp (toka->data.pos, tokb->data.pos, toka->data.length) != 0) return FALSE;
1611                         }
1612                 }
1613         }
1614
1615         if (tokb->data.flags & ANJUTA_TOKEN_PUBLIC_FLAGS)
1616         {
1617                 if ((toka->data.flags & tokb->data.flags & ANJUTA_TOKEN_PUBLIC_FLAGS) == 0)
1618                         return FALSE;
1619         }
1620
1621         return TRUE;
1622 }
1623
1624 void
1625 anjuta_token_dump (AnjutaToken *token)
1626 {
1627         if (token == NULL) return;
1628
1629         anjuta_token_dump_child (token, 0, 0);
1630 }
1631
1632 void
1633 anjuta_token_dump_link (AnjutaToken *token)
1634 {
1635         AnjutaToken *last = token;
1636
1637         while (last->last != NULL) last = last->last;
1638
1639         for (; token != last; token = anjuta_token_next (token))
1640         {
1641                 anjuta_token_show (token, 0, 0);
1642         }
1643 }
1644
1645 gboolean
1646 anjuta_token_check (AnjutaToken *token)
1647 {
1648         if ((token->children != NULL) && (token->last != NULL))
1649         {
1650                 anjuta_token_show (token, 0, 0);
1651                 fprintf(stderr, "Error: Previous token has both non NULL children and last\n");
1652
1653                 return FALSE;
1654         }
1655
1656         if (token->children != NULL)
1657         {
1658                 AnjutaToken *child;
1659
1660                 for (child = token->children; child != NULL; child = child->next)
1661                 {
1662                         if (!anjuta_token_check_child (child, token)) return FALSE;
1663                 }
1664         }
1665
1666         if (token->last != NULL)
1667         {
1668                 AnjutaToken *child;
1669
1670                 for (child = anjuta_token_next (token); child != NULL; child = anjuta_token_next (child))
1671                 {
1672                         if (!anjuta_token_check (child)) return FALSE;
1673                         if (child == token->last) break;
1674                 }
1675         }
1676
1677         return TRUE;
1678 }
1679
1680 /* Constructor & Destructor
1681  *---------------------------------------------------------------------------*/
1682
1683 AnjutaToken *anjuta_token_new_string (AnjutaTokenType type, const gchar *value)
1684 {
1685         AnjutaToken *token;
1686
1687         if (value == NULL)
1688         {
1689                 token = anjuta_token_new_static (type, NULL);
1690         }
1691         else
1692         {
1693                 token = g_slice_new0 (AnjutaToken);
1694                 token->data.type = type  & ANJUTA_TOKEN_TYPE;
1695                 token->data.flags = type & ANJUTA_TOKEN_FLAGS;
1696                 token->data.pos = g_strdup (value);
1697                 token->data.length = strlen (value);
1698         }
1699
1700         return token;
1701 }
1702
1703 AnjutaToken*
1704 anjuta_token_new_string_len (AnjutaTokenType type, gchar *value, gsize length)
1705 {
1706         AnjutaToken *token;
1707
1708         if (value == NULL)
1709         {
1710                 token = anjuta_token_new_static (type, NULL);
1711         }
1712         else
1713         {
1714                 token = g_slice_new0 (AnjutaToken);
1715                 token->data.type = type  & ANJUTA_TOKEN_TYPE;
1716                 token->data.flags = type & ANJUTA_TOKEN_FLAGS;
1717                 token->data.pos = value;
1718                 token->data.length = length;
1719         }
1720
1721         return token;
1722 }
1723
1724 AnjutaToken *
1725 anjuta_token_new_static_len (gint type, const gchar *pos, gsize length)
1726 {
1727         AnjutaToken *token;
1728
1729         token = g_slice_new0 (AnjutaToken);
1730         token->data.type = type  & ANJUTA_TOKEN_TYPE;
1731         token->data.flags = (type & ANJUTA_TOKEN_FLAGS) | ANJUTA_TOKEN_STATIC;
1732         token->data.pos = (gchar *)pos;
1733         token->data.length = length;
1734
1735         return token;
1736 };
1737
1738 AnjutaToken *anjuta_token_new_static (AnjutaTokenType type, const char *value)
1739 {
1740         return anjuta_token_new_static_len (type, value, value == NULL ? 0 : strlen (value));
1741 }
1742
1743 AnjutaToken*
1744 anjuta_token_free_children (AnjutaToken *token)
1745 {
1746         AnjutaToken *child;
1747         AnjutaToken *last;
1748
1749         if (token == NULL) return NULL;
1750
1751         for (child = token->children; child != NULL; child = token->children)
1752         {
1753                 anjuta_token_free (child);
1754         }
1755         token->children = NULL;
1756
1757         if (token->last != NULL)
1758         {
1759                 last = token->last;
1760                 for (child = anjuta_token_next (token); child != NULL; child = anjuta_token_next (token))
1761                 {
1762                         anjuta_token_free (child);
1763                         if (child == last) break;
1764                 }
1765         }
1766         token->last = NULL;
1767
1768         return token;
1769 }
1770
1771 static void
1772 free_token (AnjutaToken *token, gpointer user_data)
1773 {
1774         anjuta_token_unlink_token (token);
1775         if ((token->data.pos != NULL) && !(token->data.flags & ANJUTA_TOKEN_STATIC))
1776         {
1777                 g_free (token->data.pos);
1778         }
1779         g_slice_free (AnjutaToken, token);
1780 }
1781
1782
1783 AnjutaToken*
1784 anjuta_token_free (AnjutaToken *token)
1785 {
1786         AnjutaToken *next;
1787
1788         if (token == NULL) return NULL;
1789
1790         next = anjuta_token_foreach_post_order (token, free_token, NULL);
1791
1792         return next;
1793 }