libanjuta/anjuta-token.c

   1 /* -*- Mode: C; indent-tabs-mode: t; c-basic-offset: 4; tab-width: 4 -*- */
   2 /*
   3  * anjuta-token.c
   4  * Copyright (C) Sébastien Granjoux 2009 <seb.sfo@free.fr>
   5  *
   6  * This program is free software: you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License as published by the
   8  * Free Software Foundation, either version 3 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful, but
  12  * WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  14  * See the GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License along
  17  * with this program.  If not, see <http://www.gnu.org/licenses/>.
  18  */
  19
  20 #include "anjuta-token.h"
  21
  22 #include "anjuta-debug.h"
  23
  24 #include <glib-object.h>
  25
  26 #include <stdio.h>
  27 #include <string.h>
  28 #include <stdlib.h>
  29
  30 /**
  31  * SECTION:anjuta-token
  32  * @title: Anjuta token
  33  * @short_description: Anjuta token
  34  * @see_also:
  35  * @stability: Unstable
  36  * @include: libanjuta/anjuta-token.h
  37  *
  38  * A #AnjutaToken represents a token. It is a sequence of characters associated
  39  * with a type representing its meaning. By example, a token can represent
  40  * a keyword, a comment, a variable...
  41  *
  42  * The token can own the string or has only a pointer on some data allocated
  43  * somewhere else with a length.
  44  *
  45  * A token is linked with other tokens using three double linked lists.
  46  *
  47  * The first list using next and prev fields is used to keep the token in the
  48  * order where there are in the file. The first character of the first token is
  49  * the first character in the file.
  50  *
  51  * A second list is used to represent included
  52  * files. Such file is represented by a special token in the first list which
  53  * has a pointer, named children to a token list. Each token in this secondary
  54  * list has a pointer to its parent, it means the token representing the file
  55  * where is the token. It looks like a tree. In fact, every file is represented
  56  * by a special token, so the root node is normally a file token and has as
  57  * children all the token representing the file content. This parent/child list
  58  * is used for expanded variable too.
  59  *
  60  * A third list is used to group several tokens. A token can have a pointer to
  61  * another last token. It means that this token is a group starting from this
  62  * token to the one indicated by the last field. In addition each token in this
  63  * group has a pointer on the first token of the group. This grouping is
  64  * independent of the parent/child list. So a group can start in one file and
  65  * end in another included file. The grouping can be nested too. Typically
  66  * we can have a group representing a command, a sub group representing the
  67  * arguments and then one sub group for each argument.
  68  */
  69
  70 /*
  71  * A token with its pointer can be put in one of the following case:
  72  *
  73  * * Simple token:
  74  *              children = NULL && last = NULL
  75  *              This is the case of a simple token without any children.
  76  *
  77  * * Composite token:
  78  *              children = NULL && last != NULL
  79  *              This is the case of a token grouping several other tokens, representing
  80  *              by example an item in a list.
  81  *
  82  * * Parent token:
  83  *              children != NULL && last == NULL
  84  *              This is the case of a token having children, by example a variable where
  85  *              the children represent the content. Most of the time the value of the
  86  *              parent token is ignored.
  87  *
  88  * * Composite parent token:
  89  *              children != NULL && last != NULL && (last in list)
  90  *              This case represents a variable which is split into several tokens. The
  91  *              children represents the content of the variable. The token up to last
  92  *              corresponds to the variable name. After getting the variable content, the
  93  *              next token is after the last token of the variable.
  94  *
  95  * * Composite parent token:
  96  *              children != NULL && last != NULL && (last in children)
  97  *              This case would represent a variable which is split into several token
  98  *              with one inside the children of the variable. This is not possible as
  99  *              the complete name of the variable is needed to get the children.
 100  *
 101  * * Composite parent token:
 102  *              children != NULL && last != NULL && (last in parent)
 103  *              This case represents a variable split into several token where the
 104  *              last element is a sibling of one parent of the token. I think this case
 105  *              is possible.
 106  *
 107  */
 108
 109 typedef struct _AnjutaTokenData AnjutaTokenData;
 110
 111 struct _AnjutaTokenData
 112 {
 113         AnjutaTokenType type;
 114         gint flags;
 115         gchar *pos;
 116         gsize length;
 117 };
 118
 119 struct _AnjutaToken
 120 {
 121         AnjutaToken     *next;
 122         AnjutaToken     *prev;
 123         AnjutaToken     *parent;
 124         AnjutaToken *last;
 125         AnjutaToken *group;
 126         AnjutaToken     *children;
 127         AnjutaTokenData data;
 128 };
 129
 130 /* Helpers functions
 131  *---------------------------------------------------------------------------*/
 132
 133 /* Private functions
 134  *---------------------------------------------------------------------------*/
 135
 136 static AnjutaToken *
 137 anjuta_token_next_child (AnjutaToken *child, AnjutaToken **last)
 138 {
 139         if (child == NULL) return child;
 140
 141         if (child->children != NULL)
 142         {
 143                 child = child->children;
 144         }
 145         else
 146         {
 147                 for (;;)
 148                 {
 149                         if ((*last == NULL) || (child == *last))
 150                         {
 151                                 if (child->last == NULL)
 152                                 {
 153                                         child = NULL;
 154                                         break;
 155                                 }
 156                                 *last = child->last;
 157                         }
 158                         if (child->next != NULL)
 159                         {
 160                                 child = child->next;
 161                                 break;
 162                         }
 163                         child = child->parent;
 164                 }
 165         }
 166
 167         return child;
 168 }
 169
 170 AnjutaToken *
 171 anjuta_token_next_after_children (AnjutaToken *token)
 172 {
 173         while (token->last != NULL) token = token->last;
 174         while (token->next == NULL)
 175         {
 176                 token = token->parent;
 177                 if (token == NULL) return NULL;
 178                 while (token->last != NULL) token = token->last;
 179         };
 180
 181         return token->next;
 182 }
 183
 184 static AnjutaToken *
 185 anjuta_token_copy (AnjutaToken *token)
 186 {
 187         AnjutaToken *copy = NULL;
 188
 189         if (token != NULL)
 190         {
 191                 copy = g_slice_new0 (AnjutaToken);
 192                 copy->data.type = token->data.type;
 193                 copy->data.flags = token->data.flags;
 194                 if ((copy->data.flags & ANJUTA_TOKEN_STATIC) || (token->data.pos == NULL))
 195                 {
 196                         copy->data.pos = token->data.pos;
 197                 }
 198                 else
 199                 {
 200                         copy->data.pos = g_strdup (token->data.pos);
 201                 }
 202                 copy->data.length = token->data.length;
 203         }
 204
 205         return copy;
 206 }
 207
 208 /**
 209  * anjuta_token_unlink_token:
 210  * @token: a #AnjutaToken object.
 211  *
 212  * Unlink a single token, not the complete item, from the token tree.
 213  *
 214  * Return value: the removed token tree
 215  */
 216 static AnjutaToken *
 217 anjuta_token_unlink_token (AnjutaToken *token)
 218 {
 219
 220         if (token->prev != NULL)
 221         {
 222                 token->prev->next = token->next;
 223         }
 224         else if ((token->parent != NULL) && (token->parent->children == token))
 225         {
 226                 token->parent->children = token->next;
 227         }
 228         token->parent = NULL;
 229
 230         if ((token->group != NULL) && (token->group->last == token))
 231         {
 232                 AnjutaToken *prev;
 233
 234                 for (prev = token->prev; prev != NULL; prev = prev->prev)
 235                 {
 236                         if (prev->group == token->group)
 237                         {
 238                                 /* Find previous token in the same group */
 239                                 token->group->last = prev;
 240                                 break;
 241                         }
 242                         else if (prev == token->group)
 243                         {
 244                                 /* No more token in group */
 245                                 token->group->last = NULL;
 246                                 break;
 247                         }
 248                 }
 249         }
 250
 251         if (token->next != NULL)
 252         {
 253                 token->next->prev = token->prev;
 254                 token->next = NULL;
 255         }
 256         token->prev = NULL;
 257
 258         return token;
 259 }
 260
 261 /**
 262  * anjuta_token_insert_token_before:
 263  * @sibling: a #AnjutaToken object.
 264  * @token: a #AnjutaToken object.
 265  *
 266  * Insert token before sibling.
 267  *
 268  * Return value: inserted token
 269  */
 270 static AnjutaToken *
 271 anjuta_token_insert_token_before (AnjutaToken *sibling, AnjutaToken *token)
 272 {
 273         token->prev = sibling->prev;
 274         token->next = sibling;
 275
 276         if (token->prev != NULL)
 277         {
 278                 token->prev->next = token;
 279         }
 280         sibling->prev = token;
 281
 282         if ((sibling->parent != NULL) && (sibling->parent->children == sibling))
 283         {
 284                 sibling->parent->children = token;
 285         }
 286         token->parent = sibling->parent;
 287
 288         return token;
 289 }
 290
 291 static void
 292 anjuta_token_evaluate_token (AnjutaToken *token, GString *value, gboolean raw)
 293 {
 294         if ((token != NULL) && (token->data.length != 0))
 295         {
 296                 if (!raw)
 297                 {
 298                         switch (anjuta_token_get_type (token))
 299                         {
 300                         case ANJUTA_TOKEN_COMMENT:
 301                         case ANJUTA_TOKEN_OPEN_QUOTE:
 302                         case ANJUTA_TOKEN_CLOSE_QUOTE:
 303                         case ANJUTA_TOKEN_ESCAPE:
 304                         case ANJUTA_TOKEN_MACRO:
 305                         case ANJUTA_TOKEN_EOV:
 306                                 return;
 307                         default:
 308                                 break;
 309                         }
 310                 }
 311                 g_string_append_len (value, anjuta_token_get_string (token), anjuta_token_get_length (token));
 312         }
 313 }
 314
 315 static void
 316 anjuta_token_show (AnjutaToken *token, gint indent, gchar parent)
 317 {
 318         static gchar type[] = "\0";
 319         const gchar *string;
 320         gsize length;
 321
 322         type[0] = parent;
 323         fprintf (stderr, "%*s%s %p", indent, "", type, token);
 324         fprintf (stderr, ": %d ",
 325             anjuta_token_get_type (token));
 326         string = anjuta_token_get_string (token);
 327         length = anjuta_token_get_length (token);
 328         if (string == NULL)
 329         {
 330                 /* Value doesn't contain a newline */
 331                 fprintf (stderr, "(%lu)", length);
 332         }
 333         else
 334         {
 335                 const gchar *newline;
 336
 337                 newline = g_strrstr_len (string, length, "\n");
 338                 if (newline == NULL)
 339                 {
 340                         /* Value doesn't contain a newline */
 341                         fprintf (stderr, "\"%.*s\"",
 342                                 length,
 343                         string);
 344                 }
 345                 else
 346                 {
 347                         /* Value contains a newline, take care of indentation */
 348                         newline++;
 349                         fprintf (stderr, "\"%.*s",
 350                         newline - string,
 351                         string);
 352                         for (;;)
 353                         {
 354                                 length -= newline - string;
 355                                 string = newline;
 356
 357                                 newline = g_strrstr_len (string, length, "\n");
 358                                 if (newline == NULL) break;
 359
 360                                 newline++;
 361                                 fprintf (stderr, "%*s  %.*s",
 362                                     indent, "",
 363                                         newline - string,
 364                                 string);
 365                         }
 366                         fprintf (stderr, "%*s  %.*s\"",
 367                                     indent, "",
 368                                         length,
 369                                 string);
 370                 }
 371         }
 372         fprintf (stderr, " %p/%p (%p/%p) %s\n",
 373                 token->last, token->children,
 374                 token->group, token->parent,
 375                 anjuta_token_get_flags (token) & ANJUTA_TOKEN_REMOVED ? " (removed)" : "");
 376 }
 377
 378 static AnjutaToken*
 379 anjuta_token_dump_child (AnjutaToken *token, gint indent, gchar type)
 380 {
 381         AnjutaToken *last;
 382         AnjutaToken *child;
 383
 384         anjuta_token_show (token, indent, type);
 385         indent += 4;
 386
 387         last = token;
 388         if (token->last != NULL)
 389         {
 390                 do
 391                 {
 392                         child = last->next;
 393                         if (child == NULL) break;
 394                         last = anjuta_token_dump_child (child, indent, '+');
 395                 }
 396                 while (child != token->last);
 397         }
 398
 399         if (token->children != NULL)
 400         {
 401                 for (child = token->children; child != NULL; child = child->next)
 402                 {
 403                         child = anjuta_token_dump_child (child, indent, '*');
 404                 }
 405         }
 406
 407         return last;
 408 }
 409
 410 static gboolean
 411 anjuta_token_check_child (AnjutaToken *token, AnjutaToken *parent)
 412 {
 413         if (token->parent != parent)
 414         {
 415                 anjuta_token_show (token, 0, 0);
 416                 fprintf(stderr, "Error: Children has %p as parent instead of %p\n", token->parent, parent);
 417                 return FALSE;
 418         }
 419
 420         return anjuta_token_check (token);
 421 }
 422
 423 /* Get and set functions
 424  *---------------------------------------------------------------------------*/
 425
 426 void
 427 anjuta_token_set_type (AnjutaToken *token, gint type)
 428 {
 429         token->data.type = type;
 430 }
 431
 432 gint
 433 anjuta_token_get_type (AnjutaToken *token)
 434 {
 435         return token->data.type;
 436 }
 437
 438 void
 439 anjuta_token_set_flags (AnjutaToken *token, gint flags)
 440 {
 441         AnjutaToken *child;
 442         AnjutaToken *last = token->last;
 443
 444         for (child = token; child != NULL; child = anjuta_token_next_child (child, &last))
 445         {
 446                 child->data.flags |= flags;
 447         }
 448 }
 449
 450 void
 451 anjuta_token_clear_flags (AnjutaToken *token, gint flags)
 452 {
 453         token->data.flags &= ~flags;
 454 }
 455
 456 gint
 457 anjuta_token_get_flags (AnjutaToken *token)
 458 {
 459         return token->data.flags;
 460 }
 461
 462 void
 463 anjuta_token_set_string (AnjutaToken *token, const gchar *data, gsize length)
 464 {
 465         if (!(token->data.flags & ANJUTA_TOKEN_STATIC))
 466         {
 467                 g_free (token->data.pos);
 468                 token->data.flags |= ANJUTA_TOKEN_STATIC;
 469         }
 470         token->data.pos = (gchar *)data;
 471         token->data.length = length;
 472 }
 473
 474 const gchar *
 475 anjuta_token_get_string (AnjutaToken *token)
 476 {
 477         return token->data.pos;
 478 }
 479
 480 void
 481 anjuta_token_set_length (AnjutaToken *token, gsize length)
 482 {
 483         token->data.length = length;
 484 }
 485
 486 gsize
 487 anjuta_token_get_length (AnjutaToken *token)
 488 {
 489         return token->data.length;
 490 }
 491
 492 /* Basic move functions
 493  *---------------------------------------------------------------------------*/
 494
 495 AnjutaToken *
 496 anjuta_token_next (AnjutaToken *token)
 497 {
 498         if (token->children != NULL)
 499         {
 500                 return token->children;
 501         }
 502         else if (token->next != NULL)
 503         {
 504                 return token->next;
 505         }
 506         else if (token->parent != NULL)
 507         {
 508                 return anjuta_token_next_after_children (token->parent);
 509         }
 510         else
 511         {
 512                 return NULL;
 513         }
 514 }
 515
 516 AnjutaToken *
 517 anjuta_token_previous (AnjutaToken *token)
 518 {
 519         if (token->prev != NULL)
 520         {
 521                 return token->prev;
 522         }
 523         else
 524         {
 525                 return token->parent;
 526         }
 527 }
 528
 529 AnjutaToken *
 530 anjuta_token_last (AnjutaToken *token)
 531 {
 532         AnjutaToken *last;
 533
 534         for (last = token; last->last != NULL; last = last->last);
 535         if (last->children != NULL)
 536         {
 537                 for (last = last->children; last->next != NULL; last = last->next);
 538         }
 539
 540         return last;
 541 }
 542
 543 AnjutaToken *
 544 anjuta_token_parent (AnjutaToken *token)
 545 {
 546         return token->parent;
 547 }
 548
 549 AnjutaToken *
 550 anjuta_token_list (AnjutaToken *token)
 551 {
 552         return token->group;
 553 }
 554
 555 /* Item move functions
 556  *---------------------------------------------------------------------------*/
 557
 558 AnjutaToken *
 559 anjuta_token_last_item (AnjutaToken *list)
 560 {
 561         return list->last;
 562 }
 563
 564 AnjutaToken *
 565 anjuta_token_first_item (AnjutaToken *list)
 566 {
 567         AnjutaToken *first = NULL;
 568
 569         if (list != NULL)
 570         {
 571                 if (list->children != NULL)
 572                 {
 573                         first = list->children;
 574                 }
 575                 else if (list->last != NULL)
 576                 {
 577                         first = list->next;
 578                 }
 579         }
 580
 581         return first;
 582 }
 583
 584 AnjutaToken *
 585 anjuta_token_next_item (AnjutaToken *item)
 586 {
 587         AnjutaToken *next;
 588
 589         if (item != NULL)
 590         {
 591                 do
 592                 {
 593                         next = NULL;
 594                         if ((item->group == NULL) || (item->group->last != item))
 595                         {
 596                                 AnjutaToken *last;
 597                                 for (last = item; last->last != NULL; last = last->last);
 598                                 next = anjuta_token_next (last);
 599                                 if ((next != NULL) && (next->group != item->group)) next = NULL;
 600                         }
 601                         item = next;
 602                 }
 603                 /* Loop if the current item has been deleted */
 604                 while ((next != NULL) && (anjuta_token_get_flags (next) & ANJUTA_TOKEN_REMOVED));
 605         }
 606
 607         return next;
 608 }
 609
 610 AnjutaToken *
 611 anjuta_token_previous_item (AnjutaToken *item)
 612 {
 613         AnjutaToken *prev = NULL;
 614
 615
 616         if (item != NULL)
 617         {
 618                 do
 619                 {
 620                         for (prev = item->prev; (prev != NULL) && (prev->group != item->group); prev = prev->group);
 621                         item = prev;
 622                 }
 623                 /* Loop if the current item has been deleted */
 624                 while ((prev != NULL) && (anjuta_token_get_flags (prev) & ANJUTA_TOKEN_REMOVED));
 625         }
 626
 627         return prev;
 628 }
 629
 630 /* Add/Insert/Remove tokens
 631  *---------------------------------------------------------------------------*/
 632
 633 /**
 634  * anjuta_token_append_child:
 635  * @parent: a #AnjutaToken object used as parent.
 636  * @children: a #AnjutaToken object.
 637  *
 638  * Insert all tokens in children as the last children of the given parent.
 639  *
 640  * Return value: The first token append.
 641  */
 642 AnjutaToken *
 643 anjuta_token_append_child (AnjutaToken *parent, AnjutaToken *children)
 644 {
 645         AnjutaToken *token;
 646         AnjutaToken *last;
 647         AnjutaToken *old_group;
 648         AnjutaToken *old_parent;
 649
 650         g_return_val_if_fail (parent != NULL, NULL);
 651         g_return_val_if_fail (children != NULL, NULL);
 652
 653         old_group = children->group;
 654         old_parent = children->parent;
 655
 656         if (parent->children == NULL)
 657         {
 658                 parent->children = children;
 659
 660                 children->prev = NULL;
 661         }
 662         else
 663         {
 664                 /* Find last children */
 665                 for (last = parent->children; last->next != NULL;)
 666                 {
 667                         if ((last->last != NULL) && (last->last->parent == last->parent))
 668                         {
 669                                 last = last->last;
 670                         }
 671                         else
 672                         {
 673                                 last = last->next;
 674                         }
 675                 }
 676
 677                 last->next = children;
 678                 children->prev = last;
 679         }
 680
 681         /* Update each token */
 682         for (token = children;;)
 683         {
 684                 if (token->parent == old_parent) token->parent = parent;
 685                 if (token->group == old_group) token->group = parent->group;
 686
 687                 if (token->children != NULL)
 688                 {
 689                         token = token->children;
 690                 }
 691                 else if (token->next != NULL)
 692                 {
 693                         token = token->next;
 694                 }
 695                 else
 696                 {
 697                         while (token->parent != parent)
 698                         {
 699                                 token = token->parent;
 700                                 if (token->next != NULL) break;
 701                         }
 702                         if (token->next == NULL) break;
 703                         token = token->next;
 704                 }
 705         }
 706
 707         return children;
 708 }
 709
 710 /**
 711  * anjuta_token_prepend_child:
 712  * @parent: a #AnjutaToken object used as parent.
 713  * @children: a #AnjutaToken object.
 714  *
 715  * Insert all tokens in children as the first children of the given parent.
 716  *
 717  * Return value: The first token append.
 718  */
 719 AnjutaToken *
 720 anjuta_token_prepend_child (AnjutaToken *parent, AnjutaToken *children)
 721 {
 722         AnjutaToken *child;
 723         AnjutaToken *last = NULL;
 724
 725         g_return_val_if_fail (parent != NULL, NULL);
 726         g_return_val_if_fail (children != NULL, NULL);
 727
 728         /* Update each token */
 729         for (child = children;;)
 730         {
 731                 AnjutaToken *next;
 732
 733                 if (child->parent == children->parent) child->parent = parent;
 734                 if (child->group == children->group) child->group = parent->group;
 735
 736                 next = anjuta_token_next_child (child, &last);
 737                 if (next == NULL) break;
 738                 child = next;
 739         }
 740
 741         child->next = parent->children;
 742         if (child->next) child->next->prev = child;
 743         parent->children = children;
 744
 745         return children;
 746 }
 747
 748 /**
 749  * anjuta_token_prepend_items:
 750  * @list: a #AnjutaToken object used as list.
 751  * @item: a #AnjutaToken object.
 752  *
 753  * Insert all tokens in item as item of the given list.
 754  *
 755  * Return value: The first token append.
 756  */
 757 AnjutaToken *
 758 anjuta_token_prepend_items (AnjutaToken *list, AnjutaToken *item)
 759 {
 760         AnjutaToken *token;
 761         AnjutaToken *old_group;
 762         AnjutaToken *old_parent;
 763
 764         g_return_val_if_fail (list != NULL, NULL);
 765         g_return_val_if_fail (item != NULL, NULL);
 766
 767         old_group = item->group;
 768         old_parent = item->parent;
 769
 770         /* Update each token */
 771         for (token = item;;)
 772         {
 773                 if (token->parent == old_parent) token->parent = list->parent;
 774                 if (token->group == old_group) token->group = list;
 775
 776                 if (token->children != NULL)
 777                 {
 778                         token = token->children;
 779                 }
 780                 else if (token->next != NULL)
 781                 {
 782                         token = token->next;
 783                 }
 784                 else
 785                 {
 786                         while (token->parent != list->parent)
 787                         {
 788                                 token = token->parent;
 789                                 if (token->next != NULL) break;
 790                         }
 791                         if (token->next == NULL) break;
 792                         token = token->next;
 793                 }
 794         }
 795
 796         token->next = list->next;
 797         if (token->next) token->next->prev = token;
 798
 799         list->next = item;
 800         item->prev = list;
 801
 802         if (list->last == NULL)
 803         {
 804                 while (token->group != list) token = token->group;
 805                 list->last = token;
 806         }
 807
 808         return item;
 809 }
 810
 811 /**
 812  * anjuta_token_insert_after:
 813  * @sibling: a #AnjutaToken object.
 814  * @list: a #AnjutaToken object.
 815  *
 816  * Insert all tokens after sibling.
 817  *
 818  * Return value: The first token inserted.
 819  */
 820 AnjutaToken *
 821 anjuta_token_insert_after (AnjutaToken *sibling, AnjutaToken *list)
 822 {
 823         AnjutaToken *last;
 824         AnjutaToken *token;
 825         AnjutaToken *old_group;
 826         AnjutaToken *old_parent;
 827
 828         g_return_val_if_fail (sibling != NULL, NULL);
 829         g_return_val_if_fail (list != NULL, NULL);
 830
 831         old_group = list->group;
 832         old_parent = list->parent;
 833
 834         /* Update each token */
 835         for (token = list;;)
 836         {
 837                 if (token->parent == old_parent) token->parent = sibling->parent;
 838                 if (token->group == old_group) token->group = sibling->group;
 839
 840                 if (token->children != NULL)
 841                 {
 842                         token = token->children;
 843                 }
 844                 else if (token->next != NULL)
 845                 {
 846                         token = token->next;
 847                 }
 848                 else
 849                 {
 850                         while (token->parent != sibling->parent)
 851                         {
 852                                 token = token->parent;
 853                                 if (token->next != NULL) break;
 854                         }
 855                         if (token->next == NULL) break;
 856                         token = token->next;
 857                 }
 858         }
 859
 860         for (last = sibling; last->last != NULL; last = last->last);
 861
 862         token->next = last->next;
 863         if (token->next) token->next->prev = token;
 864
 865         last->next = list;
 866         list->prev = last;
 867
 868         if ((sibling->group != NULL) && (sibling->group->last == sibling))
 869         {
 870                 while (token->group != sibling->group) token = token->group;
 871                 sibling->group->last = token;
 872         }
 873
 874         return list;
 875 }
 876
 877 /**
 878  * anjuta_token_insert_before:
 879  * @sibling: a #AnjutaToken object.
 880  * @list: a #AnjutaToken object.
 881  *
 882  * Insert all tokens before sibling.
 883  *
 884  * Return value: The first token inserted.
 885  */
 886 AnjutaToken *
 887 anjuta_token_insert_before (AnjutaToken *sibling, AnjutaToken *list)
 888 {
 889         AnjutaToken *last;
 890         AnjutaToken *token;
 891         AnjutaToken *old_group;
 892         AnjutaToken *old_parent;
 893
 894         g_return_val_if_fail (sibling != NULL, NULL);
 895         g_return_val_if_fail (list != NULL, NULL);
 896
 897         old_group = list->group;
 898         old_parent = list->parent;
 899
 900         /* Update each token */
 901         for (token = list;;)
 902         {
 903                 if (token->parent == old_parent) token->parent = sibling->parent;
 904                 if (token->group == old_group) token->group = sibling->group;
 905
 906                 if (token->children != NULL)
 907                 {
 908                         token = token->children;
 909                 }
 910                 else if (token->next != NULL)
 911                 {
 912                         token = token->next;
 913                 }
 914                 else
 915                 {
 916                         while (token->parent != sibling->parent)
 917                         {
 918                                 token = token->parent;
 919                                 if (token->next != NULL) break;
 920                         }
 921                         if (token->next == NULL) break;
 922                         token = token->next;
 923                 }
 924         }
 925
 926         for (last = sibling; last->last != NULL; last = last->last);
 927
 928         token->next = sibling;
 929         list->prev = sibling->prev;
 930         sibling->prev = token;
 931
 932         if (list->prev) list->prev->next = list;
 933
 934         if ((list->parent != NULL) && (list->parent->children == sibling)) list->parent->children = list;
 935
 936         return list;
 937 }
 938
 939 /**
 940  * anjuta_token_delete_parent:
 941  * @parent: a #AnjutaToken object used as parent.
 942  *
 943  * Delete only the parent token.
 944  *
 945  * Return value: the first children
 946  */
 947 AnjutaToken *
 948 anjuta_token_delete_parent (AnjutaToken *parent)
 949 {
 950         AnjutaToken *token;
 951
 952         g_return_val_if_fail (parent != NULL, NULL);
 953
 954         if (parent->children == NULL) return NULL;
 955
 956         /* Update each token */
 957         for (token = parent->children;;)
 958         {
 959                 if (token->parent == parent) token->parent = parent->parent;
 960
 961                 if (token->children != NULL)
 962                 {
 963                         token = token->children;
 964                 }
 965                 else if (token->next != NULL)
 966                 {
 967                         token = token->next;
 968                 }
 969                 else
 970                 {
 971                         while (token->parent != parent->parent)
 972                         {
 973                                 token = token->parent;
 974                                 if (token->next != NULL) break;
 975                         }
 976                         if (token->next == NULL) break;
 977                         token = token->next;
 978                 }
 979         }
 980
 981         token->next = parent->next;
 982         if (token->next) token->next->prev = token;
 983
 984         parent->next = parent->children;
 985         parent->children->prev = parent;
 986         parent->children = NULL;
 987
 988         return anjuta_token_free (parent);
 989 }
 990
 991 /* Merge function
 992  *---------------------------------------------------------------------------*/
 993
 994 /* anjuta_token_merge can be used with first or end being a floating token and
 995  * on already grouped tokens to change the group organisation */
 996 AnjutaToken *
 997 anjuta_token_merge (AnjutaToken *first, AnjutaToken *end)
 998 {
 999         AnjutaToken *next;
1000
1001         if ((first == end) || (end == NULL)) return first;
1002
1003         /* Insert first or end in the same sequence if it is not already the case */
1004         for (next = first; next != end; next = anjuta_token_next (next))
1005         {
1006                 if (next == NULL)
1007                 {
1008                         if (first->parent == NULL)
1009                         {
1010                                 anjuta_token_insert_before (end, first);
1011                         }
1012                         else
1013                         {
1014                                 anjuta_token_insert_after (first, end);
1015                         }
1016                         break;
1017                 }
1018         }
1019         first->last = end;
1020         if ((end->group != NULL) && (end->group != first) && (end->group->last == end)) end->group->last = first;
1021         end->group = first;
1022
1023         return first;
1024 }
1025
1026 AnjutaToken *
1027 anjuta_token_merge_own_children (AnjutaToken *group)
1028 {
1029         AnjutaToken *token;
1030         AnjutaToken *next = NULL;
1031
1032         if (group->last != NULL) return group;
1033
1034         if (group->last->last != NULL) group->last = group->last->last;
1035
1036         for (token = anjuta_token_next (group); (token != NULL) && (token != group->last); token = anjuta_token_next (token))
1037         {
1038                 if (next == NULL)
1039                 {
1040                         if (token->last != NULL)
1041                         {
1042                                 next = token->last;
1043                                 //token->last = NULL;
1044                         }
1045                         token->group = group;
1046                 }
1047                 else if (next == token)
1048                 {
1049                         next = NULL;
1050                 }
1051         }
1052
1053         return group;
1054 }
1055
1056 AnjutaToken *
1057 anjuta_token_merge_children (AnjutaToken *first, AnjutaToken *end)
1058 {
1059         if (first == NULL)
1060         {
1061                 return end;
1062         }
1063         if ((first == end) || (end == NULL))
1064         {
1065                 return first;
1066         }
1067
1068         if (first->parent == NULL)
1069         {
1070                 first->parent = end->parent;
1071         }
1072         if (first->next == NULL)
1073         {
1074                 anjuta_token_insert_before (end, first);
1075         }
1076         anjuta_token_unlink_token (end);
1077         if (end->last != NULL)
1078         {
1079                 AnjutaToken *child;
1080
1081                 first->last = end->last;
1082                 for (child = anjuta_token_next (first); child != first->last; child = anjuta_token_next (child))
1083                 {
1084                         if (child->group == end) child->group = first;
1085                 }
1086                 first->last->group = first;
1087         }
1088         end->group = first;
1089         anjuta_token_free (end);
1090
1091         return first;
1092 }
1093
1094 /**
1095  * anjuta_token_merge_previous:
1096  * @list: a #AnjutaToken object representing a list
1097  * @first: a #AnjutaToken object for the new beginning of the list
1098  *
1099  * If the list token is not already linked with first, it is inserted
1100  * just before first.
1101  * If the list token is already linked, it must be in the same list after
1102  * first token. It it possible to have several tokens beweent list and
1103  * first.
1104  *
1105  * Return value: the new list
1106  */
1107 AnjutaToken *
1108 anjuta_token_merge_previous (AnjutaToken *list, AnjutaToken *first)
1109 {
1110         AnjutaToken *token;
1111
1112         if ((first == NULL) || (list == first)) return list;
1113
1114         /* Change group of all tokens from end to first
1115          * if the list is already linked */
1116         if ((list->prev != NULL) || (list->parent != NULL))
1117         {
1118                 for (token = first; token != NULL; token = anjuta_token_next_item (token))
1119                 {
1120                         token->group = list;
1121                 }
1122         }
1123
1124         token = anjuta_token_next (list);
1125         anjuta_token_unlink_token (list);
1126         anjuta_token_insert_token_before (first, list);
1127
1128         return list;
1129 }
1130
1131 AnjutaToken *anjuta_token_split (AnjutaToken *token, guint size)
1132 {
1133         if (token->data.length > size)
1134         {
1135                 AnjutaToken *copy;
1136
1137                 copy = anjuta_token_copy (token);
1138                 anjuta_token_insert_before (token, copy);
1139
1140                 copy->data.length = size;
1141                 if (token->data.flags & ANJUTA_TOKEN_STATIC)
1142                 {
1143                         token->data.pos += size;
1144                         token->data.length -= size;
1145                 }
1146                 else
1147                 {
1148                         memcpy(token->data.pos, token->data.pos + size, token->data.length - size);
1149                 }
1150
1151                 return copy;
1152         }
1153         else
1154         {
1155                 return token;
1156         }
1157 }
1158
1159 AnjutaToken *
1160 anjuta_token_cut (AnjutaToken *token, guint pos, guint size)
1161 {
1162         AnjutaToken *copy;
1163
1164         copy = anjuta_token_copy (token);
1165
1166         if (pos >= token->data.length)
1167         {
1168                 if (!(copy->data.flags & ANJUTA_TOKEN_STATIC))
1169                 {
1170                         g_free (copy->data.pos);
1171                 }
1172                 copy->data.pos = NULL;
1173                 copy->data.length = 0;
1174         }
1175         if ((pos + size) > token->data.length)
1176         {
1177                 size = token->data.length - pos;
1178         }
1179
1180         if (copy->data.flags & ANJUTA_TOKEN_STATIC)
1181         {
1182                 copy->data.pos += pos;
1183         }
1184         else
1185         {
1186                 memmove(copy->data.pos, copy->data.pos + pos, size);
1187         }
1188         copy->data.length = size;
1189
1190         return copy;
1191 }
1192
1193 static void
1194 concat_token (AnjutaToken *token, gpointer user_data)
1195 {
1196         AnjutaToken *first = (AnjutaToken *)user_data;
1197
1198         if (anjuta_token_get_length (token) > 0)
1199         {
1200                 if (anjuta_token_get_string (first) == NULL)
1201                 {
1202                         anjuta_token_set_string (first, anjuta_token_get_string (token), anjuta_token_get_length (token));
1203                 }
1204                 else if ((first != NULL) && ((anjuta_token_get_string(first) + anjuta_token_get_length (first)) == anjuta_token_get_string (token)))
1205                 {
1206                         anjuta_token_set_string (first, anjuta_token_get_string (first), anjuta_token_get_length (first) + anjuta_token_get_length (token));
1207                 }
1208                 else
1209                 {
1210                         AnjutaToken *new;
1211
1212                         new = anjuta_token_new_static_len (ANJUTA_TOKEN_CONTENT, anjuta_token_get_string (token), anjuta_token_get_length (token));
1213                         anjuta_token_insert_after (first, new);
1214                         anjuta_token_merge (first, new);
1215                 }
1216         }
1217 }
1218
1219
1220 AnjutaToken *
1221 anjuta_token_concat(AnjutaToken *token)
1222 {
1223         AnjutaToken *new;
1224
1225         new = anjuta_token_new_static (ANJUTA_TOKEN_CONTENT, NULL);
1226         anjuta_token_foreach_token (token, concat_token, new);
1227
1228         anjuta_token_insert_token_before (token, new);
1229         anjuta_token_free (token);
1230
1231         return new;
1232 }
1233
1234 /* Token foreach
1235  *---------------------------------------------------------------------------*/
1236
1237 void
1238 anjuta_token_foreach_token (AnjutaToken *token, AnjutaTokenForeachFunc func, gpointer user_data)
1239 {
1240         if (token != NULL)
1241         {
1242                 AnjutaToken *last_token;
1243                 gint child = 0;
1244
1245                 last_token = token->last == NULL ? token : token->last;
1246                 while (token != NULL)
1247                 {
1248                         if (child == 0) func (token, user_data);
1249
1250                         /* Check if we have found the last token */
1251                         if (token == last_token)
1252                         {
1253                                 /* Find last token */
1254                                 if (token->last == NULL)
1255                                 {
1256                                         break;
1257                                 }
1258                                 /* Last token still include additional tokens */
1259                                 last_token = token->last;
1260                         }
1261
1262                         if (token->children != NULL)
1263                         {
1264                                 /* Check children, only for last token */
1265                                 child++;
1266                                 token = token->children;
1267                         }
1268                         else if (token->next != NULL)
1269                         {
1270                                 /* Get next sibling */
1271                                 token = token->next;
1272                         }
1273                         else
1274                         {
1275                                 /* Get parent */
1276                                 for (;;)
1277                                 {
1278                                         child--;
1279                                         token = token->parent;
1280                                         if (token == NULL) break;
1281                                         if (token->next != NULL)
1282                                         {
1283                                                 token = token->next;
1284                                                 break;
1285                                         }
1286                                 }
1287                         }
1288                 }
1289         }
1290
1291         return;
1292 }
1293
1294 void
1295 anjuta_token_foreach_content (AnjutaToken *token, AnjutaTokenForeachFunc func, gpointer user_data)
1296 {
1297         if (token != NULL)
1298         {
1299                 AnjutaToken *last_parent;       /* If not NULL, token belong to a parent and
1300                                                                          * are not taken into account */
1301                 AnjutaToken *last_token;
1302                 gboolean expand = TRUE;
1303
1304                 last_parent = NULL;
1305                 last_token = token->last == NULL ? token : token->last;
1306                 for (;;)
1307                 {
1308                         if (expand && (token->children != NULL))
1309                         {
1310                                 /* Children of the last token does not belong to the group */
1311                                 if (token == last_token)
1312                                 {
1313                                         /* Find last token */
1314                                         if (token->last == NULL)
1315                                         {
1316                                                 break;
1317                                         }
1318                                         /* Last token still include additional tokens */
1319                                         last_token = token->last;
1320                                 }
1321
1322                                 /* Enumerate children */
1323                                 token = token->children;
1324                         }
1325                         else
1326                         {
1327                                 if (token->children == NULL)
1328                                 {
1329                                         /* Take into account only the content of token having no children */
1330                                         if (last_parent == NULL)
1331                                         {
1332                                                 /* Take into account only the content of group having no children */
1333                                                 func (token, user_data);
1334                                         }
1335                                 }
1336
1337                                 /* Check if we have found the last token */
1338                                 if (token == last_token)
1339                                 {
1340                                         /* Find last token */
1341                                         if (token->last == NULL)
1342                                         {
1343                                                 break;
1344                                         }
1345                                         /* Last token still include additional tokens */
1346                                         last_token = token->last;
1347                                 }
1348
1349                                 if (token == last_parent)
1350                                 {
1351                                         /* Find last parent */
1352                                         if (token->last == NULL)
1353                                         {
1354                                                 /* Found complete group having children */
1355                                                 last_parent = NULL;
1356                                         }
1357                                         else
1358                                         {
1359                                                 /* Parent group has additional token */
1360                                                 last_parent = token->last;
1361                                         }
1362                                 }
1363
1364                                 if (token->next != NULL)
1365                                 {
1366                                         /* Get next sibling */
1367                                         token = token->next;
1368                                         expand = TRUE;
1369                                 }
1370                                 else
1371                                 {
1372                                         /* Get parent */
1373                                         token = token->parent;
1374                                         if (token != NULL) last_parent = token->last;
1375                                         expand = FALSE;
1376                                 }
1377                         }
1378                 }
1379         }
1380
1381         return;
1382 }
1383
1384 void
1385 anjuta_token_foreach_container (AnjutaToken *token, AnjutaTokenForeachFunc func, gpointer user_data)
1386 {
1387         if (token != NULL)
1388         {
1389                 AnjutaToken *parent;    /* If not NULL, token belong to a parent and
1390                                                                          * are not taken into account */
1391                 AnjutaToken *last_token;
1392                 gboolean expand = TRUE;
1393
1394                 parent = NULL;
1395                 last_token = token->last == NULL ? token : token->last;
1396                 for (;;)
1397                 {
1398                         /* Take into account only the content of parent token */
1399                         if (expand && (parent == NULL))
1400                         {
1401                                 func (token, user_data);
1402                         }
1403                         if (expand && (token->children != NULL))
1404                         {
1405                                 if (parent == NULL) parent = token;
1406                                 /* Enumerate children */
1407                                 token = token->children;
1408                         }
1409                         else
1410                         {
1411                                 /* Check if we have found the last token */
1412                                 if (token == last_token)
1413                                 {
1414                                         /* Find last token */
1415                                         if (token->last == NULL)
1416                                         {
1417                                                 break;
1418                                         }
1419                                         /* Last token still include additional tokens */
1420                                         last_token = token->last;
1421                                 }
1422
1423                                 if (token->next != NULL)
1424                                 {
1425                                         /* Get next sibling */
1426                                         token = token->next;
1427                                         expand = TRUE;
1428                                 }
1429                                 else
1430                                 {
1431                                         /* Get parent */
1432                                         token = token->parent;
1433                                         if (token == parent) parent = NULL;
1434                                         expand = FALSE;
1435                                 }
1436                         }
1437                 }
1438         }
1439
1440         return;
1441 }
1442
1443 AnjutaToken *
1444 anjuta_token_foreach_post_order (AnjutaToken *token, AnjutaTokenForeachFunc func, gpointer user_data)
1445 {
1446         if (token != NULL)
1447         {
1448                 AnjutaToken *last_parent;       /* If not NULL, token belong to a parent */
1449                 AnjutaToken *last_token;
1450                 AnjutaToken buffer;                     /* Temporary token allowing func to destroy
1451                                                                          * the current token */
1452                 gboolean expand = TRUE;
1453
1454                 last_parent = NULL;
1455                 last_token = token->last == NULL ? token : token->last;
1456                 while (token != NULL)
1457                 {
1458                         if (expand && (token->children != NULL))
1459                         {
1460                                 /* Check if we have found the last token */
1461                                 if (token == last_token)
1462                                 {
1463                                         /* Find last token */
1464                                         if (token->last == NULL)
1465                                         {
1466                                                 break;
1467                                         }
1468                                         /* Last token still include additional tokens */
1469                                         last_token = token->last;
1470                                 }
1471
1472                                 /* Enumerate children */
1473                                 token = token->children;
1474                         }
1475                         else
1476                         {
1477                                 /* Save token data in case it is destroyed */
1478                                 memcpy (&buffer, token, sizeof (buffer));
1479                                 /* Take into account all token */
1480                                 func (token, user_data);
1481
1482                                 /* Check if we have found the last token */
1483                                 if (token == last_token)
1484                                 {
1485                                         /* Find last token */
1486                                         if (buffer.last == NULL)
1487                                         {
1488                                                 token = &buffer;
1489                                                 break;
1490                                         }
1491                                         /* Last token still include additional tokens */
1492                                         last_token = buffer.last;
1493                                 }
1494
1495                                 if (token == last_parent)
1496                                 {
1497                                         /* Find last parent */
1498                                         if (buffer.last == NULL)
1499                                         {
1500                                                 /* Found complete group having children */
1501                                                 last_parent = NULL;
1502                                         }
1503                                         else
1504                                         {
1505                                                 /* Parent group has additional token */
1506                                                 last_parent = buffer.last;
1507                                         }
1508                                 }
1509
1510                                 if (buffer.next != NULL)
1511                                 {
1512                                         /* Get next sibling */
1513                                         token = buffer.next;
1514                                         expand = TRUE;
1515                                 }
1516                                 else
1517                                 {
1518                                         /* Get parent */
1519                                         token = buffer.parent;
1520                                         if (token != NULL) last_parent = token->last;
1521                                         expand = FALSE;
1522                                 }
1523                         }
1524                 }
1525
1526                 while ((token != NULL) && (token->next == NULL))
1527                 {
1528                         token = token->parent;
1529                 };
1530
1531                 if (token != NULL) token = token->next;
1532         }
1533
1534         return token;
1535 }
1536
1537 /* Token evaluation
1538  *---------------------------------------------------------------------------*/
1539
1540 static void
1541 evaluate_raw_token (AnjutaToken *token, gpointer user_data)
1542 {
1543         GString *value = (GString *)user_data;
1544
1545         anjuta_token_evaluate_token (token, value, TRUE);
1546 }
1547
1548 static void
1549 evaluate_token (AnjutaToken *token, gpointer user_data)
1550 {
1551         GString *value = (GString *)user_data;
1552
1553         anjuta_token_evaluate_token (token, value, FALSE);
1554 }
1555
1556
1557 gchar *
1558 anjuta_token_evaluate (AnjutaToken *token)
1559 {
1560         GString *value = g_string_new (NULL);
1561
1562         anjuta_token_foreach_content (token, evaluate_token, value);
1563
1564         /* Return NULL and free data for an empty string */
1565         return g_string_free (value, *(value->str) == '\0');
1566 }
1567
1568 /* Does not evaluate content if token is a variable */
1569 gchar *
1570 anjuta_token_evaluate_name (AnjutaToken *token)
1571 {
1572         GString *value = g_string_new (NULL);
1573
1574         anjuta_token_foreach_container (token, evaluate_token, value);
1575
1576         /* Return NULL and free data for an empty string */
1577         return g_string_free (value, *(value->str) == '\0');
1578 }
1579
1580 gboolean
1581 anjuta_token_is_empty (AnjutaToken *token)
1582 {
1583         return (token == NULL) || ((token->data.length == 0) && (token->last == NULL) && (token->children == NULL));
1584 }
1585
1586
1587 /* Other functions
1588  *---------------------------------------------------------------------------*/
1589
1590 gboolean
1591 anjuta_token_compare (AnjutaToken *toka, AnjutaToken *tokb)
1592 {
1593         if (tokb->data.type)
1594         {
1595                 if (tokb->data.type != toka->data.type) return FALSE;
1596         }
1597
1598         if (tokb->data.type != ANJUTA_TOKEN_NONE)
1599         {
1600                 if (tokb->data.length != 0)
1601                 {
1602                         if (toka->data.length != tokb->data.length) return FALSE;
1603
1604                         if ((toka->data.flags & ANJUTA_TOKEN_CASE_INSENSITIVE)  && (tokb->data.flags & ANJUTA_TOKEN_CASE_INSENSITIVE))
1605                         {
1606                                 if (g_ascii_strncasecmp (toka->data.pos, tokb->data.pos, toka->data.length) != 0) return FALSE;
1607                         }
1608                         else
1609                         {
1610                                 if (strncmp (toka->data.pos, tokb->data.pos, toka->data.length) != 0) return FALSE;
1611                         }
1612                 }
1613         }
1614
1615         if (tokb->data.flags & ANJUTA_TOKEN_PUBLIC_FLAGS)
1616         {
1617                 if ((toka->data.flags & tokb->data.flags & ANJUTA_TOKEN_PUBLIC_FLAGS) == 0)
1618                         return FALSE;
1619         }
1620
1621         return TRUE;
1622 }
1623
1624 void
1625 anjuta_token_dump (AnjutaToken *token)
1626 {
1627         if (token == NULL) return;
1628
1629         anjuta_token_dump_child (token, 0, 0);
1630 }
1631
1632 void
1633 anjuta_token_dump_link (AnjutaToken *token)
1634 {
1635         AnjutaToken *last = token;
1636
1637         while (last->last != NULL) last = last->last;
1638
1639         for (; token != last; token = anjuta_token_next (token))
1640         {
1641                 anjuta_token_show (token, 0, 0);
1642         }
1643 }
1644
1645 gboolean
1646 anjuta_token_check (AnjutaToken *token)
1647 {
1648         if ((token->children != NULL) && (token->last != NULL))
1649         {
1650                 anjuta_token_show (token, 0, 0);
1651                 fprintf(stderr, "Error: Previous token has both non NULL children and last\n");
1652
1653                 return FALSE;
1654         }
1655
1656         if (token->children != NULL)
1657         {
1658                 AnjutaToken *child;
1659
1660                 for (child = token->children; child != NULL; child = child->next)
1661                 {
1662                         if (!anjuta_token_check_child (child, token)) return FALSE;
1663                 }
1664         }
1665
1666         if (token->last != NULL)
1667         {
1668                 AnjutaToken *child;
1669
1670                 for (child = anjuta_token_next (token); child != NULL; child = anjuta_token_next (child))
1671                 {
1672                         if (!anjuta_token_check (child)) return FALSE;
1673                         if (child == token->last) break;
1674                 }
1675         }
1676
1677         return TRUE;
1678 }
1679
1680 /* Constructor & Destructor
1681  *---------------------------------------------------------------------------*/
1682
1683 AnjutaToken *anjuta_token_new_string (AnjutaTokenType type, const gchar *value)
1684 {
1685         AnjutaToken *token;
1686
1687         if (value == NULL)
1688         {
1689                 token = anjuta_token_new_static (type, NULL);
1690         }
1691         else
1692         {
1693                 token = g_slice_new0 (AnjutaToken);
1694                 token->data.type = type  & ANJUTA_TOKEN_TYPE;
1695                 token->data.flags = type & ANJUTA_TOKEN_FLAGS;
1696                 token->data.pos = g_strdup (value);
1697                 token->data.length = strlen (value);
1698         }
1699
1700         return token;
1701 }
1702
1703 AnjutaToken*
1704 anjuta_token_new_string_len (AnjutaTokenType type, gchar *value, gsize length)
1705 {
1706         AnjutaToken *token;
1707
1708         if (value == NULL)
1709         {
1710                 token = anjuta_token_new_static (type, NULL);
1711         }
1712         else
1713         {
1714                 token = g_slice_new0 (AnjutaToken);
1715                 token->data.type = type  & ANJUTA_TOKEN_TYPE;
1716                 token->data.flags = type & ANJUTA_TOKEN_FLAGS;
1717                 token->data.pos = value;
1718                 token->data.length = length;
1719         }
1720
1721         return token;
1722 }
1723
1724 AnjutaToken *
1725 anjuta_token_new_static_len (gint type, const gchar *pos, gsize length)
1726 {
1727         AnjutaToken *token;
1728
1729         token = g_slice_new0 (AnjutaToken);
1730         token->data.type = type  & ANJUTA_TOKEN_TYPE;
1731         token->data.flags = (type & ANJUTA_TOKEN_FLAGS) | ANJUTA_TOKEN_STATIC;
1732         token->data.pos = (gchar *)pos;
1733         token->data.length = length;
1734
1735         return token;
1736 };
1737
1738 AnjutaToken *anjuta_token_new_static (AnjutaTokenType type, const char *value)
1739 {
1740         return anjuta_token_new_static_len (type, value, value == NULL ? 0 : strlen (value));
1741 }
1742
1743 AnjutaToken*
1744 anjuta_token_free_children (AnjutaToken *token)
1745 {
1746         AnjutaToken *child;
1747         AnjutaToken *last;
1748
1749         if (token == NULL) return NULL;
1750
1751         for (child = token->children; child != NULL; child = token->children)
1752         {
1753                 anjuta_token_free (child);
1754         }
1755         token->children = NULL;
1756
1757         if (token->last != NULL)
1758         {
1759                 last = token->last;
1760                 for (child = anjuta_token_next (token); child != NULL; child = anjuta_token_next (token))
1761                 {
1762                         anjuta_token_free (child);
1763                         if (child == last) break;
1764                 }
1765         }
1766         token->last = NULL;
1767
1768         return token;
1769 }
1770
1771 static void
1772 free_token (AnjutaToken *token, gpointer user_data)
1773 {
1774         anjuta_token_unlink_token (token);
1775         if ((token->data.pos != NULL) && !(token->data.flags & ANJUTA_TOKEN_STATIC))
1776         {
1777                 g_free (token->data.pos);
1778         }
1779         g_slice_free (AnjutaToken, token);
1780 }
1781
1782
1783 AnjutaToken*
1784 anjuta_token_free (AnjutaToken *token)
1785 {
1786         AnjutaToken *next;
1787
1788         if (token == NULL) return NULL;
1789
1790         next = anjuta_token_foreach_post_order (token, free_token, NULL);
1791
1792         return next;
1793 }