libanjuta/anjuta-token.c

   1 /* -*- Mode: C; indent-tabs-mode: t; c-basic-offset: 4; tab-width: 4 -*- */
   2 /*
   3  * anjuta-token.c
   4  * Copyright (C) Sébastien Granjoux 2009 <seb.sfo@free.fr>
   5  *
   6  * This program is free software: you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License as published by the
   8  * Free Software Foundation, either version 3 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful, but
  12  * WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  14  * See the GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License along
  17  * with this program.  If not, see <http://www.gnu.org/licenses/>.
  18  */
  19
  20 #include "anjuta-token.h"
  21
  22 #include "anjuta-debug.h"
  23
  24 #include <glib-object.h>
  25
  26 #include <stdio.h>
  27 #include <string.h>
  28 #include <stdlib.h>
  29
  30 /**
  31  * SECTION:anjuta-token
  32  * @title: Anjuta token
  33  * @short_description: Anjuta token
  34  * @see_also:
  35  * @stability: Unstable
  36  * @include: libanjuta/anjuta-token.h
  37  *
  38  * A #AnjutaToken represents a token. It is a sequence of characters associated
  39  * with a type representing its meaning. By example, a token can represent
  40  * a keyword, a comment, a variable...
  41  *
  42  * The token can own the string or has only a pointer on some data allocated
  43  * somewhere else with a length.
  44  *
  45  * A token is linked with other tokens using three double linked lists.
  46  *
  47  * The first list using next and prev fields is used to keep the token in the
  48  * order where there are in the file. The first character of the first token is
  49  * the first character in the file.
  50  *
  51  * A second list is used to represent included
  52  * files. Such file is represented by a special token in the first list which
  53  * has a pointer, named children to a token list. Each token in this secondary
  54  * list has a pointer to its parent, it means the token representing the file
  55  * where is the token. It looks like a tree. In fact, every file is represented
  56  * by a special token, so the root node is normally a file token and has as
  57  * children all the token representing the file content. This parent/child list
  58  * is used for expanded variable too.
  59  *
  60  * A third list is used to group several tokens. A token can have a pointer to
  61  * another last token. It means that this token is a group starting from this
  62  * token to the one indicated by the last field. In addition each token in this
  63  * group has a pointer on the first token of the group. This grouping is
  64  * independent of the parent/child list. So a group can start in one file and
  65  * end in another included file. The grouping can be nested too. Typically
  66  * we can have a group representing a command, a sub group representing the
  67  * arguments and then one sub group for each argument.
  68  */
  69
  70 /*
  71  * A token with its pointer can be put in one of the following case:
  72  *
  73  * * Simple token:
  74  *              children = NULL && last = NULL
  75  *              This is the case of a simple token without any children.
  76  *
  77  * * Composite token:
  78  *              children = NULL && last != NULL
  79  *              This is the case of a token grouping several other tokens, representing
  80  *              by example an item in a list.
  81  *
  82  * * Parent token:
  83  *              children != NULL && last == NULL
  84  *              This is the case of a token having children, by example a variable where
  85  *              the children represent the content. Most of the time the value of the
  86  *              parent token is ignored.
  87  *
  88  * * Composite parent token:
  89  *              children != NULL && last != NULL && (last in list)
  90  *              This case represents a variable which is split into several tokens. The
  91  *              children represents the content of the variable. The token up to last
  92  *              corresponds to the variable name. After getting the variable content, the
  93  *              next token is after the last token of the variable.
  94  *
  95  * * Composite parent token:
  96  *              children != NULL && last != NULL && (last in children)
  97  *              This case would represent a variable which is split into several token
  98  *              with one inside the children of the variable. This is not possible as
  99  *              the complete name of the variable is needed to get the children.
 100  *
 101  * * Composite parent token:
 102  *              children != NULL && last != NULL && (last in parent)
 103  *              This case represents a variable split into several token where the
 104  *              last element is a sibling of one parent of the token. I think this case
 105  *              is possible.
 106  *
 107  */
 108
 109 typedef struct _AnjutaTokenData AnjutaTokenData;
 110
 111 struct _AnjutaTokenData
 112 {
 113         AnjutaTokenType type;
 114         gint flags;
 115         gchar *pos;
 116         gsize length;
 117 };
 118
 119 struct _AnjutaToken
 120 {
 121         AnjutaToken     *next;
 122         AnjutaToken     *prev;
 123         AnjutaToken     *parent;
 124         AnjutaToken *last;
 125         AnjutaToken *group;
 126         AnjutaToken     *children;
 127         AnjutaTokenData data;
 128 };
 129
 130 /* Helpers functions
 131  *---------------------------------------------------------------------------*/
 132
 133 /* Private functions
 134  *---------------------------------------------------------------------------*/
 135
 136 static AnjutaToken *
 137 anjuta_token_next_child (AnjutaToken *child, AnjutaToken **last)
 138 {
 139         if (child == NULL) return child;
 140
 141         if (child->children != NULL)
 142         {
 143                 child = child->children;
 144         }
 145         else
 146         {
 147                 for (;;)
 148                 {
 149                         if ((*last == NULL) || (child == *last))
 150                         {
 151                                 if (child->last == NULL)
 152                                 {
 153                                         child = NULL;
 154                                         break;
 155                                 }
 156                                 *last = child->last;
 157                         }
 158                         if (child->next != NULL)
 159                         {
 160                                 child = child->next;
 161                                 break;
 162                         }
 163                         child = child->parent;
 164                         if (child == NULL) break;
 165                 }
 166         }
 167
 168         return child;
 169 }
 170
 171 AnjutaToken *
 172 anjuta_token_next_after_children (AnjutaToken *token)
 173 {
 174         while (token->last != NULL) token = token->last;
 175         while (token->next == NULL)
 176         {
 177                 token = token->parent;
 178                 if (token == NULL) return NULL;
 179                 while (token->last != NULL) token = token->last;
 180         };
 181
 182         return token->next;
 183 }
 184
 185 static AnjutaToken *
 186 anjuta_token_copy (AnjutaToken *token)
 187 {
 188         AnjutaToken *copy = NULL;
 189
 190         if (token != NULL)
 191         {
 192                 copy = g_slice_new0 (AnjutaToken);
 193                 copy->data.type = token->data.type;
 194                 copy->data.flags = token->data.flags;
 195                 if ((copy->data.flags & ANJUTA_TOKEN_STATIC) || (token->data.pos == NULL))
 196                 {
 197                         copy->data.pos = token->data.pos;
 198                 }
 199                 else
 200                 {
 201                         copy->data.pos = g_strdup (token->data.pos);
 202                 }
 203                 copy->data.length = token->data.length;
 204         }
 205
 206         return copy;
 207 }
 208
 209 /**
 210  * anjuta_token_unlink_token:
 211  * @token: a #AnjutaToken object.
 212  *
 213  * Unlink a single token, not the complete item, from the token tree.
 214  *
 215  * Return value: the removed token tree
 216  */
 217 static AnjutaToken *
 218 anjuta_token_unlink_token (AnjutaToken *token)
 219 {
 220
 221         if (token->prev != NULL)
 222         {
 223                 token->prev->next = token->next;
 224         }
 225         else if ((token->parent != NULL) && (token->parent->children == token))
 226         {
 227                 token->parent->children = token->next;
 228         }
 229         token->parent = NULL;
 230
 231         if ((token->group != NULL) && (token->group->last == token))
 232         {
 233                 AnjutaToken *prev;
 234
 235                 for (prev = token->prev; prev != NULL; prev = prev->prev)
 236                 {
 237                         if (prev->group == token->group)
 238                         {
 239                                 /* Find previous token in the same group */
 240                                 token->group->last = prev;
 241                                 break;
 242                         }
 243                         else if (prev == token->group)
 244                         {
 245                                 /* No more token in group */
 246                                 token->group->last = NULL;
 247                                 break;
 248                         }
 249                 }
 250         }
 251
 252         if (token->next != NULL)
 253         {
 254                 token->next->prev = token->prev;
 255                 token->next = NULL;
 256         }
 257         token->prev = NULL;
 258
 259         return token;
 260 }
 261
 262 /**
 263  * anjuta_token_insert_token_before:
 264  * @sibling: a #AnjutaToken object.
 265  * @token: a #AnjutaToken object.
 266  *
 267  * Insert token before sibling.
 268  *
 269  * Return value: inserted token
 270  */
 271 static AnjutaToken *
 272 anjuta_token_insert_token_before (AnjutaToken *sibling, AnjutaToken *token)
 273 {
 274         token->prev = sibling->prev;
 275         token->next = sibling;
 276
 277         if (token->prev != NULL)
 278         {
 279                 token->prev->next = token;
 280         }
 281         sibling->prev = token;
 282
 283         if ((sibling->parent != NULL) && (sibling->parent->children == sibling))
 284         {
 285                 sibling->parent->children = token;
 286         }
 287         token->parent = sibling->parent;
 288
 289         return token;
 290 }
 291
 292 static void
 293 anjuta_token_evaluate_token (AnjutaToken *token, GString *value, gboolean raw)
 294 {
 295         if ((token != NULL) && (token->data.length != 0))
 296         {
 297                 if (!raw)
 298                 {
 299                         switch (anjuta_token_get_type (token))
 300                         {
 301                         case ANJUTA_TOKEN_COMMENT:
 302                         case ANJUTA_TOKEN_OPEN_QUOTE:
 303                         case ANJUTA_TOKEN_CLOSE_QUOTE:
 304                         case ANJUTA_TOKEN_ESCAPE:
 305                         case ANJUTA_TOKEN_MACRO:
 306                         case ANJUTA_TOKEN_EOV:
 307                                 return;
 308                         default:
 309                                 break;
 310                         }
 311                 }
 312                 g_string_append_len (value, anjuta_token_get_string (token), anjuta_token_get_length (token));
 313         }
 314 }
 315
 316 static void
 317 anjuta_token_show (AnjutaToken *token, gint indent, gchar parent)
 318 {
 319         static gchar type[] = "\0";
 320         const gchar *string;
 321         gsize length;
 322
 323         type[0] = parent;
 324         fprintf (stderr, "%*s%s %p", indent, "", type, token);
 325         fprintf (stderr, ": %d ",
 326             anjuta_token_get_type (token));
 327         string = anjuta_token_get_string (token);
 328         length = anjuta_token_get_length (token);
 329         if (string == NULL)
 330         {
 331                 /* Value doesn't contain a newline */
 332                 fprintf (stderr, "(%" G_GSIZE_FORMAT ")", length);
 333         }
 334         else
 335         {
 336                 const gchar *newline;
 337
 338                 newline = g_strrstr_len (string, length, "\n");
 339                 if (newline == NULL)
 340                 {
 341                         /* Value doesn't contain a newline */
 342                         fprintf (stderr, "\"%.*s\"",
 343                                 (int)length,
 344                         string);
 345                 }
 346                 else
 347                 {
 348                         /* Value contains a newline, take care of indentation */
 349                         newline++;
 350                         fprintf (stderr, "\"%.*s",
 351                         (int)(newline - string),
 352                         string);
 353                         for (;;)
 354                         {
 355                                 length -= newline - string;
 356                                 string = newline;
 357
 358                                 newline = g_strrstr_len (string, length, "\n");
 359                                 if (newline == NULL) break;
 360
 361                                 newline++;
 362                                 fprintf (stderr, "%*s  %.*s",
 363                                     indent, "",
 364                                         (int)(newline - string),
 365                                 string);
 366                         }
 367                         fprintf (stderr, "%*s  %.*s\"",
 368                                     indent, "",
 369                                         (int)length,
 370                                 string);
 371                 }
 372         }
 373         fprintf (stderr, " %p/%p (%p/%p) %s\n",
 374                 token->last, token->children,
 375                 token->group, token->parent,
 376                 anjuta_token_get_flags (token) & ANJUTA_TOKEN_REMOVED ? " (removed)" : "");
 377 }
 378
 379 static AnjutaToken*
 380 anjuta_token_dump_child (AnjutaToken *token, gint indent, gchar type)
 381 {
 382         AnjutaToken *last;
 383         AnjutaToken *child;
 384
 385         anjuta_token_show (token, indent, type);
 386         indent += 4;
 387
 388         last = token;
 389         if (token->last != NULL)
 390         {
 391                 do
 392                 {
 393                         child = last->next;
 394                         if (child == NULL) break;
 395                         last = anjuta_token_dump_child (child, indent, '+');
 396                 }
 397                 while (child != token->last);
 398         }
 399
 400         if (token->children != NULL)
 401         {
 402                 for (child = token->children; child != NULL; child = child->next)
 403                 {
 404                         child = anjuta_token_dump_child (child, indent, '*');
 405                 }
 406         }
 407
 408         return last;
 409 }
 410
 411 static gboolean
 412 anjuta_token_check_child (AnjutaToken *token, AnjutaToken *parent)
 413 {
 414         if (token->parent != parent)
 415         {
 416                 anjuta_token_show (token, 0, 0);
 417                 fprintf(stderr, "Error: Children has %p as parent instead of %p\n", token->parent, parent);
 418                 return FALSE;
 419         }
 420
 421         return anjuta_token_check (token);
 422 }
 423
 424 /* Get and set functions
 425  *---------------------------------------------------------------------------*/
 426
 427 void
 428 anjuta_token_set_type (AnjutaToken *token, gint type)
 429 {
 430         token->data.type = type;
 431 }
 432
 433 gint
 434 anjuta_token_get_type (AnjutaToken *token)
 435 {
 436         return token->data.type;
 437 }
 438
 439 void
 440 anjuta_token_set_flags (AnjutaToken *token, gint flags)
 441 {
 442         AnjutaToken *child;
 443         AnjutaToken *last = token->last;
 444
 445         for (child = token; child != NULL; child = anjuta_token_next_child (child, &last))
 446         {
 447                 child->data.flags |= flags;
 448         }
 449 }
 450
 451 void
 452 anjuta_token_clear_flags (AnjutaToken *token, gint flags)
 453 {
 454         token->data.flags &= ~flags;
 455 }
 456
 457 gint
 458 anjuta_token_get_flags (AnjutaToken *token)
 459 {
 460         return token->data.flags;
 461 }
 462
 463 void
 464 anjuta_token_set_string (AnjutaToken *token, const gchar *data, gsize length)
 465 {
 466         if (!(token->data.flags & ANJUTA_TOKEN_STATIC))
 467         {
 468                 g_free (token->data.pos);
 469                 token->data.flags |= ANJUTA_TOKEN_STATIC;
 470         }
 471         token->data.pos = (gchar *)data;
 472         token->data.length = length;
 473 }
 474
 475 const gchar *
 476 anjuta_token_get_string (AnjutaToken *token)
 477 {
 478         return token->data.pos;
 479 }
 480
 481 void
 482 anjuta_token_set_length (AnjutaToken *token, gsize length)
 483 {
 484         token->data.length = length;
 485 }
 486
 487 gsize
 488 anjuta_token_get_length (AnjutaToken *token)
 489 {
 490         return token->data.length;
 491 }
 492
 493 /* Basic move functions
 494  *---------------------------------------------------------------------------*/
 495
 496 AnjutaToken *
 497 anjuta_token_next (AnjutaToken *token)
 498 {
 499         if (token->children != NULL)
 500         {
 501                 return token->children;
 502         }
 503         else if (token->next != NULL)
 504         {
 505                 return token->next;
 506         }
 507         else if (token->parent != NULL)
 508         {
 509                 return anjuta_token_next_after_children (token->parent);
 510         }
 511         else
 512         {
 513                 return NULL;
 514         }
 515 }
 516
 517 AnjutaToken *
 518 anjuta_token_previous (AnjutaToken *token)
 519 {
 520         if (token->prev != NULL)
 521         {
 522                 return token->prev;
 523         }
 524         else
 525         {
 526                 return token->parent;
 527         }
 528 }
 529
 530 AnjutaToken *
 531 anjuta_token_last (AnjutaToken *token)
 532 {
 533         AnjutaToken *last;
 534
 535         for (last = token; last->last != NULL; last = last->last);
 536         if (last->children != NULL)
 537         {
 538                 for (last = last->children; last->next != NULL; last = last->next);
 539         }
 540
 541         return last;
 542 }
 543
 544 AnjutaToken *
 545 anjuta_token_parent (AnjutaToken *token)
 546 {
 547         return token->parent;
 548 }
 549
 550 AnjutaToken *
 551 anjuta_token_list (AnjutaToken *token)
 552 {
 553         return token->group;
 554 }
 555
 556 /* Item move functions
 557  *---------------------------------------------------------------------------*/
 558
 559 AnjutaToken *
 560 anjuta_token_last_item (AnjutaToken *list)
 561 {
 562         return list->last;
 563 }
 564
 565 AnjutaToken *
 566 anjuta_token_first_item (AnjutaToken *list)
 567 {
 568         AnjutaToken *first = NULL;
 569
 570         if (list != NULL)
 571         {
 572                 if (list->children != NULL)
 573                 {
 574                         first = list->children;
 575                 }
 576                 else if (list->last != NULL)
 577                 {
 578                         first = list->next;
 579                 }
 580         }
 581
 582         return first;
 583 }
 584
 585 AnjutaToken *
 586 anjuta_token_next_item (AnjutaToken *item)
 587 {
 588         AnjutaToken *next;
 589
 590         if (item != NULL)
 591         {
 592                 do
 593                 {
 594                         next = NULL;
 595                         if ((item->group == NULL) || (item->group->last != item))
 596                         {
 597                                 AnjutaToken *last;
 598                                 for (last = item; last->last != NULL; last = last->last);
 599                                 next = anjuta_token_next (last);
 600                                 if ((next != NULL) && (next->group != item->group)) next = NULL;
 601                         }
 602                         item = next;
 603                 }
 604                 /* Loop if the current item has been deleted */
 605                 while ((next != NULL) && (anjuta_token_get_flags (next) & ANJUTA_TOKEN_REMOVED));
 606         }
 607
 608         return next;
 609 }
 610
 611 AnjutaToken *
 612 anjuta_token_previous_item (AnjutaToken *item)
 613 {
 614         AnjutaToken *prev = NULL;
 615
 616
 617         if (item != NULL)
 618         {
 619                 do
 620                 {
 621                         for (prev = item->prev; (prev != NULL) && (prev->group != item->group); prev = prev->group);
 622                         item = prev;
 623                 }
 624                 /* Loop if the current item has been deleted */
 625                 while ((prev != NULL) && (anjuta_token_get_flags (prev) & ANJUTA_TOKEN_REMOVED));
 626         }
 627
 628         return prev;
 629 }
 630
 631 /* Add/Insert/Remove tokens
 632  *---------------------------------------------------------------------------*/
 633
 634 /**
 635  * anjuta_token_append_child:
 636  * @parent: a #AnjutaToken object used as parent.
 637  * @children: a #AnjutaToken object.
 638  *
 639  * Insert all tokens in children as the last children of the given parent.
 640  *
 641  * Return value: The first token append.
 642  */
 643 AnjutaToken *
 644 anjuta_token_append_child (AnjutaToken *parent, AnjutaToken *children)
 645 {
 646         AnjutaToken *token;
 647         AnjutaToken *last;
 648         AnjutaToken *old_group;
 649         AnjutaToken *old_parent;
 650
 651         g_return_val_if_fail (parent != NULL, NULL);
 652         g_return_val_if_fail (children != NULL, NULL);
 653
 654         old_group = children->group;
 655         old_parent = children->parent;
 656
 657         if (parent->children == NULL)
 658         {
 659                 parent->children = children;
 660
 661                 children->prev = NULL;
 662         }
 663         else
 664         {
 665                 /* Find last children */
 666                 for (last = parent->children; last->next != NULL;)
 667                 {
 668                         if ((last->last != NULL) && (last->last->parent == last->parent))
 669                         {
 670                                 last = last->last;
 671                         }
 672                         else
 673                         {
 674                                 last = last->next;
 675                         }
 676                 }
 677
 678                 last->next = children;
 679                 children->prev = last;
 680         }
 681
 682         /* Update each token */
 683         for (token = children;;)
 684         {
 685                 if (token->parent == old_parent) token->parent = parent;
 686                 if (token->group == old_group) token->group = parent->group;
 687
 688                 if (token->children != NULL)
 689                 {
 690                         token = token->children;
 691                 }
 692                 else if (token->next != NULL)
 693                 {
 694                         token = token->next;
 695                 }
 696                 else
 697                 {
 698                         while (token->parent != parent)
 699                         {
 700                                 token = token->parent;
 701                                 if (token->next != NULL) break;
 702                         }
 703                         if (token->next == NULL) break;
 704                         token = token->next;
 705                 }
 706         }
 707
 708         return children;
 709 }
 710
 711 /**
 712  * anjuta_token_prepend_child:
 713  * @parent: a #AnjutaToken object used as parent.
 714  * @children: a #AnjutaToken object.
 715  *
 716  * Insert all tokens in children as the first children of the given parent.
 717  *
 718  * Return value: The first token append.
 719  */
 720 AnjutaToken *
 721 anjuta_token_prepend_child (AnjutaToken *parent, AnjutaToken *children)
 722 {
 723         AnjutaToken *child;
 724         AnjutaToken *last = NULL;
 725
 726         g_return_val_if_fail (parent != NULL, NULL);
 727         g_return_val_if_fail (children != NULL, NULL);
 728
 729         /* Update each token */
 730         for (child = children;;)
 731         {
 732                 AnjutaToken *next;
 733
 734                 if (child->parent == children->parent) child->parent = parent;
 735                 if (child->group == children->group) child->group = parent->group;
 736
 737                 next = anjuta_token_next_child (child, &last);
 738                 if (next == NULL) break;
 739                 child = next;
 740         }
 741
 742         child->next = parent->children;
 743         if (child->next) child->next->prev = child;
 744         parent->children = children;
 745
 746         return children;
 747 }
 748
 749 /**
 750  * anjuta_token_prepend_items:
 751  * @list: a #AnjutaToken object used as list.
 752  * @item: a #AnjutaToken object.
 753  *
 754  * Insert all tokens in item as item of the given list.
 755  *
 756  * Return value: The first token append.
 757  */
 758 AnjutaToken *
 759 anjuta_token_prepend_items (AnjutaToken *list, AnjutaToken *item)
 760 {
 761         AnjutaToken *token;
 762         AnjutaToken *old_group;
 763         AnjutaToken *old_parent;
 764
 765         g_return_val_if_fail (list != NULL, NULL);
 766         g_return_val_if_fail (item != NULL, NULL);
 767
 768         old_group = item->group;
 769         old_parent = item->parent;
 770
 771         /* Update each token */
 772         for (token = item;;)
 773         {
 774                 if (token->parent == old_parent) token->parent = list->parent;
 775                 if (token->group == old_group) token->group = list;
 776
 777                 if (token->children != NULL)
 778                 {
 779                         token = token->children;
 780                 }
 781                 else if (token->next != NULL)
 782                 {
 783                         token = token->next;
 784                 }
 785                 else
 786                 {
 787                         while (token->parent != list->parent)
 788                         {
 789                                 token = token->parent;
 790                                 if (token->next != NULL) break;
 791                         }
 792                         if (token->next == NULL) break;
 793                         token = token->next;
 794                 }
 795         }
 796
 797         token->next = list->next;
 798         if (token->next) token->next->prev = token;
 799
 800         list->next = item;
 801         item->prev = list;
 802
 803         if (list->last == NULL)
 804         {
 805                 while (token->group != list) token = token->group;
 806                 list->last = token;
 807         }
 808
 809         return item;
 810 }
 811
 812 /**
 813  * anjuta_token_insert_after:
 814  * @sibling: a #AnjutaToken object.
 815  * @list: a #AnjutaToken object.
 816  *
 817  * Insert all tokens after sibling.
 818  *
 819  * Return value: The first token inserted.
 820  */
 821 AnjutaToken *
 822 anjuta_token_insert_after (AnjutaToken *sibling, AnjutaToken *list)
 823 {
 824         AnjutaToken *last;
 825         AnjutaToken *token;
 826         AnjutaToken *old_group;
 827         AnjutaToken *old_parent;
 828
 829         g_return_val_if_fail (sibling != NULL, NULL);
 830         g_return_val_if_fail (list != NULL, NULL);
 831
 832         old_group = list->group;
 833         old_parent = list->parent;
 834
 835         /* Update each token */
 836         for (token = list;;)
 837         {
 838                 if (token->parent == old_parent) token->parent = sibling->parent;
 839                 if (token->group == old_group) token->group = sibling->group;
 840
 841                 if (token->children != NULL)
 842                 {
 843                         token = token->children;
 844                 }
 845                 else if (token->next != NULL)
 846                 {
 847                         token = token->next;
 848                 }
 849                 else
 850                 {
 851                         while (token->parent != sibling->parent)
 852                         {
 853                                 token = token->parent;
 854                                 if (token->next != NULL) break;
 855                         }
 856                         if (token->next == NULL) break;
 857                         token = token->next;
 858                 }
 859         }
 860
 861         for (last = sibling; last->last != NULL; last = last->last);
 862
 863         token->next = last->next;
 864         if (token->next) token->next->prev = token;
 865
 866         last->next = list;
 867         list->prev = last;
 868
 869         if ((sibling->group != NULL) && (sibling->group->last == sibling))
 870         {
 871                 while (token->group != sibling->group) token = token->group;
 872                 sibling->group->last = token;
 873         }
 874
 875         return list;
 876 }
 877
 878 /**
 879  * anjuta_token_insert_before:
 880  * @sibling: a #AnjutaToken object.
 881  * @list: a #AnjutaToken object.
 882  *
 883  * Insert all tokens before sibling.
 884  *
 885  * Return value: The first token inserted.
 886  */
 887 AnjutaToken *
 888 anjuta_token_insert_before (AnjutaToken *sibling, AnjutaToken *list)
 889 {
 890         AnjutaToken *last;
 891         AnjutaToken *token;
 892         AnjutaToken *old_group;
 893         AnjutaToken *old_parent;
 894
 895         g_return_val_if_fail (sibling != NULL, NULL);
 896         g_return_val_if_fail (list != NULL, NULL);
 897
 898         old_group = list->group;
 899         old_parent = list->parent;
 900
 901         /* Update each token */
 902         for (token = list;;)
 903         {
 904                 if (token->parent == old_parent) token->parent = sibling->parent;
 905                 if (token->group == old_group) token->group = sibling->group;
 906
 907                 if (token->children != NULL)
 908                 {
 909                         token = token->children;
 910                 }
 911                 else if (token->next != NULL)
 912                 {
 913                         token = token->next;
 914                 }
 915                 else
 916                 {
 917                         while (token->parent != sibling->parent)
 918                         {
 919                                 token = token->parent;
 920                                 if (token->next != NULL) break;
 921                         }
 922                         if (token->next == NULL) break;
 923                         token = token->next;
 924                 }
 925         }
 926
 927         for (last = sibling; last->last != NULL; last = last->last);
 928
 929         token->next = sibling;
 930         list->prev = sibling->prev;
 931         sibling->prev = token;
 932
 933         if (list->prev) list->prev->next = list;
 934
 935         if ((list->parent != NULL) && (list->parent->children == sibling)) list->parent->children = list;
 936
 937         return list;
 938 }
 939
 940 /**
 941  * anjuta_token_delete_parent:
 942  * @parent: a #AnjutaToken object used as parent.
 943  *
 944  * Delete only the parent token.
 945  *
 946  * Return value: the first children
 947  */
 948 AnjutaToken *
 949 anjuta_token_delete_parent (AnjutaToken *parent)
 950 {
 951         AnjutaToken *token;
 952
 953         g_return_val_if_fail (parent != NULL, NULL);
 954
 955         if (parent->children == NULL) return NULL;
 956
 957         /* Update each token */
 958         for (token = parent->children;;)
 959         {
 960                 if (token->parent == parent) token->parent = parent->parent;
 961
 962                 if (token->children != NULL)
 963                 {
 964                         token = token->children;
 965                 }
 966                 else if (token->next != NULL)
 967                 {
 968                         token = token->next;
 969                 }
 970                 else
 971                 {
 972                         while (token->parent != parent->parent)
 973                         {
 974                                 token = token->parent;
 975                                 if (token->next != NULL) break;
 976                         }
 977                         if (token->next == NULL) break;
 978                         token = token->next;
 979                 }
 980         }
 981
 982         token->next = parent->next;
 983         if (token->next) token->next->prev = token;
 984
 985         parent->next = parent->children;
 986         parent->children->prev = parent;
 987         parent->children = NULL;
 988
 989         return anjuta_token_free (parent);
 990 }
 991
 992 /* Merge function
 993  *---------------------------------------------------------------------------*/
 994
 995 /* anjuta_token_merge can be used with first or end being a floating token and
 996  * on already grouped tokens to change the group organisation */
 997 AnjutaToken *
 998 anjuta_token_merge (AnjutaToken *first, AnjutaToken *end)
 999 {
1000         AnjutaToken *next;
1001
1002         if ((first == end) || (end == NULL)) return first;
1003
1004         /* Insert first or end in the same sequence if it is not already the case */
1005         for (next = first; next != end; next = anjuta_token_next (next))
1006         {
1007                 if (next == NULL)
1008                 {
1009                         if (first->parent == NULL)
1010                         {
1011                                 anjuta_token_insert_before (end, first);
1012                         }
1013                         else
1014                         {
1015                                 anjuta_token_insert_after (first, end);
1016                         }
1017                         break;
1018                 }
1019         }
1020         first->last = end;
1021         if ((end->group != NULL) && (end->group != first) && (end->group->last == end)) end->group->last = first;
1022         end->group = first;
1023
1024         return first;
1025 }
1026
1027 AnjutaToken *
1028 anjuta_token_merge_own_children (AnjutaToken *group)
1029 {
1030         AnjutaToken *token;
1031         AnjutaToken *next = NULL;
1032
1033         if (group->last != NULL) return group;
1034
1035         if (group->last->last != NULL) group->last = group->last->last;
1036
1037         for (token = anjuta_token_next (group); (token != NULL) && (token != group->last); token = anjuta_token_next (token))
1038         {
1039                 if (next == NULL)
1040                 {
1041                         if (token->last != NULL)
1042                         {
1043                                 next = token->last;
1044                                 //token->last = NULL;
1045                         }
1046                         token->group = group;
1047                 }
1048                 else if (next == token)
1049                 {
1050                         next = NULL;
1051                 }
1052         }
1053
1054         return group;
1055 }
1056
1057 AnjutaToken *
1058 anjuta_token_merge_children (AnjutaToken *first, AnjutaToken *end)
1059 {
1060         if (first == NULL)
1061         {
1062                 return end;
1063         }
1064         if ((first == end) || (end == NULL))
1065         {
1066                 return first;
1067         }
1068
1069         if (first->parent == NULL)
1070         {
1071                 first->parent = end->parent;
1072         }
1073         if (first->next == NULL)
1074         {
1075                 anjuta_token_insert_before (end, first);
1076         }
1077         anjuta_token_unlink_token (end);
1078         if (end->last != NULL)
1079         {
1080                 AnjutaToken *child;
1081
1082                 first->last = end->last;
1083                 for (child = anjuta_token_next (first); child != first->last; child = anjuta_token_next (child))
1084                 {
1085                         if (child->group == end) child->group = first;
1086                 }
1087                 first->last->group = first;
1088         }
1089         end->group = first;
1090         anjuta_token_free (end);
1091
1092         return first;
1093 }
1094
1095 /**
1096  * anjuta_token_merge_previous:
1097  * @list: a #AnjutaToken object representing a list
1098  * @first: a #AnjutaToken object for the new beginning of the list
1099  *
1100  * If the list token is not already linked with first, it is inserted
1101  * just before first.
1102  * If the list token is already linked, it must be in the same list after
1103  * first token. It it possible to have several tokens beweent list and
1104  * first.
1105  *
1106  * Return value: the new list
1107  */
1108 AnjutaToken *
1109 anjuta_token_merge_previous (AnjutaToken *list, AnjutaToken *first)
1110 {
1111         AnjutaToken *token;
1112
1113         if ((first == NULL) || (list == first)) return list;
1114
1115         /* Change group of all tokens from end to first
1116          * if the list is already linked */
1117         if ((list->prev != NULL) || (list->parent != NULL))
1118         {
1119                 for (token = first; token != NULL; token = anjuta_token_next_item (token))
1120                 {
1121                         token->group = list;
1122                 }
1123         }
1124
1125         token = anjuta_token_next (list);
1126         anjuta_token_unlink_token (list);
1127         anjuta_token_insert_token_before (first, list);
1128
1129         return list;
1130 }
1131
1132 AnjutaToken *anjuta_token_split (AnjutaToken *token, guint size)
1133 {
1134         if (token->data.length > size)
1135         {
1136                 AnjutaToken *copy;
1137
1138                 copy = anjuta_token_copy (token);
1139                 anjuta_token_insert_before (token, copy);
1140
1141                 copy->data.length = size;
1142                 if (token->data.flags & ANJUTA_TOKEN_STATIC)
1143                 {
1144                         token->data.pos += size;
1145                         token->data.length -= size;
1146                 }
1147                 else
1148                 {
1149                         memcpy(token->data.pos, token->data.pos + size, token->data.length - size);
1150                 }
1151
1152                 return copy;
1153         }
1154         else
1155         {
1156                 return token;
1157         }
1158 }
1159
1160 AnjutaToken *
1161 anjuta_token_cut (AnjutaToken *token, guint pos, guint size)
1162 {
1163         AnjutaToken *copy;
1164
1165         copy = anjuta_token_copy (token);
1166
1167         if (pos >= token->data.length)
1168         {
1169                 if (!(copy->data.flags & ANJUTA_TOKEN_STATIC))
1170                 {
1171                         g_free (copy->data.pos);
1172                 }
1173                 copy->data.pos = NULL;
1174                 copy->data.length = 0;
1175         }
1176         if ((pos + size) > token->data.length)
1177         {
1178                 size = token->data.length - pos;
1179         }
1180
1181         if (copy->data.flags & ANJUTA_TOKEN_STATIC)
1182         {
1183                 copy->data.pos += pos;
1184         }
1185         else
1186         {
1187                 memmove(copy->data.pos, copy->data.pos + pos, size);
1188         }
1189         copy->data.length = size;
1190
1191         return copy;
1192 }
1193
1194 static void
1195 concat_token (AnjutaToken *token, gpointer user_data)
1196 {
1197         AnjutaToken *first = (AnjutaToken *)user_data;
1198
1199         if (anjuta_token_get_length (token) > 0)
1200         {
1201                 if (anjuta_token_get_string (first) == NULL)
1202                 {
1203                         anjuta_token_set_string (first, anjuta_token_get_string (token), anjuta_token_get_length (token));
1204                 }
1205                 else if ((first != NULL) && ((anjuta_token_get_string(first) + anjuta_token_get_length (first)) == anjuta_token_get_string (token)))
1206                 {
1207                         anjuta_token_set_string (first, anjuta_token_get_string (first), anjuta_token_get_length (first) + anjuta_token_get_length (token));
1208                 }
1209                 else
1210                 {
1211                         AnjutaToken *new;
1212
1213                         new = anjuta_token_new_static_len (ANJUTA_TOKEN_CONTENT, anjuta_token_get_string (token), anjuta_token_get_length (token));
1214                         anjuta_token_insert_after (first, new);
1215                         anjuta_token_merge (first, new);
1216                 }
1217         }
1218 }
1219
1220
1221 AnjutaToken *
1222 anjuta_token_concat(AnjutaToken *token)
1223 {
1224         AnjutaToken *new;
1225
1226         new = anjuta_token_new_static (ANJUTA_TOKEN_CONTENT, NULL);
1227         anjuta_token_foreach_token (token, concat_token, new);
1228
1229         anjuta_token_insert_token_before (token, new);
1230         anjuta_token_free (token);
1231
1232         return new;
1233 }
1234
1235 /* Token foreach
1236  *---------------------------------------------------------------------------*/
1237
1238 void
1239 anjuta_token_foreach_token (AnjutaToken *token, AnjutaTokenForeachFunc func, gpointer user_data)
1240 {
1241         if (token != NULL)
1242         {
1243                 AnjutaToken *last_token;
1244                 gint child = 0;
1245
1246                 last_token = token->last == NULL ? token : token->last;
1247                 while (token != NULL)
1248                 {
1249                         if (child == 0) func (token, user_data);
1250
1251                         /* Check if we have found the last token */
1252                         if (token == last_token)
1253                         {
1254                                 /* Find last token */
1255                                 if (token->last == NULL)
1256                                 {
1257                                         break;
1258                                 }
1259                                 /* Last token still include additional tokens */
1260                                 last_token = token->last;
1261                         }
1262
1263                         if (token->children != NULL)
1264                         {
1265                                 /* Check children, only for last token */
1266                                 child++;
1267                                 token = token->children;
1268                         }
1269                         else if (token->next != NULL)
1270                         {
1271                                 /* Get next sibling */
1272                                 token = token->next;
1273                         }
1274                         else
1275                         {
1276                                 /* Get parent */
1277                                 for (;;)
1278                                 {
1279                                         child--;
1280                                         token = token->parent;
1281                                         if (token == NULL) break;
1282                                         if (token->next != NULL)
1283                                         {
1284                                                 token = token->next;
1285                                                 break;
1286                                         }
1287                                 }
1288                         }
1289                 }
1290         }
1291
1292         return;
1293 }
1294
1295 void
1296 anjuta_token_foreach_content (AnjutaToken *token, AnjutaTokenForeachFunc func, gpointer user_data)
1297 {
1298         if (token != NULL)
1299         {
1300                 AnjutaToken *last_parent;       /* If not NULL, token belong to a parent and
1301                                                                          * are not taken into account */
1302                 AnjutaToken *last_token;
1303                 gboolean expand = TRUE;
1304
1305                 last_parent = NULL;
1306                 last_token = token->last == NULL ? token : token->last;
1307                 for (;;)
1308                 {
1309                         if (expand && (token->children != NULL))
1310                         {
1311                                 /* Children of the last token does not belong to the group */
1312                                 if (token == last_token)
1313                                 {
1314                                         /* Find last token */
1315                                         if (token->last == NULL)
1316                                         {
1317                                                 break;
1318                                         }
1319                                         /* Last token still include additional tokens */
1320                                         last_token = token->last;
1321                                 }
1322
1323                                 /* Enumerate children */
1324                                 token = token->children;
1325                         }
1326                         else
1327                         {
1328                                 if (token->children == NULL)
1329                                 {
1330                                         /* Take into account only the content of token having no children */
1331                                         if (last_parent == NULL)
1332                                         {
1333                                                 /* Take into account only the content of group having no children */
1334                                                 func (token, user_data);
1335                                         }
1336                                 }
1337
1338                                 /* Check if we have found the last token */
1339                                 if (token == last_token)
1340                                 {
1341                                         /* Find last token */
1342                                         if (token->last == NULL)
1343                                         {
1344                                                 break;
1345                                         }
1346                                         /* Last token still include additional tokens */
1347                                         last_token = token->last;
1348                                 }
1349
1350                                 if (token == last_parent)
1351                                 {
1352                                         /* Find last parent */
1353                                         if (token->last == NULL)
1354                                         {
1355                                                 /* Found complete group having children */
1356                                                 last_parent = NULL;
1357                                         }
1358                                         else
1359                                         {
1360                                                 /* Parent group has additional token */
1361                                                 last_parent = token->last;
1362                                         }
1363                                 }
1364
1365                                 if (token->next != NULL)
1366                                 {
1367                                         /* Get next sibling */
1368                                         token = token->next;
1369                                         expand = TRUE;
1370                                 }
1371                                 else
1372                                 {
1373                                         /* Get parent */
1374                                         token = token->parent;
1375                                         expand = FALSE;
1376                                         if (token == NULL) break;
1377                                         last_parent = token->last;
1378                                 }
1379                         }
1380                 }
1381         }
1382
1383         return;
1384 }
1385
1386 static void
1387 anjuta_token_foreach_container (AnjutaToken *token, AnjutaTokenForeachFunc func, gpointer user_data)
1388 {
1389         if (token != NULL)
1390         {
1391                 AnjutaToken *parent;    /* If not NULL, token belong to a parent and
1392                                                                          * are not taken into account */
1393                 AnjutaToken *last_token;
1394                 gboolean expand = TRUE;
1395
1396                 parent = NULL;
1397                 last_token = token->last == NULL ? token : token->last;
1398                 for (;;)
1399                 {
1400                         /* Take into account only the content of parent token */
1401                         if (expand && (parent == NULL))
1402                         {
1403                                 func (token, user_data);
1404                         }
1405                         if (expand && (token->children != NULL))
1406                         {
1407                                 if (parent == NULL) parent = token;
1408                                 /* Enumerate children */
1409                                 token = token->children;
1410                         }
1411                         else
1412                         {
1413                                 /* Check if we have found the last token */
1414                                 if (token == last_token)
1415                                 {
1416                                         /* Find last token */
1417                                         if (token->last == NULL)
1418                                         {
1419                                                 break;
1420                                         }
1421                                         /* Last token still include additional tokens */
1422                                         last_token = token->last;
1423                                 }
1424
1425                                 if (token->next != NULL)
1426                                 {
1427                                         /* Get next sibling */
1428                                         token = token->next;
1429                                         expand = TRUE;
1430                                 }
1431                                 else
1432                                 {
1433                                         /* Get parent */
1434                                         token = token->parent;
1435                                         if (token == parent) parent = NULL;
1436                                         expand = FALSE;
1437                                 }
1438                         }
1439                 }
1440         }
1441
1442         return;
1443 }
1444
1445 AnjutaToken *
1446 anjuta_token_foreach_post_order (AnjutaToken *token, AnjutaTokenForeachFunc func, gpointer user_data)
1447 {
1448         if (token != NULL)
1449         {
1450                 AnjutaToken *last_parent;       /* If not NULL, token belong to a parent */
1451                 AnjutaToken *last_token;
1452                 AnjutaToken buffer;                     /* Temporary token allowing func to destroy
1453                                                                          * the current token */
1454                 gboolean expand = TRUE;
1455
1456                 last_parent = NULL;
1457                 last_token = token->last == NULL ? token : token->last;
1458                 while (token != NULL)
1459                 {
1460                         if (expand && (token->children != NULL))
1461                         {
1462                                 /* Check if we have found the last token */
1463                                 if (token == last_token)
1464                                 {
1465                                         /* Find last token */
1466                                         if (token->last == NULL)
1467                                         {
1468                                                 break;
1469                                         }
1470                                         /* Last token still include additional tokens */
1471                                         last_token = token->last;
1472                                 }
1473
1474                                 /* Enumerate children */
1475                                 token = token->children;
1476                         }
1477                         else
1478                         {
1479                                 /* Save token data in case it is destroyed */
1480                                 memcpy (&buffer, token, sizeof (buffer));
1481                                 /* Take into account all token */
1482                                 func (token, user_data);
1483
1484                                 /* Check if we have found the last token */
1485                                 if (token == last_token)
1486                                 {
1487                                         /* Find last token */
1488                                         if (buffer.last == NULL)
1489                                         {
1490                                                 token = &buffer;
1491                                                 break;
1492                                         }
1493                                         /* Last token still include additional tokens */
1494                                         last_token = buffer.last;
1495                                 }
1496
1497                                 if (token == last_parent)
1498                                 {
1499                                         /* Find last parent */
1500                                         if (buffer.last == NULL)
1501                                         {
1502                                                 /* Found complete group having children */
1503                                                 last_parent = NULL;
1504                                         }
1505                                         else
1506                                         {
1507                                                 /* Parent group has additional token */
1508                                                 last_parent = buffer.last;
1509                                         }
1510                                 }
1511
1512                                 if (buffer.next != NULL)
1513                                 {
1514                                         /* Get next sibling */
1515                                         token = buffer.next;
1516                                         expand = TRUE;
1517                                 }
1518                                 else
1519                                 {
1520                                         /* Get parent */
1521                                         token = buffer.parent;
1522                                         if (token != NULL) last_parent = token->last;
1523                                         expand = FALSE;
1524                                 }
1525                         }
1526                 }
1527
1528                 while ((token != NULL) && (token->next == NULL))
1529                 {
1530                         token = token->parent;
1531                 };
1532
1533                 if (token != NULL) token = token->next;
1534         }
1535
1536         return token;
1537 }
1538
1539 /* Token evaluation
1540  *---------------------------------------------------------------------------*/
1541
1542 static void
1543 evaluate_token (AnjutaToken *token, gpointer user_data)
1544 {
1545         GString *value = (GString *)user_data;
1546
1547         anjuta_token_evaluate_token (token, value, FALSE);
1548 }
1549
1550
1551 gchar *
1552 anjuta_token_evaluate (AnjutaToken *token)
1553 {
1554         GString *value = g_string_new (NULL);
1555
1556         anjuta_token_foreach_content (token, evaluate_token, value);
1557
1558         /* Return NULL and free data for an empty string */
1559         return g_string_free (value, *(value->str) == '\0');
1560 }
1561
1562 /* Does not evaluate content if token is a variable */
1563 gchar *
1564 anjuta_token_evaluate_name (AnjutaToken *token)
1565 {
1566         GString *value = g_string_new (NULL);
1567
1568         anjuta_token_foreach_container (token, evaluate_token, value);
1569
1570         /* Return NULL and free data for an empty string */
1571         return g_string_free (value, *(value->str) == '\0');
1572 }
1573
1574 gboolean
1575 anjuta_token_is_empty (AnjutaToken *token)
1576 {
1577         return (token == NULL) || ((token->data.length == 0) && (token->last == NULL) && (token->children == NULL));
1578 }
1579
1580
1581 /* Other functions
1582  *---------------------------------------------------------------------------*/
1583
1584 gboolean
1585 anjuta_token_compare (AnjutaToken *toka, AnjutaToken *tokb)
1586 {
1587         if (tokb->data.type)
1588         {
1589                 if (tokb->data.type != toka->data.type) return FALSE;
1590         }
1591
1592         if (tokb->data.type != ANJUTA_TOKEN_NONE)
1593         {
1594                 if (tokb->data.length != 0)
1595                 {
1596                         if (toka->data.length != tokb->data.length) return FALSE;
1597
1598                         if ((toka->data.flags & ANJUTA_TOKEN_CASE_INSENSITIVE)  && (tokb->data.flags & ANJUTA_TOKEN_CASE_INSENSITIVE))
1599                         {
1600                                 if (g_ascii_strncasecmp (toka->data.pos, tokb->data.pos, toka->data.length) != 0) return FALSE;
1601                         }
1602                         else
1603                         {
1604                                 if (strncmp (toka->data.pos, tokb->data.pos, toka->data.length) != 0) return FALSE;
1605                         }
1606                 }
1607         }
1608
1609         if (tokb->data.flags & ANJUTA_TOKEN_PUBLIC_FLAGS)
1610         {
1611                 if ((toka->data.flags & tokb->data.flags & ANJUTA_TOKEN_PUBLIC_FLAGS) == 0)
1612                         return FALSE;
1613         }
1614
1615         return TRUE;
1616 }
1617
1618 void
1619 anjuta_token_dump (AnjutaToken *token)
1620 {
1621         if (token == NULL) return;
1622
1623         anjuta_token_dump_child (token, 0, 0);
1624 }
1625
1626 void
1627 anjuta_token_dump_link (AnjutaToken *token)
1628 {
1629         AnjutaToken *last = token;
1630
1631         while (last->last != NULL) last = last->last;
1632
1633         for (; token != last; token = anjuta_token_next (token))
1634         {
1635                 anjuta_token_show (token, 0, 0);
1636         }
1637 }
1638
1639 gboolean
1640 anjuta_token_check (AnjutaToken *token)
1641 {
1642         if ((token->children != NULL) && (token->last != NULL))
1643         {
1644                 anjuta_token_show (token, 0, 0);
1645                 fprintf(stderr, "Error: Previous token has both non NULL children and last\n");
1646
1647                 return FALSE;
1648         }
1649
1650         if (token->children != NULL)
1651         {
1652                 AnjutaToken *child;
1653
1654                 for (child = token->children; child != NULL; child = child->next)
1655                 {
1656                         if (!anjuta_token_check_child (child, token)) return FALSE;
1657                 }
1658         }
1659
1660         if (token->last != NULL)
1661         {
1662                 AnjutaToken *child;
1663
1664                 for (child = anjuta_token_next (token); child != NULL; child = anjuta_token_next (child))
1665                 {
1666                         if (!anjuta_token_check (child)) return FALSE;
1667                         if (child == token->last) break;
1668                 }
1669         }
1670
1671         return TRUE;
1672 }
1673
1674 /* Constructor & Destructor
1675  *---------------------------------------------------------------------------*/
1676
1677 AnjutaToken *anjuta_token_new_string (AnjutaTokenType type, const gchar *value)
1678 {
1679         AnjutaToken *token;
1680
1681         if (value == NULL)
1682         {
1683                 token = anjuta_token_new_static (type, NULL);
1684         }
1685         else
1686         {
1687                 token = g_slice_new0 (AnjutaToken);
1688                 token->data.type = type  & ANJUTA_TOKEN_TYPE;
1689                 token->data.flags = type & ANJUTA_TOKEN_FLAGS;
1690                 token->data.pos = g_strdup (value);
1691                 token->data.length = strlen (value);
1692         }
1693
1694         return token;
1695 }
1696
1697 AnjutaToken*
1698 anjuta_token_new_string_len (AnjutaTokenType type, gchar *value, gsize length)
1699 {
1700         AnjutaToken *token;
1701
1702         if (value == NULL)
1703         {
1704                 token = anjuta_token_new_static (type, NULL);
1705         }
1706         else
1707         {
1708                 token = g_slice_new0 (AnjutaToken);
1709                 token->data.type = type  & ANJUTA_TOKEN_TYPE;
1710                 token->data.flags = type & ANJUTA_TOKEN_FLAGS;
1711                 token->data.pos = value;
1712                 token->data.length = length;
1713         }
1714
1715         return token;
1716 }
1717
1718 AnjutaToken *
1719 anjuta_token_new_static_len (gint type, const gchar *pos, gsize length)
1720 {
1721         AnjutaToken *token;
1722
1723         token = g_slice_new0 (AnjutaToken);
1724         token->data.type = type  & ANJUTA_TOKEN_TYPE;
1725         token->data.flags = (type & ANJUTA_TOKEN_FLAGS) | ANJUTA_TOKEN_STATIC;
1726         token->data.pos = (gchar *)pos;
1727         token->data.length = length;
1728
1729         return token;
1730 };
1731
1732 AnjutaToken *anjuta_token_new_static (AnjutaTokenType type, const char *value)
1733 {
1734         return anjuta_token_new_static_len (type, value, value == NULL ? 0 : strlen (value));
1735 }
1736
1737 AnjutaToken*
1738 anjuta_token_free_children (AnjutaToken *token)
1739 {
1740         AnjutaToken *child;
1741         AnjutaToken *last;
1742
1743         if (token == NULL) return NULL;
1744
1745         for (child = token->children; child != NULL; child = token->children)
1746         {
1747                 anjuta_token_free (child);
1748         }
1749         token->children = NULL;
1750
1751         if (token->last != NULL)
1752         {
1753                 last = token->last;
1754                 for (child = anjuta_token_next (token); child != NULL; child = anjuta_token_next (token))
1755                 {
1756                         anjuta_token_free (child);
1757                         if (child == last) break;
1758                 }
1759         }
1760         token->last = NULL;
1761
1762         return token;
1763 }
1764
1765 static void
1766 free_token (AnjutaToken *token, gpointer user_data)
1767 {
1768         anjuta_token_unlink_token (token);
1769         if ((token->data.pos != NULL) && !(token->data.flags & ANJUTA_TOKEN_STATIC))
1770         {
1771                 g_free (token->data.pos);
1772         }
1773         g_slice_free (AnjutaToken, token);
1774 }
1775
1776
1777 AnjutaToken*
1778 anjuta_token_free (AnjutaToken *token)
1779 {
1780         AnjutaToken *next;
1781
1782         if (token == NULL) return NULL;
1783
1784         next = anjuta_token_foreach_post_order (token, free_token, NULL);
1785
1786         return next;
1787 }