libanjuta/anjuta-token.c

   1 /* -*- Mode: C; indent-tabs-mode: t; c-basic-offset: 4; tab-width: 4 -*- */
   2 /*
   3  * anjuta-token.c
   4  * Copyright (C) Sébastien Granjoux 2009 <seb.sfo@free.fr>
   5  *
   6  * This program is free software: you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License as published by the
   8  * Free Software Foundation, either version 3 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful, but
  12  * WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  14  * See the GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License along
  17  * with this program.  If not, see <http://www.gnu.org/licenses/>.
  18  */
  19
  20 #include "anjuta-token.h"
  21
  22 #include "anjuta-debug.h"
  23
  24 #include <glib-object.h>
  25
  26 #include <stdio.h>
  27 #include <string.h>
  28 #include <stdlib.h>
  29
  30 /**
  31  * SECTION:anjuta-token
  32  * @title: Anjuta token
  33  * @short_description: Anjuta token
  34  * @see_also:
  35  * @stability: Unstable
  36  * @include: libanjuta/anjuta-token.h
  37  *
  38  * A #AnjutaToken represents a token. It is a sequence of characters associated
  39  * with a type representing its meaning. By example, a token can represent
  40  * a keyword, a comment, a variable...
  41  *
  42  * The token can own the string or has only a pointer on some data allocated
  43  * somewhere else with a length.
  44  *
  45  * A token is linked with other tokens using three double linked lists.
  46  *
  47  * The first list using next and prev fields is used to keep the token in the
  48  * order where there are in the file. The first character of the first token is
  49  * the first character in the file.
  50  *
  51  * A second list is used to represent included
  52  * files. Such file is represented by a special token in the first list which
  53  * has a pointer, named children to a token list. Each token in this secondary
  54  * list has a pointer to its parent, it means the token representing the file
  55  * where is the token. It looks like a tree. In fact, every file is represented
  56  * by a special token, so the root node is normally a file token and has as
  57  * children all the token representing the file content. This parent/child list
  58  * is used for expanded variable too.
  59  *
  60  * A third list is used to group several tokens. A token can have a pointer to
  61  * another last token. It means that this token is a group starting from this
  62  * token to the one indicated by the last field. In addition each token in this
  63  * group has a pointer on the first token of the group. This grouping is
  64  * independent of the parent/child list. So a group can start in one file and
  65  * end in another included file. The grouping can be nested too. Typically
  66  * we can have a group representing a command, a sub group representing the
  67  * arguments and then one sub group for each argument.
  68  */
  69
  70 /*
  71  * A token with its pointer can be put in one of the following case:
  72  *
  73  * * Simple token:
  74  *              children = NULL && last = NULL
  75  *              This is the case of a simple token without any children.
  76  *
  77  * * Composite token:
  78  *              children = NULL && last != NULL
  79  *              This is the case of a token grouping several other tokens, representing
  80  *              by example an item in a list.
  81  *
  82  * * Parent token:
  83  *              children != NULL && last == NULL
  84  *              This is the case of a token having children, by example a variable where
  85  *              the children represent the content. Most of the time the value of the
  86  *              parent token is ignored.
  87  *
  88  * * Composite parent token:
  89  *              children != NULL && last != NULL && (last in list)
  90  *              This case represents a variable which is split into several tokens. The
  91  *              children represents the content of the variable. The token up to last
  92  *              corresponds to the variable name. After getting the variable content, the
  93  *              next token is after the last token of the variable.
  94  *
  95  * * Composite parent token:
  96  *              children != NULL && last != NULL && (last in children)
  97  *              This case would represent a variable which is split into several token
  98  *              with one inside the children of the variable. This is not possible as
  99  *              the complete name of the variable is needed to get the children.
 100  *
 101  * * Composite parent token:
 102  *              children != NULL && last != NULL && (last in parent)
 103  *              This case represents a variable split into several token where the
 104  *              last element is a sibling of one parent of the token. I think this case
 105  *              is possible.
 106  *
 107  */
 108
 109 typedef struct _AnjutaTokenData AnjutaTokenData;
 110
 111 struct _AnjutaTokenData
 112 {
 113         AnjutaTokenType type;
 114         gint flags;
 115         gchar *pos;
 116         gsize length;
 117 };
 118
 119 struct _AnjutaToken
 120 {
 121         AnjutaToken     *next;
 122         AnjutaToken     *prev;
 123         AnjutaToken     *parent;
 124         AnjutaToken *last;
 125         AnjutaToken *group;
 126         AnjutaToken     *children;
 127         AnjutaTokenData data;
 128 };
 129
 130 /* Helpers functions
 131  *---------------------------------------------------------------------------*/
 132
 133 /* Private functions
 134  *---------------------------------------------------------------------------*/
 135
 136 static AnjutaToken *
 137 anjuta_token_next_child (AnjutaToken *child, AnjutaToken **last)
 138 {
 139         if (child == NULL) return child;
 140
 141         if (child->children != NULL)
 142         {
 143                 child = child->children;
 144         }
 145         else
 146         {
 147                 for (;;)
 148                 {
 149                         if ((*last == NULL) || (child == *last))
 150                         {
 151                                 if (child->last == NULL)
 152                                 {
 153                                         child = NULL;
 154                                         break;
 155                                 }
 156                                 *last = child->last;
 157                         }
 158                         if (child->next != NULL)
 159                         {
 160                                 child = child->next;
 161                                 break;
 162                         }
 163                         child = child->parent;
 164                 }
 165         }
 166
 167         return child;
 168 }
 169
 170 AnjutaToken *
 171 anjuta_token_next_after_children (AnjutaToken *token)
 172 {
 173         while (token->next == NULL)
 174         {
 175                 token = token->parent;
 176                 if (token == NULL) return NULL;
 177         };
 178
 179         return token->next;
 180 }
 181
 182 static AnjutaToken *
 183 anjuta_token_copy (AnjutaToken *token)
 184 {
 185         AnjutaToken *copy = NULL;
 186
 187         if (token != NULL)
 188         {
 189                 copy = g_slice_new0 (AnjutaToken);
 190                 copy->data.type = token->data.type;
 191                 copy->data.flags = token->data.flags;
 192                 if ((copy->data.flags & ANJUTA_TOKEN_STATIC) || (token->data.pos == NULL))
 193                 {
 194                         copy->data.pos = token->data.pos;
 195                 }
 196                 else
 197                 {
 198                         copy->data.pos = g_strdup (token->data.pos);
 199                 }
 200                 copy->data.length = token->data.length;
 201         }
 202
 203         return copy;
 204 }
 205
 206 /**
 207  * anjuta_token_unlink_token:
 208  * @token: a #AnjutaToken object.
 209  *
 210  * Unlink a single token, not the complete item, from the token tree.
 211  *
 212  * Return value: the removed token tree
 213  */
 214 static AnjutaToken *
 215 anjuta_token_unlink_token (AnjutaToken *token)
 216 {
 217
 218         if (token->prev != NULL)
 219         {
 220                 token->prev->next = token->next;
 221         }
 222         else if ((token->parent != NULL) && (token->parent->children == token))
 223         {
 224                 token->parent->children = token->next;
 225         }
 226         token->parent = NULL;
 227
 228         if ((token->group != NULL) && (token->group->last == token))
 229         {
 230                 AnjutaToken *prev;
 231
 232                 for (prev = token->prev; prev != NULL; prev = prev->prev)
 233                 {
 234                         if (prev->group == token->group)
 235                         {
 236                                 /* Find previous token in the same group */
 237                                 token->group->last = prev;
 238                                 break;
 239                         }
 240                         else if (prev == token->group)
 241                         {
 242                                 /* No more token in group */
 243                                 token->group->last = NULL;
 244                                 break;
 245                         }
 246                 }
 247         }
 248
 249         if (token->next != NULL)
 250         {
 251                 token->next->prev = token->prev;
 252                 token->next = NULL;
 253         }
 254         token->prev = NULL;
 255
 256         return token;
 257 }
 258
 259 /**
 260  * anjuta_token_insert_token_before:
 261  * @sibling: a #AnjutaToken object.
 262  * @token: a #AnjutaToken object.
 263  *
 264  * Insert token before sibling.
 265  *
 266  * Return value: inserted token
 267  */
 268 static AnjutaToken *
 269 anjuta_token_insert_token_before (AnjutaToken *sibling, AnjutaToken *token)
 270 {
 271         token->prev = sibling->prev;
 272         token->next = sibling;
 273
 274         if (token->prev != NULL)
 275         {
 276                 token->prev->next = token;
 277         }
 278         sibling->prev = token;
 279
 280         if ((sibling->parent != NULL) && (sibling->parent->children == sibling))
 281         {
 282                 sibling->parent->children = token;
 283         }
 284         token->parent = sibling->parent;
 285
 286         return token;
 287 }
 288
 289 static void
 290 anjuta_token_evaluate_token (AnjutaToken *token, GString *value, gboolean raw)
 291 {
 292         if ((token != NULL) && (token->data.length != 0))
 293         {
 294                 if (!raw)
 295                 {
 296                         switch (anjuta_token_get_type (token))
 297                         {
 298                         case ANJUTA_TOKEN_COMMENT:
 299                         case ANJUTA_TOKEN_OPEN_QUOTE:
 300                         case ANJUTA_TOKEN_CLOSE_QUOTE:
 301                         case ANJUTA_TOKEN_ESCAPE:
 302                         case ANJUTA_TOKEN_MACRO:
 303                         case ANJUTA_TOKEN_EOV:
 304                                 return;
 305                         default:
 306                                 break;
 307                         }
 308                 }
 309                 g_string_append_len (value, anjuta_token_get_string (token), anjuta_token_get_length (token));
 310         }
 311 }
 312
 313 static void
 314 anjuta_token_show (AnjutaToken *token, gint indent, gchar parent)
 315 {
 316         static gchar type[] = "\0";
 317         const gchar *string;
 318         gsize length;
 319
 320         type[0] = parent;
 321         fprintf (stderr, "%*s%s %p", indent, "", type, token);
 322         fprintf (stderr, ": %d ",
 323             anjuta_token_get_type (token));
 324         string = anjuta_token_get_string (token);
 325         length = anjuta_token_get_length (token);
 326         if (string == NULL)
 327         {
 328                 /* Value doesn't contain a newline */
 329                 fprintf (stderr, "(%lu)", length);
 330         }
 331         else
 332         {
 333                 const gchar *newline;
 334
 335                 newline = g_strrstr_len (string, length, "\n");
 336                 if (newline == NULL)
 337                 {
 338                         /* Value doesn't contain a newline */
 339                         fprintf (stderr, "\"%.*s\"",
 340                                 length,
 341                         string);
 342                 }
 343                 else
 344                 {
 345                         /* Value contains a newline, take care of indentation */
 346                         newline++;
 347                         fprintf (stderr, "\"%.*s",
 348                         newline - string,
 349                         string);
 350                         for (;;)
 351                         {
 352                                 length -= newline - string;
 353                                 string = newline;
 354
 355                                 newline = g_strrstr_len (string, length, "\n");
 356                                 if (newline == NULL) break;
 357
 358                                 newline++;
 359                                 fprintf (stderr, "%*s  %.*s",
 360                                     indent, "",
 361                                         newline - string,
 362                                 string);
 363                         }
 364                         fprintf (stderr, "%*s  %.*s\"",
 365                                     indent, "",
 366                                         length,
 367                                 string);
 368                 }
 369         }
 370         fprintf (stderr, " %p/%p (%p/%p) %s\n",
 371                 token->last, token->children,
 372                 token->group, token->parent,
 373                 anjuta_token_get_flags (token) & ANJUTA_TOKEN_REMOVED ? " (removed)" : "");
 374 }
 375
 376 static AnjutaToken*
 377 anjuta_token_dump_child (AnjutaToken *token, gint indent, gchar type)
 378 {
 379         AnjutaToken *last;
 380         AnjutaToken *child;
 381
 382         anjuta_token_show (token, indent, type);
 383         indent += 4;
 384
 385         last = token;
 386         if (token->last != NULL)
 387         {
 388                 do
 389                 {
 390                         child = last->next;
 391                         if (child == NULL) break;
 392                         last = anjuta_token_dump_child (child, indent, '+');
 393                 }
 394                 while (child != token->last);
 395         }
 396
 397         if (token->children != NULL)
 398         {
 399                 for (child = token->children; child != NULL; child = child->next)
 400                 {
 401                         child = anjuta_token_dump_child (child, indent, '*');
 402                 }
 403         }
 404
 405         return last;
 406 }
 407
 408 static gboolean
 409 anjuta_token_check_child (AnjutaToken *token, AnjutaToken *parent)
 410 {
 411         if (token->parent != parent)
 412         {
 413                 anjuta_token_show (token, 0, 0);
 414                 fprintf(stderr, "Error: Children has %p as parent instead of %p\n", token->parent, parent);
 415                 return FALSE;
 416         }
 417
 418         return anjuta_token_check (token);
 419 }
 420
 421 /* Get and set functions
 422  *---------------------------------------------------------------------------*/
 423
 424 void
 425 anjuta_token_set_type (AnjutaToken *token, gint type)
 426 {
 427         token->data.type = type;
 428 }
 429
 430 gint
 431 anjuta_token_get_type (AnjutaToken *token)
 432 {
 433         return token->data.type;
 434 }
 435
 436 void
 437 anjuta_token_set_flags (AnjutaToken *token, gint flags)
 438 {
 439         AnjutaToken *child;
 440         AnjutaToken *last = token->last;
 441
 442         for (child = token; child != NULL; child = anjuta_token_next_child (child, &last))
 443         {
 444                 child->data.flags |= flags;
 445         }
 446 }
 447
 448 void
 449 anjuta_token_clear_flags (AnjutaToken *token, gint flags)
 450 {
 451         token->data.flags &= ~flags;
 452 }
 453
 454 gint
 455 anjuta_token_get_flags (AnjutaToken *token)
 456 {
 457         return token->data.flags;
 458 }
 459
 460 void
 461 anjuta_token_set_string (AnjutaToken *token, const gchar *data, gsize length)
 462 {
 463         if (!(token->data.flags & ANJUTA_TOKEN_STATIC))
 464         {
 465                 g_free (token->data.pos);
 466                 token->data.flags |= ANJUTA_TOKEN_STATIC;
 467         }
 468         token->data.pos = (gchar *)data;
 469         token->data.length = length;
 470 }
 471
 472 const gchar *
 473 anjuta_token_get_string (AnjutaToken *token)
 474 {
 475         return token->data.pos;
 476 }
 477
 478 void
 479 anjuta_token_set_length (AnjutaToken *token, gsize length)
 480 {
 481         token->data.length = length;
 482 }
 483
 484 gsize
 485 anjuta_token_get_length (AnjutaToken *token)
 486 {
 487         return token->data.length;
 488 }
 489
 490 /* Basic move functions
 491  *---------------------------------------------------------------------------*/
 492
 493 AnjutaToken *
 494 anjuta_token_next (AnjutaToken *token)
 495 {
 496         if (token->children != NULL)
 497         {
 498                 return token->children;
 499         }
 500         else if (token->next != NULL)
 501         {
 502                 return token->next;
 503         }
 504         else if (token->parent != NULL)
 505         {
 506                 return anjuta_token_next_after_children (token->parent);
 507         }
 508         else
 509         {
 510                 return NULL;
 511         }
 512 }
 513
 514 AnjutaToken *
 515 anjuta_token_previous (AnjutaToken *token)
 516 {
 517         if (token->prev != NULL)
 518         {
 519                 return token->prev;
 520         }
 521         else
 522         {
 523                 return token->parent;
 524         }
 525 }
 526
 527 AnjutaToken *
 528 anjuta_token_last (AnjutaToken *token)
 529 {
 530         AnjutaToken *last;
 531
 532         for (last = token; last->last != NULL; last = last->last);
 533         if (last->children != NULL)
 534         {
 535                 for (last = last->children; last->next != NULL; last = last->next);
 536         }
 537
 538         return last;
 539 }
 540
 541 AnjutaToken *
 542 anjuta_token_parent (AnjutaToken *token)
 543 {
 544         return token->parent;
 545 }
 546
 547 AnjutaToken *
 548 anjuta_token_list (AnjutaToken *token)
 549 {
 550         return token->group;
 551 }
 552
 553 /* Item move functions
 554  *---------------------------------------------------------------------------*/
 555
 556 AnjutaToken *
 557 anjuta_token_last_item (AnjutaToken *list)
 558 {
 559         return list->last;
 560 }
 561
 562 AnjutaToken *
 563 anjuta_token_first_item (AnjutaToken *list)
 564 {
 565         AnjutaToken *first = NULL;
 566
 567         if (list != NULL)
 568         {
 569                 if (list->children != NULL)
 570                 {
 571                         first = list->children;
 572                 }
 573                 else if (list->last != NULL)
 574                 {
 575                         first = list->next;
 576                 }
 577         }
 578
 579         return first;
 580 }
 581
 582 AnjutaToken *
 583 anjuta_token_next_item (AnjutaToken *item)
 584 {
 585         AnjutaToken *next;
 586
 587         if (item != NULL)
 588         {
 589                 do
 590                 {
 591                         next = NULL;
 592                         if ((item->group == NULL) || (item->group->last != item))
 593                         {
 594                                 AnjutaToken *last;
 595                                 for (last = item; last->last != NULL; last = last->last);
 596                                 next = anjuta_token_next (last);
 597                                 if ((next != NULL) && (next->group != item->group)) next = NULL;
 598                         }
 599                         item = next;
 600                 }
 601                 /* Loop if the current item has been deleted */
 602                 while ((next != NULL) && (anjuta_token_get_flags (next) & ANJUTA_TOKEN_REMOVED));
 603         }
 604
 605         return next;
 606 }
 607
 608 AnjutaToken *
 609 anjuta_token_previous_item (AnjutaToken *item)
 610 {
 611         AnjutaToken *prev = NULL;
 612
 613
 614         if (item != NULL)
 615         {
 616                 do
 617                 {
 618                         for (prev = item->prev; (prev != NULL) && (prev->group != item->group); prev = prev->group);
 619                         item = prev;
 620                 }
 621                 /* Loop if the current item has been deleted */
 622                 while ((prev != NULL) && (anjuta_token_get_flags (prev) & ANJUTA_TOKEN_REMOVED));
 623         }
 624
 625         return prev;
 626 }
 627
 628 /* Add/Insert/Remove tokens
 629  *---------------------------------------------------------------------------*/
 630
 631 /**
 632  * anjuta_token_append_child:
 633  * @parent: a #AnjutaToken object used as parent.
 634  * @children: a #AnjutaToken object.
 635  *
 636  * Insert all tokens in children as the last children of the given parent.
 637  *
 638  * Return value: The first token append.
 639  */
 640 AnjutaToken *
 641 anjuta_token_append_child (AnjutaToken *parent, AnjutaToken *children)
 642 {
 643         AnjutaToken *token;
 644         AnjutaToken *last;
 645         AnjutaToken *old_group;
 646         AnjutaToken *old_parent;
 647
 648         g_return_val_if_fail (parent != NULL, NULL);
 649         g_return_val_if_fail (children != NULL, NULL);
 650
 651         old_group = children->group;
 652         old_parent = children->parent;
 653
 654         if (parent->children == NULL)
 655         {
 656                 parent->children = children;
 657
 658                 children->prev = NULL;
 659         }
 660         else
 661         {
 662                 /* Find last children */
 663                 for (last = parent->children; last->next != NULL;)
 664                 {
 665                         if ((last->last != NULL) && (last->last->parent == last->parent))
 666                         {
 667                                 last = last->last;
 668                         }
 669                         else
 670                         {
 671                                 last = last->next;
 672                         }
 673                 }
 674
 675                 last->next = children;
 676                 children->prev = last;
 677         }
 678
 679         /* Update each token */
 680         for (token = children;;)
 681         {
 682                 if (token->parent == old_parent) token->parent = parent;
 683                 if (token->group == old_group) token->group = parent->group;
 684
 685                 if (token->children != NULL)
 686                 {
 687                         token = token->children;
 688                 }
 689                 else if (token->next != NULL)
 690                 {
 691                         token = token->next;
 692                 }
 693                 else
 694                 {
 695                         while (token->parent != parent)
 696                         {
 697                                 token = token->parent;
 698                                 if (token->next != NULL) break;
 699                         }
 700                         if (token->next == NULL) break;
 701                         token = token->next;
 702                 }
 703         }
 704
 705         return children;
 706 }
 707
 708 /**
 709  * anjuta_token_prepend_child:
 710  * @parent: a #AnjutaToken object used as parent.
 711  * @children: a #AnjutaToken object.
 712  *
 713  * Insert all tokens in children as the first children of the given parent.
 714  *
 715  * Return value: The first token append.
 716  */
 717 AnjutaToken *
 718 anjuta_token_prepend_child (AnjutaToken *parent, AnjutaToken *children)
 719 {
 720         AnjutaToken *child;
 721         AnjutaToken *last = NULL;
 722
 723         g_return_val_if_fail (parent != NULL, NULL);
 724         g_return_val_if_fail (children != NULL, NULL);
 725
 726         /* Update each token */
 727         for (child = children;;)
 728         {
 729                 AnjutaToken *next;
 730
 731                 if (child->parent == children->parent) child->parent = parent;
 732                 if (child->group == children->group) child->group = parent->group;
 733
 734                 next = anjuta_token_next_child (child, &last);
 735                 if (next == NULL) break;
 736                 child = next;
 737         }
 738
 739         child->next = parent->children;
 740         if (child->next) child->next->prev = child;
 741         parent->children = children;
 742
 743         return children;
 744 }
 745
 746 /**
 747  * anjuta_token_prepend_items:
 748  * @list: a #AnjutaToken object used as list.
 749  * @item: a #AnjutaToken object.
 750  *
 751  * Insert all tokens in item as item of the given list.
 752  *
 753  * Return value: The first token append.
 754  */
 755 AnjutaToken *
 756 anjuta_token_prepend_items (AnjutaToken *list, AnjutaToken *item)
 757 {
 758         AnjutaToken *token;
 759         AnjutaToken *old_group;
 760         AnjutaToken *old_parent;
 761
 762         g_return_val_if_fail (list != NULL, NULL);
 763         g_return_val_if_fail (item != NULL, NULL);
 764
 765         old_group = item->group;
 766         old_parent = item->parent;
 767
 768         /* Update each token */
 769         for (token = item;;)
 770         {
 771                 if (token->parent == old_parent) token->parent = list->parent;
 772                 if (token->group == old_group) token->group = list;
 773
 774                 if (token->children != NULL)
 775                 {
 776                         token = token->children;
 777                 }
 778                 else if (token->next != NULL)
 779                 {
 780                         token = token->next;
 781                 }
 782                 else
 783                 {
 784                         while (token->parent != list->parent)
 785                         {
 786                                 token = token->parent;
 787                                 if (token->next != NULL) break;
 788                         }
 789                         if (token->next == NULL) break;
 790                         token = token->next;
 791                 }
 792         }
 793
 794         token->next = list->next;
 795         if (token->next) token->next->prev = token;
 796
 797         list->next = item;
 798         item->prev = list;
 799
 800         if (list->last == NULL)
 801         {
 802                 while (token->group != list) token = token->group;
 803                 list->last = token;
 804         }
 805
 806         return item;
 807 }
 808
 809 /**
 810  * anjuta_token_insert_after:
 811  * @sibling: a #AnjutaToken object.
 812  * @item: a #AnjutaToken object.
 813  *
 814  * Insert all tokens after sibling.
 815  *
 816  * Return value: The first token inserted.
 817  */
 818 AnjutaToken *
 819 anjuta_token_insert_after (AnjutaToken *sibling, AnjutaToken *list)
 820 {
 821         AnjutaToken *last;
 822         AnjutaToken *token;
 823         AnjutaToken *old_group;
 824         AnjutaToken *old_parent;
 825
 826         g_return_val_if_fail (sibling != NULL, NULL);
 827         g_return_val_if_fail (list != NULL, NULL);
 828
 829         old_group = list->group;
 830         old_parent = list->parent;
 831
 832         /* Update each token */
 833         for (token = list;;)
 834         {
 835                 if (token->parent == old_parent) token->parent = sibling->parent;
 836                 if (token->group == old_group) token->group = sibling->group;
 837
 838                 if (token->children != NULL)
 839                 {
 840                         token = token->children;
 841                 }
 842                 else if (token->next != NULL)
 843                 {
 844                         token = token->next;
 845                 }
 846                 else
 847                 {
 848                         while (token->parent != sibling->parent)
 849                         {
 850                                 token = token->parent;
 851                                 if (token->next != NULL) break;
 852                         }
 853                         if (token->next == NULL) break;
 854                         token = token->next;
 855                 }
 856         }
 857
 858         for (last = sibling; last->last != NULL; last = last->last);
 859
 860         token->next = last->next;
 861         if (token->next) token->next->prev = token;
 862
 863         last->next = list;
 864         list->prev = last;
 865
 866         if ((sibling->group != NULL) && (sibling->group->last == sibling))
 867         {
 868                 while (token->group != sibling->group) token = token->group;
 869                 sibling->group->last = token;
 870         }
 871
 872         return list;
 873 }
 874
 875 /**
 876  * anjuta_token_insert_before:
 877  * @sibling: a #AnjutaToken object.
 878  * @item: a #AnjutaToken object.
 879  *
 880  * Insert all tokens before sibling.
 881  *
 882  * Return value: The first token inserted.
 883  */
 884 AnjutaToken *
 885 anjuta_token_insert_before (AnjutaToken *sibling, AnjutaToken *list)
 886 {
 887         AnjutaToken *last;
 888         AnjutaToken *token;
 889         AnjutaToken *old_group;
 890         AnjutaToken *old_parent;
 891
 892         g_return_val_if_fail (sibling != NULL, NULL);
 893         g_return_val_if_fail (list != NULL, NULL);
 894
 895         old_group = list->group;
 896         old_parent = list->parent;
 897
 898         /* Update each token */
 899         for (token = list;;)
 900         {
 901                 if (token->parent == old_parent) token->parent = sibling->parent;
 902                 if (token->group == old_group) token->group = sibling->group;
 903
 904                 if (token->children != NULL)
 905                 {
 906                         token = token->children;
 907                 }
 908                 else if (token->next != NULL)
 909                 {
 910                         token = token->next;
 911                 }
 912                 else
 913                 {
 914                         while (token->parent != sibling->parent)
 915                         {
 916                                 token = token->parent;
 917                                 if (token->next != NULL) break;
 918                         }
 919                         if (token->next == NULL) break;
 920                         token = token->next;
 921                 }
 922         }
 923
 924         for (last = sibling; last->last != NULL; last = last->last);
 925
 926         token->next = sibling;
 927         list->prev = sibling->prev;
 928         sibling->prev = token;
 929
 930         if (list->prev) list->prev->next = list;
 931
 932         if ((list->parent != NULL) && (list->parent->children == sibling)) list->parent->children = list;
 933
 934         return list;
 935 }
 936
 937 /**
 938  * anjuta_token_delete_parent:
 939  * @parent: a #AnjutaToken object used as parent.
 940  *
 941  * Delete only the parent token.
 942  *
 943  * Return value: the first children
 944  */
 945 AnjutaToken *
 946 anjuta_token_delete_parent (AnjutaToken *parent)
 947 {
 948         AnjutaToken *token;
 949
 950         g_return_val_if_fail (parent != NULL, NULL);
 951
 952         if (parent->children == NULL) return NULL;
 953
 954         /* Update each token */
 955         for (token = parent->children;;)
 956         {
 957                 if (token->parent == parent) token->parent = parent->parent;
 958
 959                 if (token->children != NULL)
 960                 {
 961                         token = token->children;
 962                 }
 963                 else if (token->next != NULL)
 964                 {
 965                         token = token->next;
 966                 }
 967                 else
 968                 {
 969                         while (token->parent != parent->parent)
 970                         {
 971                                 token = token->parent;
 972                                 if (token->next != NULL) break;
 973                         }
 974                         if (token->next == NULL) break;
 975                         token = token->next;
 976                 }
 977         }
 978
 979         token->next = parent->next;
 980         if (token->next) token->next->prev = token;
 981
 982         parent->next = parent->children;
 983         parent->children->prev = parent;
 984         parent->children = NULL;
 985
 986         return anjuta_token_free (parent);
 987 }
 988
 989 /* Merge function
 990  *---------------------------------------------------------------------------*/
 991
 992 /* anjuta_token_merge can be used with first or end being a floating token and
 993  * on already grouped tokens to change the group organisation */
 994 AnjutaToken *
 995 anjuta_token_merge (AnjutaToken *first, AnjutaToken *end)
 996 {
 997         AnjutaToken *next;
 998
 999         if ((first == end) || (end == NULL)) return first;
1000
1001         /* Insert first or end in the same sequence if it is not already the case */
1002         for (next = first; next != end; next = anjuta_token_next (next))
1003         {
1004                 if (next == NULL)
1005                 {
1006                         if (first->parent == NULL)
1007                         {
1008                                 anjuta_token_insert_before (end, first);
1009                         }
1010                         else
1011                         {
1012                                 anjuta_token_insert_after (first, end);
1013                         }
1014                         break;
1015                 }
1016         }
1017         first->last = end;
1018         if ((end->group != NULL) && (end->group != first) && (end->group->last == end)) end->group->last = first;
1019         end->group = first;
1020
1021         return first;
1022 }
1023
1024 AnjutaToken *
1025 anjuta_token_merge_own_children (AnjutaToken *group)
1026 {
1027         AnjutaToken *token;
1028         AnjutaToken *next = NULL;
1029
1030         if (group->last != NULL) return group;
1031
1032         if (group->last->last != NULL) group->last = group->last->last;
1033
1034         for (token = anjuta_token_next (group); (token != NULL) && (token != group->last); token = anjuta_token_next (token))
1035         {
1036                 if (next == NULL)
1037                 {
1038                         if (token->last != NULL)
1039                         {
1040                                 next = token->last;
1041                                 //token->last = NULL;
1042                         }
1043                         token->group = group;
1044                 }
1045                 else if (next == token)
1046                 {
1047                         next = NULL;
1048                 }
1049         }
1050
1051         return group;
1052 }
1053
1054 AnjutaToken *
1055 anjuta_token_merge_children (AnjutaToken *first, AnjutaToken *end)
1056 {
1057         if ((first == end) || (end == NULL))
1058         {
1059                 return first;
1060         }
1061
1062         if (first->parent == NULL)
1063         {
1064                 first->parent = end->parent;
1065         }
1066         if (first->next == NULL)
1067         {
1068                 anjuta_token_insert_before (end, first);
1069         }
1070         anjuta_token_unlink_token (end);
1071         if (end->last != NULL)
1072         {
1073                 AnjutaToken *child;
1074
1075                 first->last = end->last;
1076                 for (child = anjuta_token_next (first); child != first->last; child = anjuta_token_next (child))
1077                 {
1078                         if (child->group == end) child->group = first;
1079                 }
1080                 first->last->group = first;
1081         }
1082         end->group = first;
1083         anjuta_token_free (end);
1084
1085         return first;
1086 }
1087
1088 /**
1089  * anjuta_token_merge_previous:
1090  * @list: a #AnjutaToken object representing a list
1091  * @first: a #AnjutaToken object for the new beginning of the list
1092  *
1093  * If the list token is not already linked with first, it is inserted
1094  * just before first.
1095  * If the list token is already linked, it must be in the same list after
1096  * first token. It it possible to have several tokens beweent list and
1097  * first.
1098  *
1099  * Return value: the new list
1100  */
1101 AnjutaToken *
1102 anjuta_token_merge_previous (AnjutaToken *list, AnjutaToken *first)
1103 {
1104         AnjutaToken *token;
1105
1106         if ((first == NULL) || (list == first)) return list;
1107
1108         /* Change group of all tokens from end to first
1109          * if the list is already linked */
1110         if ((list->prev != NULL) || (list->parent != NULL))
1111         {
1112                 for (token = first; token != NULL; token = anjuta_token_next_item (token))
1113                 {
1114                         token->group = list;
1115                 }
1116         }
1117
1118         token = anjuta_token_next (list);
1119         anjuta_token_unlink_token (list);
1120         anjuta_token_insert_token_before (first, list);
1121
1122         return list;
1123 }
1124
1125 AnjutaToken *anjuta_token_split (AnjutaToken *token, guint size)
1126 {
1127         if (token->data.length > size)
1128         {
1129                 AnjutaToken *copy;
1130
1131                 copy = anjuta_token_copy (token);
1132                 anjuta_token_insert_before (token, copy);
1133
1134                 copy->data.length = size;
1135                 if (token->data.flags & ANJUTA_TOKEN_STATIC)
1136                 {
1137                         token->data.pos += size;
1138                         token->data.length -= size;
1139                 }
1140                 else
1141                 {
1142                         memcpy(token->data.pos, token->data.pos + size, token->data.length - size);
1143                 }
1144
1145                 return copy;
1146         }
1147         else
1148         {
1149                 return token;
1150         }
1151 }
1152
1153 AnjutaToken *
1154 anjuta_token_cut (AnjutaToken *token, guint pos, guint size)
1155 {
1156         AnjutaToken *copy;
1157
1158         copy = anjuta_token_copy (token);
1159
1160         if (pos >= token->data.length)
1161         {
1162                 if (!(copy->data.flags & ANJUTA_TOKEN_STATIC))
1163                 {
1164                         g_free (copy->data.pos);
1165                 }
1166                 copy->data.pos = NULL;
1167                 copy->data.length = 0;
1168         }
1169         if ((pos + size) > token->data.length)
1170         {
1171                 size = token->data.length - pos;
1172         }
1173
1174         if (copy->data.flags & ANJUTA_TOKEN_STATIC)
1175         {
1176                 copy->data.pos += pos;
1177         }
1178         else
1179         {
1180                 memcpy(copy->data.pos, copy->data.pos + pos, size);
1181         }
1182         copy->data.length = size;
1183
1184         return copy;
1185 }
1186
1187 static void
1188 concat_token (AnjutaToken *token, gpointer user_data)
1189 {
1190         AnjutaToken *first = (AnjutaToken *)user_data;
1191
1192         if (anjuta_token_get_length (token) > 0)
1193         {
1194                 if (anjuta_token_get_string (first) == NULL)
1195                 {
1196                         anjuta_token_set_string (first, anjuta_token_get_string (token), anjuta_token_get_length (token));
1197                 }
1198                 else if ((first != NULL) && ((anjuta_token_get_string(first) + anjuta_token_get_length (first)) == anjuta_token_get_string (token)))
1199                 {
1200                         anjuta_token_set_string (first, anjuta_token_get_string (first), anjuta_token_get_length (first) + anjuta_token_get_length (token));
1201                 }
1202                 else
1203                 {
1204                         AnjutaToken *new;
1205
1206                         new = anjuta_token_new_static_len (ANJUTA_TOKEN_CONTENT, anjuta_token_get_string (token), anjuta_token_get_length (token));
1207                         anjuta_token_insert_after (first, new);
1208                         anjuta_token_merge (first, new);
1209                 }
1210         }
1211 }
1212
1213
1214 AnjutaToken *
1215 anjuta_token_concat(AnjutaToken *token)
1216 {
1217         AnjutaToken *new;
1218
1219         new = anjuta_token_new_static (ANJUTA_TOKEN_CONTENT, NULL);
1220         anjuta_token_foreach_token (token, concat_token, new);
1221
1222         anjuta_token_insert_token_before (token, new);
1223         anjuta_token_free (token);
1224
1225         return new;
1226 }
1227
1228 /* Token foreach
1229  *---------------------------------------------------------------------------*/
1230
1231 void
1232 anjuta_token_foreach_token (AnjutaToken *token, AnjutaTokenForeachFunc func, gpointer user_data)
1233 {
1234         if (token != NULL)
1235         {
1236                 AnjutaToken *last_token;
1237                 gint child = 0;
1238
1239                 last_token = token->last == NULL ? token : token->last;
1240                 while (token != NULL)
1241                 {
1242                         if (child == 0) func (token, user_data);
1243
1244                         /* Check if we have found the last token */
1245                         if (token == last_token)
1246                         {
1247                                 /* Find last token */
1248                                 if (token->last == NULL)
1249                                 {
1250                                         break;
1251                                 }
1252                                 /* Last token still include additional tokens */
1253                                 last_token = token->last;
1254                         }
1255
1256                         if (token->children != NULL)
1257                         {
1258                                 /* Check children, only for last token */
1259                                 child++;
1260                                 token = token->children;
1261                         }
1262                         else if (token->next != NULL)
1263                         {
1264                                 /* Get next sibling */
1265                                 token = token->next;
1266                         }
1267                         else
1268                         {
1269                                 /* Get parent */
1270                                 for (;;)
1271                                 {
1272                                         child--;
1273                                         token = token->parent;
1274                                         if (token == NULL) break;
1275                                         if (token->next != NULL)
1276                                         {
1277                                                 token = token->next;
1278                                                 break;
1279                                         }
1280                                 }
1281                         }
1282                 }
1283         }
1284
1285         return;
1286 }
1287
1288 void
1289 anjuta_token_foreach_content (AnjutaToken *token, AnjutaTokenForeachFunc func, gpointer user_data)
1290 {
1291         if (token != NULL)
1292         {
1293                 AnjutaToken *last_parent;       /* If not NULL, token belong to a parent and
1294                                                                          * are not taken into account */
1295                 AnjutaToken *last_token;
1296                 gboolean expand = TRUE;
1297
1298                 last_parent = NULL;
1299                 last_token = token->last == NULL ? token : token->last;
1300                 for (;;)
1301                 {
1302                         if (expand && (token->children != NULL))
1303                         {
1304                                 /* Children of the last token does not belong to the group */
1305                                 if (token == last_token)
1306                                 {
1307                                         /* Find last token */
1308                                         if (token->last == NULL)
1309                                         {
1310                                                 break;
1311                                         }
1312                                         /* Last token still include additional tokens */
1313                                         last_token = token->last;
1314                                 }
1315
1316                                 /* Enumerate children */
1317                                 token = token->children;
1318                         }
1319                         else
1320                         {
1321                                 if (token->children == NULL)
1322                                 {
1323                                         /* Take into account only the content of token having no children */
1324                                         if (last_parent == NULL)
1325                                         {
1326                                                 /* Take into account only the content of group having no children */
1327                                                 func (token, user_data);
1328                                         }
1329                                 }
1330
1331                                 /* Check if we have found the last token */
1332                                 if (token == last_token)
1333                                 {
1334                                         /* Find last token */
1335                                         if (token->last == NULL)
1336                                         {
1337                                                 break;
1338                                         }
1339                                         /* Last token still include additional tokens */
1340                                         last_token = token->last;
1341                                 }
1342
1343                                 if (token == last_parent)
1344                                 {
1345                                         /* Find last parent */
1346                                         if (token->last == NULL)
1347                                         {
1348                                                 /* Found complete group having children */
1349                                                 last_parent = NULL;
1350                                         }
1351                                         else
1352                                         {
1353                                                 /* Parent group has additional token */
1354                                                 last_parent = token->last;
1355                                         }
1356                                 }
1357
1358                                 if (token->next != NULL)
1359                                 {
1360                                         /* Get next sibling */
1361                                         token = token->next;
1362                                         expand = TRUE;
1363                                 }
1364                                 else
1365                                 {
1366                                         /* Get parent */
1367                                         token = token->parent;
1368                                         if (token != NULL) last_parent = token->last;
1369                                         expand = FALSE;
1370                                 }
1371                         }
1372                 }
1373         }
1374
1375         return;
1376 }
1377
1378 void
1379 anjuta_token_foreach_container (AnjutaToken *token, AnjutaTokenForeachFunc func, gpointer user_data)
1380 {
1381         if (token != NULL)
1382         {
1383                 AnjutaToken *parent;    /* If not NULL, token belong to a parent and
1384                                                                          * are not taken into account */
1385                 AnjutaToken *last_token;
1386                 gboolean expand = TRUE;
1387
1388                 parent = NULL;
1389                 last_token = token->last == NULL ? token : token->last;
1390                 for (;;)
1391                 {
1392                         if (expand && (token->children != NULL))
1393                         {
1394                                 if (parent == NULL) parent = token;
1395                                 /* Enumerate children */
1396                                 token = token->children;
1397                         }
1398                         else
1399                         {
1400                                 /* Take into account only the content of parent token */
1401                                 if (parent == NULL)
1402                                 {
1403                                         /* Take into account only the content of group having no children */
1404                                         func (token, user_data);
1405                                 }
1406
1407                                 /* Check if we have found the last token */
1408                                 if (token == last_token)
1409                                 {
1410                                         /* Find last token */
1411                                         if (token->last == NULL)
1412                                         {
1413                                                 break;
1414                                         }
1415                                         /* Last token still include additional tokens */
1416                                         last_token = token->last;
1417                                 }
1418
1419                                 if (token->next != NULL)
1420                                 {
1421                                         /* Get next sibling */
1422                                         token = token->next;
1423                                         expand = TRUE;
1424                                 }
1425                                 else
1426                                 {
1427                                         /* Get parent */
1428                                         token = token->parent;
1429                                         if (token == parent) parent = NULL;
1430                                         expand = FALSE;
1431                                 }
1432                         }
1433                 }
1434         }
1435
1436         return;
1437 }
1438
1439 AnjutaToken *
1440 anjuta_token_foreach_post_order (AnjutaToken *token, AnjutaTokenForeachFunc func, gpointer user_data)
1441 {
1442         if (token != NULL)
1443         {
1444                 AnjutaToken *last_parent;       /* If not NULL, token belong to a parent */
1445                 AnjutaToken *last_token;
1446                 AnjutaToken buffer;                     /* Temporary token allowing func to destroy
1447                                                                          * the current token */
1448                 gboolean expand = TRUE;
1449
1450                 last_parent = NULL;
1451                 last_token = token->last == NULL ? token : token->last;
1452                 while (token != NULL)
1453                 {
1454                         if (expand && (token->children != NULL))
1455                         {
1456                                 /* Check if we have found the last token */
1457                                 if (token == last_token)
1458                                 {
1459                                         /* Find last token */
1460                                         if (token->last == NULL)
1461                                         {
1462                                                 break;
1463                                         }
1464                                         /* Last token still include additional tokens */
1465                                         last_token = token->last;
1466                                 }
1467
1468                                 /* Enumerate children */
1469                                 token = token->children;
1470                         }
1471                         else
1472                         {
1473                                 /* Save token data in case it is destroyed */
1474                                 memcpy (&buffer, token, sizeof (buffer));
1475                                 /* Take into account all token */
1476                                 func (token, user_data);
1477
1478                                 /* Check if we have found the last token */
1479                                 if (token == last_token)
1480                                 {
1481                                         /* Find last token */
1482                                         if (buffer.last == NULL)
1483                                         {
1484                                                 token = &buffer;
1485                                                 break;
1486                                         }
1487                                         /* Last token still include additional tokens */
1488                                         last_token = buffer.last;
1489                                 }
1490
1491                                 if (token == last_parent)
1492                                 {
1493                                         /* Find last parent */
1494                                         if (buffer.last == NULL)
1495                                         {
1496                                                 /* Found complete group having children */
1497                                                 last_parent = NULL;
1498                                         }
1499                                         else
1500                                         {
1501                                                 /* Parent group has additional token */
1502                                                 last_parent = buffer.last;
1503                                         }
1504                                 }
1505
1506                                 if (buffer.next != NULL)
1507                                 {
1508                                         /* Get next sibling */
1509                                         token = buffer.next;
1510                                         expand = TRUE;
1511                                 }
1512                                 else
1513                                 {
1514                                         /* Get parent */
1515                                         token = buffer.parent;
1516                                         if (token != NULL) last_parent = token->last;
1517                                         expand = FALSE;
1518                                 }
1519                         }
1520                 }
1521
1522                 while ((token != NULL) && (token->next == NULL))
1523                 {
1524                         token = token->parent;
1525                 };
1526
1527                 if (token != NULL) token = token->next;
1528         }
1529
1530         return token;
1531 }
1532
1533 /* Token evaluation
1534  *---------------------------------------------------------------------------*/
1535
1536 static void
1537 evaluate_raw_token (AnjutaToken *token, gpointer user_data)
1538 {
1539         GString *value = (GString *)user_data;
1540
1541         anjuta_token_evaluate_token (token, value, TRUE);
1542 }
1543
1544 static void
1545 evaluate_token (AnjutaToken *token, gpointer user_data)
1546 {
1547         GString *value = (GString *)user_data;
1548
1549         anjuta_token_evaluate_token (token, value, FALSE);
1550 }
1551
1552
1553 gchar *
1554 anjuta_token_evaluate (AnjutaToken *token)
1555 {
1556         GString *value = g_string_new (NULL);
1557
1558         anjuta_token_foreach_content (token, evaluate_token, value);
1559
1560         /* Return NULL and free data for an empty string */
1561         return g_string_free (value, *(value->str) == '\0');
1562 }
1563
1564 /* Does not evaluate content if token is a variable */
1565 gchar *
1566 anjuta_token_evaluate_name (AnjutaToken *token)
1567 {
1568         GString *value = g_string_new (NULL);
1569
1570         anjuta_token_foreach_container (token, evaluate_token, value);
1571
1572         /* Return NULL and free data for an empty string */
1573         return g_string_free (value, *(value->str) == '\0');
1574 }
1575
1576 gboolean
1577 anjuta_token_is_empty (AnjutaToken *token)
1578 {
1579         return (token == NULL) || ((token->data.length == 0) && (token->last == NULL) && (token->children == NULL));
1580 }
1581
1582
1583 /* Other functions
1584  *---------------------------------------------------------------------------*/
1585
1586 gboolean
1587 anjuta_token_compare (AnjutaToken *toka, AnjutaToken *tokb)
1588 {
1589         if (tokb->data.type)
1590         {
1591                 if (tokb->data.type != toka->data.type) return FALSE;
1592         }
1593
1594         if (tokb->data.type != ANJUTA_TOKEN_NONE)
1595         {
1596                 if (tokb->data.length != 0)
1597                 {
1598                         if (toka->data.length != tokb->data.length) return FALSE;
1599
1600                         if ((toka->data.flags & ANJUTA_TOKEN_CASE_INSENSITIVE)  && (tokb->data.flags & ANJUTA_TOKEN_CASE_INSENSITIVE))
1601                         {
1602                                 if (g_ascii_strncasecmp (toka->data.pos, tokb->data.pos, toka->data.length) != 0) return FALSE;
1603                         }
1604                         else
1605                         {
1606                                 if (strncmp (toka->data.pos, tokb->data.pos, toka->data.length) != 0) return FALSE;
1607                         }
1608                 }
1609         }
1610
1611         if (tokb->data.flags & ANJUTA_TOKEN_PUBLIC_FLAGS)
1612         {
1613                 if ((toka->data.flags & tokb->data.flags & ANJUTA_TOKEN_PUBLIC_FLAGS) == 0)
1614                         return FALSE;
1615         }
1616
1617         return TRUE;
1618 }
1619
1620 void
1621 anjuta_token_dump (AnjutaToken *token)
1622 {
1623         if (token == NULL) return;
1624
1625         anjuta_token_dump_child (token, 0, 0);
1626 }
1627
1628 void
1629 anjuta_token_dump_link (AnjutaToken *token)
1630 {
1631         AnjutaToken *last = token;
1632
1633         while (last->last != NULL) last = last->last;
1634
1635         for (; token != last; token = anjuta_token_next (token))
1636         {
1637                 anjuta_token_show (token, 0, 0);
1638         }
1639 }
1640
1641 gboolean
1642 anjuta_token_check (AnjutaToken *token)
1643 {
1644         if ((token->children != NULL) && (token->last != NULL))
1645         {
1646                 anjuta_token_show (token, 0, 0);
1647                 fprintf(stderr, "Error: Previous token has both non NULL children and last\n");
1648
1649                 return FALSE;
1650         }
1651
1652         if (token->children != NULL)
1653         {
1654                 AnjutaToken *child;
1655
1656                 for (child = token->children; child != NULL; child = child->next)
1657                 {
1658                         if (!anjuta_token_check_child (child, token)) return FALSE;
1659                 }
1660         }
1661
1662         if (token->last != NULL)
1663         {
1664                 AnjutaToken *child;
1665
1666                 for (child = anjuta_token_next (token); child != NULL; child = anjuta_token_next (child))
1667                 {
1668                         if (!anjuta_token_check (child)) return FALSE;
1669                         if (child == token->last) break;
1670                 }
1671         }
1672
1673         return TRUE;
1674 }
1675
1676 /* Constructor & Destructor
1677  *---------------------------------------------------------------------------*/
1678
1679 AnjutaToken *anjuta_token_new_string (AnjutaTokenType type, const gchar *value)
1680 {
1681         AnjutaToken *token;
1682
1683         if (value == NULL)
1684         {
1685                 token = anjuta_token_new_static (type, NULL);
1686         }
1687         else
1688         {
1689                 token = g_slice_new0 (AnjutaToken);
1690                 token->data.type = type  & ANJUTA_TOKEN_TYPE;
1691                 token->data.flags = type & ANJUTA_TOKEN_FLAGS;
1692                 token->data.pos = g_strdup (value);
1693                 token->data.length = strlen (value);
1694         }
1695
1696         return token;
1697 }
1698
1699 AnjutaToken*
1700 anjuta_token_new_string_len (AnjutaTokenType type, gchar *value, gsize length)
1701 {
1702         AnjutaToken *token;
1703
1704         if (value == NULL)
1705         {
1706                 token = anjuta_token_new_static (type, NULL);
1707         }
1708         else
1709         {
1710                 token = g_slice_new0 (AnjutaToken);
1711                 token->data.type = type  & ANJUTA_TOKEN_TYPE;
1712                 token->data.flags = type & ANJUTA_TOKEN_FLAGS;
1713                 token->data.pos = value;
1714                 token->data.length = length;
1715         }
1716
1717         return token;
1718 }
1719
1720 AnjutaToken *
1721 anjuta_token_new_static_len (gint type, const gchar *pos, gsize length)
1722 {
1723         AnjutaToken *token;
1724
1725         token = g_slice_new0 (AnjutaToken);
1726         token->data.type = type  & ANJUTA_TOKEN_TYPE;
1727         token->data.flags = (type & ANJUTA_TOKEN_FLAGS) | ANJUTA_TOKEN_STATIC;
1728         token->data.pos = (gchar *)pos;
1729         token->data.length = length;
1730
1731         return token;
1732 };
1733
1734 AnjutaToken *anjuta_token_new_static (AnjutaTokenType type, const char *value)
1735 {
1736         return anjuta_token_new_static_len (type, value, value == NULL ? 0 : strlen (value));
1737 }
1738
1739 AnjutaToken*
1740 anjuta_token_free_children (AnjutaToken *token)
1741 {
1742         AnjutaToken *child;
1743         AnjutaToken *last;
1744
1745         if (token == NULL) return NULL;
1746
1747         for (child = token->children; child != NULL; child = token->children)
1748         {
1749                 anjuta_token_free (child);
1750         }
1751         token->children = NULL;
1752
1753         if (token->last != NULL)
1754         {
1755                 last = token->last;
1756                 for (child = anjuta_token_next (token); child != NULL; child = anjuta_token_next (token))
1757                 {
1758                         anjuta_token_free (child);
1759                         if (child == last) break;
1760                 }
1761         }
1762         token->last = NULL;
1763
1764         return token;
1765 }
1766
1767 static void
1768 free_token (AnjutaToken *token, gpointer user_data)
1769 {
1770         anjuta_token_unlink_token (token);
1771         if ((token->data.pos != NULL) && !(token->data.flags & ANJUTA_TOKEN_STATIC))
1772         {
1773                 g_free (token->data.pos);
1774         }
1775         g_slice_free (AnjutaToken, token);
1776 }
1777
1778
1779 AnjutaToken*
1780 anjuta_token_free (AnjutaToken *token)
1781 {
1782         AnjutaToken *next;
1783
1784         if (token == NULL) return NULL;
1785
1786         next = anjuta_token_foreach_post_order (token, free_token, NULL);
1787
1788         return next;
1789 }