src/files.c

   1 /*
   2  * Grace - GRaphing, Advanced Computation and Exploration of data
   3  *
   4  * Home page: http://plasma-gate.weizmann.ac.il/Grace/
   5  *
   6  * Copyright (c) 1991-1995 Paul J Turner, Portland, OR
   7  * Copyright (c) 1996-2006 Grace Development Team
   8  *
   9  * Maintained by Evgeny Stambulchik
  10  *
  11  *
  12  *                           All Rights Reserved
  13  *
  14  *    This program is free software; you can redistribute it and/or modify
  15  *    it under the terms of the GNU General Public License as published by
  16  *    the Free Software Foundation; either version 2 of the License, or
  17  *    (at your option) any later version.
  18  *
  19  *    This program is distributed in the hope that it will be useful,
  20  *    but WITHOUT ANY WARRANTY; without even the implied warranty of
  21  *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  22  *    GNU General Public License for more details.
  23  *
  24  *    You should have received a copy of the GNU General Public License
  25  *    along with this program; if not, write to the Free Software
  26  *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  27  */
  28
  29 /*
  30  *
  31  * read data files
  32  *
  33  */
  34
  35 #include <config.h>
  36
  37 #include <stdio.h>
  38 #include <stdlib.h>
  39 #include <unistd.h>
  40 #include <string.h>
  41 #include <sys/types.h>
  42 #include <sys/stat.h>
  43 #include <sys/time.h>
  44 #include <errno.h>
  45 #ifdef HAVE_SYS_SELECT_H
  46 #  include <sys/select.h>
  47 #endif
  48 #ifdef HAVE_FCNTL_H
  49 #  include <fcntl.h>
  50 #endif
  51
  52 #include "graceapp.h"
  53 #include "utils.h"
  54 #include "files.h"
  55 #include "ssdata.h"
  56 #include "core_utils.h"
  57
  58 #include "xprotos.h"
  59
  60 #define MAXERR 5
  61
  62 /*
  63  * number of rows to allocate for each call to realloc
  64  */
  65 #define BUFSIZE  512
  66
  67 /*
  68  * number of bytes in each line chunk
  69  * (should be related to system pipe size, typically 4K)
  70  */
  71 #ifndef PIPE_BUF
  72 #  define PIPE_BUF 4096
  73 #endif
  74 #define CHUNKSIZE 2*PIPE_BUF
  75
  76 char *close_input;              /* name of real-time input to close */
  77
  78 struct timeval read_begin = {0l, 0l};   /* used to check too long inputs */
  79
  80 static Input_buffer dummy_ib = {-1, 0, 0, 0, 0, 0, NULL, 0, 0, NULL, 0l};
  81
  82 int nb_rt = 0;                  /* number of real time file descriptors */
  83 Input_buffer *ib_tbl = 0;       /* table for each open input */
  84 int ib_tblsize = 0;             /* number of elements in ib_tbl */
  85
  86 static int time_spent(void);
  87 static int expand_ib_tbl(int delay);
  88 static int expand_line_buffer(char **adrBuf, int *ptrSize, char **adrPtr);
  89 static int reopen_real_time_input(GraceApp *gr, Input_buffer *ib);
  90 static int read_real_time_lines(Input_buffer *ib);
  91 static int process_complete_lines(GraceApp *gapp, Input_buffer *ib);
  92
  93 static int read_long_line(FILE *fp, char **linebuf, int *buflen);
  94
  95 /*
  96  * part of the time sliced already spent in milliseconds
  97  */
  98 static int time_spent(void)
  99 {
 100     struct timeval now;
 101
 102     gettimeofday(&now, NULL);
 103
 104     return 1000 * (now.tv_sec - read_begin.tv_sec)
 105         + (now.tv_usec - read_begin.tv_usec) / 1000;
 106
 107 }
 108
 109
 110 /*
 111  * expand the table of monitored real time inputs
 112  */
 113 static int expand_ib_tbl(int delay)
 114 {
 115     int i, new_size;
 116     Input_buffer *new_tbl;
 117
 118     new_size = (ib_tblsize > 0) ? 2*ib_tblsize : 5;
 119     new_tbl  = xcalloc(new_size, sizeof(Input_buffer));
 120     if (new_tbl == NULL) {
 121         return RETURN_FAILURE;
 122     }
 123
 124     for (i = 0; i < new_size; i++) {
 125         new_tbl[i] = (i < ib_tblsize) ? ib_tbl[i] : dummy_ib;
 126         new_tbl[i].delay = delay;
 127     }
 128
 129     if (ib_tblsize > 0) {
 130         xfree((void *) ib_tbl);
 131     }
 132     ib_tbl  = new_tbl;
 133     ib_tblsize = new_size;
 134
 135     return RETURN_SUCCESS;
 136 }
 137
 138
 139 /*
 140  * expand a line buffer
 141  */
 142 static int expand_line_buffer(char **adrBuf, int *ptrSize, char **adrPtr)
 143 {
 144     char *newbuf;
 145     int   newsize;
 146
 147     newsize = *ptrSize + CHUNKSIZE;
 148     newbuf = xmalloc(newsize);
 149     if (newbuf == 0) {
 150         return RETURN_FAILURE;
 151     }
 152
 153     if (*ptrSize == 0) {
 154         /* this is the first time through */
 155         if (adrPtr) {
 156             *adrPtr = newbuf;
 157         }
 158     } else {
 159         /* we are expanding an existing line */
 160         strncpy(newbuf, *adrBuf, *ptrSize);
 161         if (adrPtr) {
 162             *adrPtr += newbuf - *adrBuf;
 163         }
 164         xfree(*adrBuf);
 165     }
 166
 167     *adrBuf  = newbuf;
 168     *ptrSize = newsize;
 169
 170     return RETURN_SUCCESS;
 171 }
 172
 173
 174 /*
 175  * reopen an Input_buffer (surely a fifo)
 176  */
 177 static int reopen_real_time_input(GraceApp *gapp, Input_buffer *ib)
 178 {
 179     int fd;
 180     char buf[256];
 181
 182     /* in order to avoid race conditions (broken pipe on the write
 183        side), we open a new file descriptor before closing the
 184        existing one */
 185     fd = open(ib->name, O_RDONLY | O_NONBLOCK);
 186     if (fd < 0) {
 187         sprintf(buf, "Can't reopen real time input %s", ib->name);
 188         errmsg(buf);
 189         unregister_real_time_input(ib->name);
 190         return RETURN_FAILURE;
 191     }
 192
 193 #ifndef NONE_GUI
 194     xunregister_rti(ib);
 195 #endif
 196
 197     /* swapping the file descriptors */
 198     close(ib->fd);
 199     ib->fd = fd;
 200
 201 #ifndef NONE_GUI
 202     xregister_rti(ib);
 203 #endif
 204
 205     return RETURN_SUCCESS;
 206
 207 }
 208
 209
 210 /*
 211  * unregister a file descriptor no longer monitored
 212  * (since Input_buffer structures dedicated to static inputs
 213  *  are not kept in the table, it is not an error to unregister
 214  *  an input not already registered)
 215  */
 216 void unregister_real_time_input(const char *name)
 217 {
 218     Input_buffer *ib;
 219     int           l1, l2;
 220
 221     l1 = strlen(name);
 222
 223     nb_rt = 0;
 224     for (ib = ib_tbl; ib < ib_tbl + ib_tblsize; ib++) {
 225         l2 = (ib->name == NULL) ? -1 : strlen(ib->name);
 226         if (l1 == l2 && strcmp (name, ib->name) == 0) {
 227             /* name is usually the same pointer as ib->name so we cannot */
 228             /* free the string and output the message using name afterwards */
 229 #ifndef NONE_GUI
 230             xunregister_rti(ib);
 231 #endif
 232             close(ib->fd);
 233             ib->fd = -1;
 234             xfree(ib->name);
 235             ib->name = NULL;
 236         } else
 237         if (l2 > 0) {
 238             /* this descriptor (if not dummy!) is still in use */
 239             nb_rt++;
 240         }
 241     }
 242 }
 243
 244 /*
 245  * register a file descriptor for monitoring
 246  */
 247 int register_real_time_input(GraceApp *gapp, int fd, const char *name, int reopen)
 248 {
 249     Input_buffer *ib;
 250     char buf[256];
 251
 252     /* some safety checks */
 253     if (fd < 0) {
 254         sprintf(buf, "%s : internal error, wrong file descriptor", name);
 255         errmsg(buf);
 256         return RETURN_FAILURE;
 257     }
 258
 259 #ifdef HAVE_FCNTL
 260     if (fcntl(fd, F_GETFL) & O_WRONLY) {
 261         fprintf(stderr,
 262                 "Descriptor %d not open for reading\n",
 263                 fd);
 264         return RETURN_FAILURE;
 265     }
 266 #endif
 267
 268     /* remove previous entry for the same set if any */
 269     unregister_real_time_input(name);
 270
 271     /* find an empty slot in the table */
 272     for (ib = ib_tbl; ib < ib_tbl + ib_tblsize; ib++) {
 273         if (ib->fd == fd) {
 274             sprintf(buf, "%s : internal error, file descriptor already in use",
 275                     name);
 276             errmsg(buf);
 277             return RETURN_FAILURE;
 278         } else if (ib->fd < 0) {
 279             break;
 280         }
 281     }
 282
 283     if (ib == ib_tbl + ib_tblsize) {
 284         /* the table was full, we expand it */
 285         int old_size = ib_tblsize;
 286         if (expand_ib_tbl(gapp->rt->timer_delay) != RETURN_SUCCESS) {
 287             return RETURN_FAILURE;
 288         }
 289         ib = ib_tbl + old_size;
 290     }
 291
 292     /* we keep the current buffer (even if 0),
 293        and only say everything is available */
 294     ib->fd     = fd;
 295     ib->errors = 0;
 296     ib->lineno = 0;
 297     ib->zeros  = 0;
 298     ib->reopen = reopen;
 299     ib->name   = copy_string(ib->name, name);
 300     ib->used   = 0;
 301 #ifndef NONE_GUI
 302     xregister_rti (ib);
 303 #endif
 304
 305     nb_rt++;
 306
 307     return RETURN_SUCCESS;
 308 }
 309
 310 /*
 311  * read a real-time line (but do not process it)
 312  */
 313 static int read_real_time_lines(Input_buffer *ib)
 314 {
 315     char *cursor;
 316     int   available, nbread;
 317     char buf[256];
 318
 319     cursor     = ib->buf  + ib->used;
 320     available  = ib->size - ib->used;
 321
 322     /* have we enough space to store the characters ? */
 323     if (available < 2) {
 324         if (expand_line_buffer(&(ib->buf), &(ib->size), &cursor)
 325             != RETURN_SUCCESS) {
 326             return RETURN_FAILURE;
 327         }
 328         available = ib->buf + ib->size - cursor;
 329     }
 330
 331     /* read as much as possible */
 332     nbread = read(ib->fd, (void *) cursor, available - 1);
 333
 334     if (nbread < 0) {
 335         sprintf(buf, "%s : read error on real time input",
 336                 ib->name);
 337         errmsg(buf);
 338         return RETURN_FAILURE;
 339     } else {
 340         if (nbread == 0) {
 341             ib->zeros++;
 342         } else {
 343             ib->zeros = 0;
 344             ib->used += nbread;
 345             ib->buf[ib->used] = '\0';
 346         }
 347     }
 348
 349     return RETURN_SUCCESS;
 350 }
 351
 352
 353 /*
 354  * process complete lines that have already been read
 355  */
 356 static int process_complete_lines(GraceApp *gapp, Input_buffer *ib)
 357 {
 358     int line_corrupted;
 359     char *begin_of_line, *end_of_line;
 360     char buf[256];
 361
 362     if (ib->used <= 0) {
 363         return RETURN_SUCCESS;
 364     }
 365
 366     end_of_line = NULL;
 367     do {
 368         /* loop over the embedded lines */
 369         begin_of_line  = (end_of_line == NULL) ? ib->buf : (end_of_line + 1);
 370         end_of_line    = begin_of_line;
 371         line_corrupted = 0;
 372         while (end_of_line != NULL && *end_of_line != '\n') {
 373             /* trying to find a complete line */
 374             if (end_of_line == ib->buf + ib->used) {
 375                 end_of_line = NULL;
 376             } else {
 377                 if (*end_of_line == '\0') {
 378                     line_corrupted = 1;
 379                 }
 380                 ++end_of_line;
 381             }
 382         }
 383
 384         if (end_of_line != NULL) {
 385             /* we have a whole line */
 386
 387             ++(ib->lineno);
 388             *end_of_line = '\0';
 389             close_input = NULL;
 390
 391             if (line_corrupted ||
 392                 graal_parse_line(grace_get_graal(gapp->grace),
 393                     begin_of_line, gproject_get_top(gapp->gp)) != RETURN_SUCCESS) {
 394                 sprintf(buf, "Error at line %d", ib->lineno);
 395                 errmsg(buf);
 396                 ++(ib->errors);
 397                 if (ib->errors > MAXERR) {
 398
 399 #ifndef NONE_GUI
 400                     /* this prevents from being called recursively by
 401                        the inner X loop of yesno */
 402                     xunregister_rti(ib);
 403 #endif
 404                     if (yesno("Lots of errors, abort?", NULL, NULL, NULL)) {
 405                         close_input = copy_string(close_input, "");
 406                     }
 407 #ifndef NONE_GUI
 408                     xregister_rti(ib);
 409 #endif
 410                     ib->errors = 0;
 411
 412                 }
 413             }
 414
 415             if (close_input != NULL) {
 416                 /* something should be closed */
 417                 if (close_input[0] == '\0') {
 418                     unregister_real_time_input(ib->name);
 419                 } else {
 420                     unregister_real_time_input(close_input);
 421                 }
 422
 423                 xfree(close_input);
 424                 close_input = NULL;
 425
 426                 if (ib->fd < 0) {
 427                     /* we have closed ourselves */
 428                     return RETURN_SUCCESS;
 429                 }
 430
 431             }
 432
 433         }
 434
 435     } while (end_of_line != NULL);
 436
 437     if (end_of_line != NULL) {
 438         /* the line has just been processed */
 439         begin_of_line = end_of_line + 1;
 440     }
 441
 442     if (begin_of_line > ib->buf) {
 443         /* move the remaining data to the beginning */
 444         ib->used -= begin_of_line - ib->buf;
 445         memmove(ib->buf, begin_of_line, ib->used);
 446         ib->buf[ib->used] = '\0';
 447
 448     }
 449
 450     return RETURN_SUCCESS;
 451
 452 }
 453
 454 int real_time_under_monitoring(void)
 455 {
 456     return nb_rt > 0;
 457 }
 458
 459 /*
 460  * monitor the set of registered file descriptors for pending input
 461  */
 462 int monitor_input(GraceApp *gapp, Input_buffer *tbl, int tblsize, int no_wait)
 463 {
 464 #ifdef HAVE_SELECT
 465     Input_buffer *ib;
 466     fd_set rfds;
 467     int remaining;
 468     struct timeval timeout;
 469     int highest, first_time, retsel;
 470
 471     /* we don't want to get stuck here, we memorize the start date
 472        and will check we do not exceed our allowed time slice */
 473     gettimeofday(&read_begin, NULL);
 474     first_time    = 1;
 475     retsel        = 1;
 476     while (((time_spent() < tbl->delay) || first_time) && retsel > 0) {
 477
 478         /* register all the monitored descriptors */
 479         highest = -1;
 480         FD_ZERO(&rfds);
 481         for (ib = tbl; ib < tbl + tblsize; ib++) {
 482             if (ib->fd >= 0) {
 483                 FD_SET(ib->fd, &rfds);
 484                 if (ib->fd > highest) {
 485                     highest = ib->fd;
 486                 }
 487             }
 488         }
 489
 490         if (highest < 0) {
 491             /* there's nothing to do */
 492             return RETURN_SUCCESS;
 493         }
 494
 495         if (no_wait) {
 496             /* just check for available data without waiting */
 497             remaining = 0;
 498         } else {
 499             /* wait until data or end of time slice arrive */
 500             remaining = tbl->delay - time_spent();
 501             if (remaining < 0) {
 502                 remaining = 0;
 503             }
 504         }
 505         timeout.tv_sec = remaining / 1000;
 506         timeout.tv_usec = 1000l * (remaining % 1000);
 507         retsel = select(highest + 1, &rfds, NULL, NULL, &timeout);
 508
 509         for (ib = tbl;
 510              ((time_spent() < tbl->delay) || first_time) && ib < tbl + tblsize;
 511              ib++) {
 512             if (ib->fd >= 0 && FD_ISSET(ib->fd, &rfds)) {
 513                 /* there is pending input */
 514                 if (read_real_time_lines(ib) != RETURN_SUCCESS
 515                     || process_complete_lines(gapp, ib) != RETURN_SUCCESS) {
 516                     return RETURN_FAILURE;
 517                 }
 518
 519                 if (ib->zeros >= 5) {
 520                     /* we were told five times something happened, but
 521                        never got any byte : we assume the pipe (or
 522                        whatever) has been closed by the peer */
 523                     if (ib->reopen) {
 524                         /* we should reset the input buffer, in case
 525                            the peer also reopens it */
 526                         if (reopen_real_time_input(gapp, ib) != RETURN_SUCCESS) {
 527                             return RETURN_FAILURE;
 528                         }
 529                     } else {
 530                         unregister_real_time_input(ib->name);
 531                     }
 532
 533                     /* we have changed the table, we should end the loop */
 534                     break;
 535                 }
 536             }
 537         }
 538
 539         /* after one pass, we obey timeout */
 540         first_time = 0;
 541     }
 542
 543     return RETURN_SUCCESS;
 544 #else
 545     return RETURN_FAILURE;
 546 #endif
 547 }
 548
 549 /*
 550  * read a line increasing buffer as necessary
 551  */
 552 static int read_long_line(FILE * fp, char **linebuf, int *buflen)
 553 {
 554     char *cursor;
 555     int  available;
 556     int  nbread, retval;
 557
 558     cursor    = *linebuf;
 559     available = *buflen;
 560     retval    = RETURN_FAILURE;
 561     do {
 562         /* do we have enough space to store the characters ? */
 563         if (available < 2) {
 564             if (expand_line_buffer(linebuf, buflen, &cursor)
 565                 != RETURN_SUCCESS) {
 566                 return RETURN_FAILURE;
 567             }
 568         }
 569         available = (int)(*linebuf-cursor) + *buflen;
 570
 571         /* read as much as possible */
 572         if (grace_fgets(cursor, available, fp) == NULL) {
 573             return retval;
 574         }
 575         nbread = strlen(cursor);
 576         if (nbread < 1) {
 577             return retval;
 578         } else {
 579             retval = RETURN_SUCCESS;
 580         }
 581
 582         /* prepare next read */
 583         cursor    += nbread;
 584         available -= nbread;
 585
 586     } while (*(cursor - 1) != '\n');
 587
 588     return retval;
 589 }
 590
 591
 592 /* open a file for write */
 593 FILE *gapp_openw(GraceApp *gapp, const char *fn)
 594 {
 595     struct stat statb;
 596     char buf[GR_MAXPATHLEN + 50];
 597     FILE *retval;
 598
 599     if (!fn || !fn[0]) {
 600         errmsg("No file name given");
 601         return NULL;
 602     } else if (strcmp(fn, "-") == 0 || strcmp(fn, "stdout") == 0) {
 603         return stdout;
 604     } else {
 605         if (stat(fn, &statb) == 0) {
 606             /* check to make sure this is a file and not a dir */
 607             if (S_ISREG(statb.st_mode)) {
 608                 sprintf(buf, "Overwrite %s?", fn);
 609                 if (!yesno(buf, NULL, NULL, NULL)) {
 610                     return NULL;
 611                 }
 612             } else {
 613                 sprintf(buf, "%s is not a regular file!", fn);
 614                 errmsg(buf);
 615                 return NULL;
 616             }
 617         }
 618         retval = filter_write(gapp, fn);
 619         if (!retval) {
 620             sprintf(buf, "Can't write to file %s, check permissions!", fn);
 621             errmsg(buf);
 622         }
 623         return retval;
 624     }
 625 }
 626
 627 char *gapp_exe_path(GraceApp *gapp, const char *fn)
 628 {
 629     static char buf[GR_MAXPATHLEN], *epath;
 630     char *cp;
 631
 632     if (fn == NULL) {
 633         return NULL;
 634     } else {
 635         cp = strchr(fn, ' ');
 636         if (cp == NULL) {
 637             epath = grace_path(gapp->grace, fn);
 638             strcpy(buf, exe_path_translate(epath));
 639             xfree(epath);
 640             return buf;
 641         } else {
 642             strcpy(buf, fn);
 643             buf[cp - fn] = '\0';
 644             epath = grace_path(gapp->grace, buf);
 645             strcpy(buf, epath);
 646             xfree(epath);
 647             strcat(buf, " ");
 648             strcat(buf, cp);
 649             return exe_path_translate(buf);
 650         }
 651     }
 652 }
 653
 654 /* open a file for read */
 655 FILE *gapp_openr(GraceApp *gapp, const char *fn, int src)
 656 {
 657     struct stat statb;
 658     char *tfn;
 659     char buf[GR_MAXPATHLEN + 50];
 660     FILE *fp;
 661
 662     if (!fn || !fn[0]) {
 663         errmsg("No file name given");
 664         return NULL;
 665     }
 666     switch (src) {
 667     case SOURCE_DISK:
 668         tfn = grace_path(gapp->grace, fn);
 669         if (strcmp(tfn, "-") == 0 || strcmp(tfn, "stdin") == 0) {
 670             xfree(tfn);
 671             return stdin;
 672         } else if (stat(tfn, &statb)) {
 673             sprintf(buf, "Can't stat file %s", tfn);
 674             errmsg(buf);
 675             xfree(tfn);
 676             return NULL;
 677         /* check to make sure this is a file and not a dir */
 678         } else if (!S_ISREG(statb.st_mode)) {
 679             sprintf(buf, "%s is not a regular file", tfn);
 680             errmsg(buf);
 681             xfree(tfn);
 682             return NULL;
 683         } else {
 684             fp = filter_read(gapp, tfn);
 685             xfree(tfn);
 686             return fp;
 687         }
 688         break;
 689     case SOURCE_PIPE:
 690         tfn = gapp_exe_path(gapp, fn);
 691         fp = popen(tfn, "r");
 692         xfree(tfn);
 693         return fp;
 694         break;
 695     default:
 696         errmsg("Wrong call to gapp_openr()");
 697         return NULL;
 698     }
 699 }
 700
 701 /*
 702  * close either a pipe or a file pointer
 703  *
 704  */
 705 void gapp_close(FILE *fp)
 706 {
 707     if (fp == stdin || fp == stderr || fp == stdout) {
 708         return;
 709     }
 710     if (pclose(fp) == -1) {
 711         fclose(fp);
 712     }
 713 }
 714
 715 FILE *gapp_tmpfile(char *templateval)
 716 {
 717     FILE *fp;
 718 #if defined(HAVE_MKSTEMP) && defined(HAVE_FDOPEN)
 719     int fd;
 720
 721     fd = mkstemp(templateval);
 722     if (fd < 0) {
 723         fp = NULL;
 724     } else {
 725         fp = fdopen(fd, "wb");
 726     }
 727 #else
 728     tmpnam(templateval);
 729     fp = fopen(templateval, "wb");
 730 #endif
 731
 732     if (!fp) {
 733         errmsg("Can't open temporary file");
 734     }
 735
 736     return fp;
 737 }
 738
 739
 740 int uniread(Quark *pr, FILE *fp,
 741     DataParser parse_cb, DataStore store_cb, void *udata)
 742 {
 743     int nrows, nrows_allocated;
 744     int ok, readerror;
 745     Quark *q = NULL;
 746     char *linebuf = NULL;
 747     int linebuflen = 0;
 748     int linecount;
 749
 750     linecount = 0;
 751     readerror = 0;
 752     nrows = 0;
 753     nrows_allocated = 0;
 754
 755     ok = TRUE;
 756     while (ok) {
 757         char *s;
 758         int maybe_data;
 759         int ncols, nncols, nscols;
 760         int *formats;
 761
 762         if (read_long_line(fp, &linebuf, &linebuflen) == RETURN_SUCCESS) {
 763             linecount++;
 764             s = linebuf;
 765
 766             /* skip leading whitespaces */
 767             while (*s == ' ' || *s == '\t') {
 768                 s++;
 769             }
 770
 771             /* skip comments */
 772             if (*s == '#') {
 773                 continue;
 774             }
 775
 776             /*     EOL           EOD    */
 777             if (*s == '\n' || *s == '\0') {
 778                 maybe_data = FALSE;
 779             } else
 780             if (parse_cb && parse_cb(s, udata) == RETURN_SUCCESS) {
 781                 maybe_data = FALSE;
 782             } else {
 783                 maybe_data = TRUE;
 784             }
 785         } else {
 786             ok = FALSE;
 787             maybe_data = FALSE;
 788         }
 789
 790         if (maybe_data) {
 791             if (!nrows) {
 792                 /* parse the data line */
 793                 if (parse_ss_row(pr, s, &nncols, &nscols, &formats) != RETURN_SUCCESS) {
 794                     errmsg("Can't parse data");
 795                     xfree(linebuf);
 796                     return RETURN_FAILURE;
 797                 }
 798                 ncols = nncols + nscols;
 799
 800                 /* init the SSD */
 801                 q = gapp_ssd_new(pr);
 802                 if (!q || ssd_set_ncols(q, ncols, formats) != RETURN_SUCCESS) {
 803                     errmsg("Malloc failed in uniread()");
 804                     quark_free(q);
 805                     xfree(formats);
 806                     xfree(linebuf);
 807                     return RETURN_FAILURE;
 808                 }
 809                 xfree(formats);
 810             }
 811
 812             if (nrows >= nrows_allocated) {
 813                 if (!nrows_allocated) {
 814                     nrows_allocated = BUFSIZE;
 815                 } else {
 816                     nrows_allocated *= 2;
 817                 }
 818                 if (ssd_set_nrows(q, nrows_allocated) != RETURN_SUCCESS) {
 819                     errmsg("Malloc failed in uniread()");
 820                     quark_free(q);
 821                     xfree(linebuf);
 822                     return RETURN_FAILURE;
 823                 }
 824             }
 825
 826             if (insert_data_row(q, nrows, s) != RETURN_SUCCESS) {
 827                 char tbuf[128];
 828
 829                 sprintf(tbuf, "Error parsing line %d, skipped", linecount);
 830                 errmsg(tbuf);
 831                 readerror++;
 832                 if (readerror > MAXERR) {
 833                     if (yesno("Lots of errors, abort?", NULL, NULL, NULL)) {
 834                         quark_free(q);
 835                         xfree(linebuf);
 836                         return RETURN_FAILURE;
 837                     } else {
 838                         readerror = 0;
 839                     }
 840                 }
 841             } else {
 842                 nrows++;
 843             }
 844         } else
 845         if (nrows) {
 846             /* free excessive storage */
 847             ssd_set_nrows(q, nrows);
 848
 849             /* store accumulated data */
 850             if (store_cb && store_cb(q, udata) != RETURN_SUCCESS) {
 851                 quark_free(q);
 852                 xfree(linebuf);
 853                 return RETURN_FAILURE;
 854             }
 855
 856             /* reset state registers */
 857             nrows = 0;
 858             nrows_allocated = 0;
 859             readerror = 0;
 860         }
 861     }
 862
 863     xfree(linebuf);
 864
 865     return RETURN_SUCCESS;
 866 }
 867
 868
 869 typedef struct {
 870     char *label;
 871     int load_type;
 872     int settype;
 873 } ascii_data;
 874
 875 static int store_cb(Quark *q, void *udata)
 876 {
 877     ascii_data *adata = (ascii_data *) udata;
 878
 879     /* label the SSD */
 880     quark_idstr_set(q, adata->label);
 881
 882     return store_data(q, adata->load_type, adata->settype);
 883 }
 884
 885 int getdata(Quark *gr, char *fn, int settype, int load_type)
 886 {
 887     FILE *fp;
 888     int retval;
 889     GraceApp *gapp = gapp_from_quark(gr);
 890     ascii_data adata;
 891
 892     fp = gapp_openr(gapp, fn, SOURCE_DISK);
 893     if (fp == NULL) {
 894         return RETURN_FAILURE;
 895     }
 896
 897     adata.label = fn;
 898     adata.load_type = load_type;
 899     adata.settype = settype;
 900
 901     retval = uniread(gr, fp, NULL, store_cb, &adata);
 902
 903     gapp_close(fp);
 904
 905     if (load_type != LOAD_BLOCK) {
 906         autoscale_graph(gr, gapp->rt->autoscale_onread);
 907     }
 908
 909     return retval;
 910 }
 911
 912
 913 int write_ssd(const Quark *ssd, unsigned int ncols, const int *cols, FILE *fp)
 914 {
 915     char *sep = "\t";
 916     unsigned int nrows = ssd_get_nrows(ssd), i, j;
 917
 918     unsigned int prec = project_get_prec(get_parent_project(ssd));
 919
 920     fputs("# ", fp);
 921     for (j = 0; j < ncols; j++) {
 922         char *lab = ssd_get_col_label(ssd, cols[j]);
 923         if (j != 0) {
 924             fputs(sep, fp);
 925         }
 926         fputs(lab ? lab:"?", fp);
 927     }
 928     fputs("\n", fp);
 929
 930     for (i = 0; i < nrows; i++) {
 931         for (j = 0; j < ncols; j++) {
 932             unsigned int col = cols[j];
 933             ss_column *scol = ssd_get_col(ssd, col);
 934             if (!scol) {
 935                 return RETURN_FAILURE;
 936             }
 937
 938             if (j != 0) {
 939                 fputs(sep, fp);
 940             }
 941
 942             if (scol->format == FFORMAT_STRING) {
 943                 char **s = ((char **) scol->data);
 944                 fprintf(fp, " \"%s\"", escapequotes(s[i]));
 945             } else {
 946                 double *x = ((double *) scol->data);
 947                 fprintf(fp, "%.*g", prec, x[i]);
 948             }
 949         }
 950         fputs("\n", fp);
 951     }
 952     fputs("\n", fp);
 953     return RETURN_SUCCESS;
 954 }
 955
 956 #if 0
 957 /*
 958  * read data to the set from a file overriding the current contents
 959  */
 960 int update_set_from_file(Quark *pset)
 961 {
 962     int retval;
 963     Dataset *dsp;
 964
 965     dsp = set_get_dataset(pset);
 966
 967     if (!dsp) {
 968         retval = RETURN_FAILURE;
 969     } else {
 970         FILE *fp;
 971         RunTime *rt = rt_from_quark(pset);
 972
 973         fp = gapp_openr(gapp_from_quark(pset), dsp->hotfile, dsp->hotsrc);
 974
 975         killsetdata(pset);
 976         rt->curtype = set_get_type(pset);
 977         retval = uniread(get_parent_project(pset), fp, LOAD_SINGLE, dsp->hotfile);
 978
 979         gapp_close(fp);
 980     }
 981
 982     return retval;
 983 }
 984 #endif
 985
 986 static int project_cb(Quark *pr, int etype, void *data)
 987 {
 988     if (etype == QUARK_ETYPE_MODIFY) {
 989 #if 0
 990         /* TODO: */
 991         if ((dirtystate > SOME_LIMIT) ||
 992             (current_time - autosave_time > ANOTHER_LIMIT) ) {
 993             autosave();
 994         }
 995 #endif
 996     }
 997
 998     return RETURN_SUCCESS;
 999 }
1000
1001
1002 static int project_hook(Quark *q, void *udata, QTraverseClosure *closure)
1003 {
1004     switch (quark_fid_get(q)) {
1005     case QFlavorProject:
1006         quark_cb_add(q, project_cb, NULL);
1007         break;
1008     case QFlavorSSD:
1009         quark_cb_add(q, kill_ssd_cb, NULL);
1010         break;
1011     }
1012
1013     return TRUE;
1014 }
1015
1016 GProject *load_any_project(GraceApp *gapp, const char *fn)
1017 {
1018     GProject *gp;
1019
1020     /* FIXME: A temporary hack */
1021     if (fn && strstr(fn, ".xgr")) {
1022         gp = load_xgr_project(gapp, fn);
1023     } else {
1024         gp = load_agr_project(gapp, fn);
1025     }
1026
1027     if (gp) {
1028         quark_traverse(gproject_get_top(gp), project_hook, NULL);
1029     }
1030
1031     return gp;
1032 }
1033
1034 static GProject *load_project_file(GraceApp *gapp, const char *fn, int as_template)
1035 {
1036     GProject *gp;
1037     Quark *project, *gr, **graphs;
1038     int i, ngraphs;
1039     AMem *amem;
1040
1041     gp = load_any_project(gapp, fn);
1042     if (!gp) {
1043         errmsg("Failed loading project file");
1044         return NULL;
1045     }
1046
1047     gapp->rt->print_file[0] = '\0';
1048
1049     if (as_template) {
1050         grfile_free(gp->grf);
1051         gp->grf = NULL;
1052     }
1053
1054     gapp_add_gproject(gapp, gp);
1055
1056     project = gproject_get_top(gp);
1057
1058     amem = quark_get_amem(project);
1059
1060     /* Set undo limit of 128MB */
1061     amem_set_undo_limit(amem, 0x8000000L);
1062     /* Get initial memory snapshot */
1063     amem_snapshot(amem);
1064
1065     /* try to switch to the first active graph */
1066     ngraphs = project_get_graphs(project, &graphs);
1067     for (i = 0; i < ngraphs; i++) {
1068         gr = graphs[i];
1069         if (select_graph(gr) == RETURN_SUCCESS) {
1070             break;
1071         }
1072     }
1073     xfree(graphs);
1074
1075     return gp;
1076 }
1077
1078 int load_project(GraceApp *gapp, char *fn)
1079 {
1080     unsigned int i;
1081     int is_open;
1082     char *epath;
1083     char *docname;
1084     GProject *gp;
1085
1086     is_open = FALSE;
1087     epath = grace_path(gapp->grace, fn);
1088
1089     for (i = 0; i < gapp->gpcount; i++) {
1090         docname = gproject_get_docname(gapp->gplist[i]);
1091         if (epath && docname && !strcmp(epath, docname)) {
1092             is_open = TRUE;
1093             break;
1094         }
1095     }
1096     xfree(epath);
1097
1098     if (is_open && gapp->gplist[i] != gapp->gp) {
1099         gapp_set_active_gproject(gapp, gapp->gplist[i]);
1100     }
1101
1102     if (is_open) {
1103         return RETURN_SUCCESS;
1104     }
1105
1106     gp = load_project_file(gapp, fn, FALSE);
1107     if (gp) {
1108         gapp_set_active_gproject(gapp, gp);
1109
1110         return RETURN_SUCCESS;
1111     } else {
1112         return RETURN_FAILURE;
1113     }
1114 }
1115
1116 int revert_project(GraceApp *gapp, GProject *gp)
1117 {
1118     char *docname;
1119     GProject *rgp = NULL;
1120
1121     docname = gproject_get_docname(gp);
1122     if (docname) {
1123         rgp = load_project_file(gapp, docname, FALSE);
1124     } else {
1125         rgp = load_project_file(gapp, "templates/Default.xgr", TRUE);
1126     }
1127
1128     if (rgp) {
1129         gapp_set_gproject_id(gapp, rgp, gapp_get_gproject_id(gapp, gp));
1130
1131         if (gapp->gp == gp) {
1132             gapp_set_active_gproject(gapp, rgp);
1133         } else {
1134             quark_set_active2(gproject_get_top(rgp), FALSE);
1135         }
1136
1137         gapp_delete_gproject(gapp, gp);
1138
1139         return RETURN_SUCCESS;
1140     } else {
1141         return RETURN_FAILURE;
1142     }
1143
1144 }
1145
1146 int new_project(GraceApp *gapp, char *pr_template)
1147 {
1148     GProject *gp = NULL;
1149     char *s;
1150
1151     if (string_is_empty(pr_template)) {
1152         gp = load_project_file(gapp, "templates/Default.xgr", TRUE);
1153     } else if (pr_template[0] == '/') {
1154         gp = load_project_file(gapp, pr_template, TRUE);
1155     } else {
1156         s = xmalloc(strlen("templates/") + strlen(pr_template) + 1);
1157         if (s) {
1158             sprintf(s, "templates/%s", pr_template);
1159             gp = load_project_file(gapp, s, TRUE);
1160             xfree(s);
1161         }
1162     }
1163
1164     if (gp) {
1165         gapp_set_active_gproject(gapp, gp);
1166
1167         return RETURN_SUCCESS;
1168     } else {
1169         return RETURN_FAILURE;
1170     }
1171 }
1172
1173
1174 int save_project(GProject *gp, char *fn)
1175 {
1176     GrFILE *grf;
1177     Quark *project = gproject_get_top(gp);
1178     GUI *gui = gui_from_quark(project);
1179     int noask_save;
1180     static int save_unsupported = FALSE;
1181     int retval;
1182
1183     if (!project || !fn) {
1184         return RETURN_FAILURE;
1185     }
1186
1187     if (fn && strstr(fn, ".agr")) {
1188         errmsg("Cowardly refusing to overwrite an agr file");
1189         return RETURN_FAILURE;
1190     }
1191     if (!save_unsupported &&
1192         !yesno("The current format may be unsupported by the final release. Continue?",
1193             "Yeah, I'm brave!", NULL, "doc/UsersGuide.html#unsupported_format")) {
1194         return RETURN_FAILURE;
1195     }
1196     save_unsupported = TRUE;
1197
1198     noask_save = gui->noask;
1199     if (strings_are_equal(gproject_get_docname(gp), fn)) {
1200         /* If saving under the same name, don't warn about overwriting */
1201         gui->noask = TRUE;
1202     }
1203
1204     grf = grfile_openw(fn);
1205     if (!grf) {
1206         return RETURN_FAILURE;
1207     }
1208
1209     gui->noask = noask_save;
1210
1211     retval = gproject_save(gp, grf);
1212
1213     grfile_free(grf);
1214
1215     return retval;
1216 }
1217
1218 GProject *load_xgr_project(GraceApp *gapp, const char *fn)
1219 {
1220     GrFILE *grf;
1221     GProject *gp;
1222     char *epath;
1223
1224     epath = grace_path(gapp->grace, fn);
1225     grf = grfile_openr(epath);
1226     xfree(epath);
1227     if (grf == NULL) {
1228         return NULL;
1229     }
1230
1231     gp = gproject_load(gapp->pc, gapp->grace, grf, AMEM_MODEL_LIBUNDO);
1232
1233     grfile_free(grf);
1234
1235     return gp;
1236 }