src/files.c

   1 /*
   2  * Grace - GRaphing, Advanced Computation and Exploration of data
   3  *
   4  * Home page: http://plasma-gate.weizmann.ac.il/Grace/
   5  *
   6  * Copyright (c) 1991-1995 Paul J Turner, Portland, OR
   7  * Copyright (c) 1996-2003 Grace Development Team
   8  *
   9  * Maintained by Evgeny Stambulchik
  10  *
  11  *
  12  *                           All Rights Reserved
  13  *
  14  *    This program is free software; you can redistribute it and/or modify
  15  *    it under the terms of the GNU General Public License as published by
  16  *    the Free Software Foundation; either version 2 of the License, or
  17  *    (at your option) any later version.
  18  *
  19  *    This program is distributed in the hope that it will be useful,
  20  *    but WITHOUT ANY WARRANTY; without even the implied warranty of
  21  *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  22  *    GNU General Public License for more details.
  23  *
  24  *    You should have received a copy of the GNU General Public License
  25  *    along with this program; if not, write to the Free Software
  26  *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  27  */
  28
  29 /*
  30  *
  31  * read data files
  32  *
  33  */
  34
  35 #include <config.h>
  36
  37 #include <stdio.h>
  38 #include <stdlib.h>
  39 #include <unistd.h>
  40 #include <string.h>
  41 #include <sys/types.h>
  42 #include <sys/stat.h>
  43 #include <sys/time.h>
  44 #include <errno.h>
  45 #ifdef HAVE_SYS_SELECT_H
  46 #  include <sys/select.h>
  47 #endif
  48 #ifdef HAVE_FCNTL_H
  49 #  include <fcntl.h>
  50 #endif
  51
  52 #ifdef HAVE_NETCDF
  53 #  include <netcdf.h>
  54 #endif
  55
  56 #include "grace.h"
  57 #include "utils.h"
  58 #include "dicts.h"
  59 #include "files.h"
  60 #include "ssdata.h"
  61 #include "core_utils.h"
  62 #include "parser.h"
  63
  64 #include "protos.h"
  65
  66 #define MAXERR 5
  67
  68 /*
  69  * number of rows to allocate for each call to realloc
  70  */
  71 #define BUFSIZE  512
  72
  73 /*
  74  * number of bytes in each line chunk
  75  * (should be related to system pipe size, typically 4K)
  76  */
  77 #ifndef PIPE_BUF
  78 #  define PIPE_BUF 4096
  79 #endif
  80 #define CHUNKSIZE 2*PIPE_BUF
  81
  82 char *close_input;              /* name of real-time input to close */
  83
  84 struct timeval read_begin = {0l, 0l};   /* used to check too long inputs */
  85
  86 static Input_buffer dummy_ib = {-1, 0, 0, 0, 0, 0, NULL, 0, 0, NULL, 0l};
  87
  88 int nb_rt = 0;                  /* number of real time file descriptors */
  89 Input_buffer *ib_tbl = 0;       /* table for each open input */
  90 int ib_tblsize = 0;             /* number of elements in ib_tbl */
  91
  92 static int time_spent(void);
  93 static int expand_ib_tbl(int delay);
  94 static int expand_line_buffer(char **adrBuf, int *ptrSize, char **adrPtr);
  95 static int reopen_real_time_input(Grace *gr, Input_buffer *ib);
  96 static int read_real_time_lines(Input_buffer *ib);
  97 static int process_complete_lines(Input_buffer *ib);
  98
  99 static int read_long_line(FILE *fp, char **linebuf, int *buflen);
 100
 101 static int uniread(Quark *pr, FILE *fp, int load_type, char *label);
 102
 103 /*
 104  * part of the time sliced already spent in milliseconds
 105  */
 106 static int time_spent(void)
 107 {
 108     struct timeval now;
 109
 110     gettimeofday(&now, NULL);
 111
 112     return 1000 * (now.tv_sec - read_begin.tv_sec)
 113         + (now.tv_usec - read_begin.tv_usec) / 1000;
 114
 115 }
 116
 117
 118 /*
 119  * expand the table of monitored real time inputs
 120  */
 121 static int expand_ib_tbl(int delay)
 122 {
 123     int i, new_size;
 124     Input_buffer *new_tbl;
 125
 126     new_size = (ib_tblsize > 0) ? 2*ib_tblsize : 5;
 127     new_tbl  = xcalloc(new_size, sizeof(Input_buffer));
 128     if (new_tbl == NULL) {
 129         return RETURN_FAILURE;
 130     }
 131
 132     for (i = 0; i < new_size; i++) {
 133         new_tbl[i] = (i < ib_tblsize) ? ib_tbl[i] : dummy_ib;
 134         new_tbl[i].delay = delay;
 135     }
 136
 137     if (ib_tblsize > 0) {
 138         xfree((void *) ib_tbl);
 139     }
 140     ib_tbl  = new_tbl;
 141     ib_tblsize = new_size;
 142
 143     return RETURN_SUCCESS;
 144 }
 145
 146
 147 /*
 148  * expand a line buffer
 149  */
 150 static int expand_line_buffer(char **adrBuf, int *ptrSize, char **adrPtr)
 151 {
 152     char *newbuf;
 153     int   newsize;
 154
 155     newsize = *ptrSize + CHUNKSIZE;
 156     newbuf = xmalloc(newsize);
 157     if (newbuf == 0) {
 158         return RETURN_FAILURE;
 159     }
 160
 161     if (*ptrSize == 0) {
 162         /* this is the first time through */
 163         if (adrPtr) {
 164             *adrPtr = newbuf;
 165         }
 166     } else {
 167         /* we are expanding an existing line */
 168         strncpy(newbuf, *adrBuf, *ptrSize);
 169         if (adrPtr) {
 170             *adrPtr += newbuf - *adrBuf;
 171         }
 172         xfree(*adrBuf);
 173     }
 174
 175     *adrBuf  = newbuf;
 176     *ptrSize = newsize;
 177
 178     return RETURN_SUCCESS;
 179 }
 180
 181
 182 /*
 183  * reopen an Input_buffer (surely a fifo)
 184  */
 185 static int reopen_real_time_input(Grace *grace, Input_buffer *ib)
 186 {
 187     int fd;
 188     char buf[256];
 189
 190     /* in order to avoid race conditions (broken pipe on the write
 191        side), we open a new file descriptor before closing the
 192        existing one */
 193     fd = open(ib->name, O_RDONLY | O_NONBLOCK);
 194     if (fd < 0) {
 195         sprintf(buf, "Can't reopen real time input %s", ib->name);
 196         errmsg(buf);
 197         unregister_real_time_input(ib->name);
 198         return RETURN_FAILURE;
 199     }
 200
 201 #ifndef NONE_GUI
 202     xunregister_rti(ib);
 203 #endif
 204
 205     /* swapping the file descriptors */
 206     close(ib->fd);
 207     ib->fd = fd;
 208
 209 #ifndef NONE_GUI
 210     xregister_rti(ib);
 211 #endif
 212
 213     return RETURN_SUCCESS;
 214
 215 }
 216
 217
 218 /*
 219  * unregister a file descriptor no longer monitored
 220  * (since Input_buffer structures dedicated to static inputs
 221  *  are not kept in the table, it is not an error to unregister
 222  *  an input not already registered)
 223  */
 224 void unregister_real_time_input(const char *name)
 225 {
 226     Input_buffer *ib;
 227     int           l1, l2;
 228
 229     l1 = strlen(name);
 230
 231     nb_rt = 0;
 232     for (ib = ib_tbl; ib < ib_tbl + ib_tblsize; ib++) {
 233         l2 = (ib->name == 0) ? -1 : strlen(ib->name);
 234         if (l1 == l2 && strcmp (name, ib->name) == 0) {
 235             /* name is usually the same pointer as ib->name so we cannot */
 236             /* free the string and output the message using name afterwards */
 237 #ifndef NONE_GUI
 238             xunregister_rti(ib);
 239 #endif
 240             close(ib->fd);
 241             ib->fd = -1;
 242             xfree(ib->name);
 243             ib->name = NULL;
 244         } else {
 245             /* this descriptor is still in use */
 246             nb_rt++;
 247         }
 248     }
 249 }
 250
 251 /*
 252  * register a file descriptor for monitoring
 253  */
 254 int register_real_time_input(Grace *grace, int fd, const char *name, int reopen)
 255 {
 256     Input_buffer *ib;
 257     char buf[256];
 258
 259     /* some safety checks */
 260     if (fd < 0) {
 261         sprintf(buf, "%s : internal error, wrong file descriptor", name);
 262         errmsg(buf);
 263         return RETURN_FAILURE;
 264     }
 265
 266 #ifdef HAVE_FCNTL
 267     if (fcntl(fd, F_GETFL) & O_WRONLY) {
 268         fprintf(stderr,
 269                 "Descriptor %d not open for reading\n",
 270                 fd);
 271         return RETURN_FAILURE;
 272     }
 273 #endif
 274
 275     /* remove previous entry for the same set if any */
 276     unregister_real_time_input(name);
 277
 278     /* find an empty slot in the table */
 279     for (ib = ib_tbl; ib < ib_tbl + ib_tblsize; ib++) {
 280         if (ib->fd == fd) {
 281             sprintf(buf, "%s : internal error, file descriptor already in use",
 282                     name);
 283             errmsg(buf);
 284             return RETURN_FAILURE;
 285         } else if (ib->fd < 0) {
 286             break;
 287         }
 288     }
 289
 290     if (ib == ib_tbl + ib_tblsize) {
 291         /* the table was full, we expand it */
 292         int old_size = ib_tblsize;
 293         if (expand_ib_tbl(grace->rt->timer_delay) != RETURN_SUCCESS) {
 294             return RETURN_FAILURE;
 295         }
 296         ib = ib_tbl + old_size;
 297     }
 298
 299     /* we keep the current buffer (even if 0),
 300        and only say everything is available */
 301     ib->fd     = fd;
 302     ib->errors = 0;
 303     ib->lineno = 0;
 304     ib->zeros  = 0;
 305     ib->reopen = reopen;
 306     ib->name   = copy_string(ib->name, name);
 307     ib->used   = 0;
 308 #ifndef NONE_GUI
 309     xregister_rti (ib);
 310 #endif
 311
 312     nb_rt++;
 313
 314     return RETURN_SUCCESS;
 315 }
 316
 317 /*
 318  * read a real-time line (but do not process it)
 319  */
 320 static int read_real_time_lines(Input_buffer *ib)
 321 {
 322     char *cursor;
 323     int   available, nbread;
 324     char buf[256];
 325
 326     cursor     = ib->buf  + ib->used;
 327     available  = ib->size - ib->used;
 328
 329     /* have we enough space to store the characters ? */
 330     if (available < 2) {
 331         if (expand_line_buffer(&(ib->buf), &(ib->size), &cursor)
 332             != RETURN_SUCCESS) {
 333             return RETURN_FAILURE;
 334         }
 335         available = ib->buf + ib->size - cursor;
 336     }
 337
 338     /* read as much as possible */
 339     nbread = read(ib->fd, (void *) cursor, available - 1);
 340
 341     if (nbread < 0) {
 342         sprintf(buf, "%s : read error on real time input",
 343                 ib->name);
 344         errmsg(buf);
 345         return RETURN_FAILURE;
 346     } else {
 347         if (nbread == 0) {
 348             ib->zeros++;
 349         } else {
 350             ib->zeros = 0;
 351             ib->used += nbread;
 352             ib->buf[ib->used] = '\0';
 353         }
 354     }
 355
 356     return RETURN_SUCCESS;
 357 }
 358
 359
 360 /*
 361  * process complete lines that have already been read
 362  */
 363 static int process_complete_lines(Input_buffer *ib)
 364 {
 365     int line_corrupted;
 366     char *begin_of_line, *end_of_line;
 367     char buf[256];
 368
 369     if (ib->used <= 0) {
 370         return RETURN_SUCCESS;
 371     }
 372
 373     end_of_line = NULL;
 374     do {
 375         /* loop over the embedded lines */
 376         begin_of_line  = (end_of_line == NULL) ? ib->buf : (end_of_line + 1);
 377         end_of_line    = begin_of_line;
 378         line_corrupted = 0;
 379         while (end_of_line != NULL && *end_of_line != '\n') {
 380             /* trying to find a complete line */
 381             if (end_of_line == ib->buf + ib->used) {
 382                 end_of_line = NULL;
 383             } else {
 384                 if (*end_of_line == '\0') {
 385                     line_corrupted = 1;
 386                 }
 387                 ++end_of_line;
 388             }
 389         }
 390
 391         if (end_of_line != NULL) {
 392             /* we have a whole line */
 393
 394             ++(ib->lineno);
 395             *end_of_line = '\0';
 396             close_input = NULL;
 397
 398             if (line_corrupted || scanner(begin_of_line)) {
 399                 sprintf(buf, "Error at line %d", ib->lineno);
 400                 errmsg(buf);
 401                 ++(ib->errors);
 402                 if (ib->errors > MAXERR) {
 403
 404 #ifndef NONE_GUI
 405                     /* this prevents from being called recursively by
 406                        the inner X loop of yesno */
 407                     xunregister_rti(ib);
 408 #endif
 409                     if (yesno("Lots of errors, abort?", NULL, NULL, NULL)) {
 410                         close_input = copy_string(close_input, "");
 411                     }
 412 #ifndef NONE_GUI
 413                     xregister_rti(ib);
 414 #endif
 415                     ib->errors = 0;
 416
 417                 }
 418             }
 419
 420             if (close_input != NULL) {
 421                 /* something should be closed */
 422                 if (close_input[0] == '\0') {
 423                     unregister_real_time_input(ib->name);
 424                 } else {
 425                     unregister_real_time_input(close_input);
 426                 }
 427
 428                 xfree(close_input);
 429                 close_input = NULL;
 430
 431                 if (ib->fd < 0) {
 432                     /* we have closed ourselves */
 433                     return RETURN_SUCCESS;
 434                 }
 435
 436             }
 437
 438         }
 439
 440     } while (end_of_line != NULL);
 441
 442     if (end_of_line != NULL) {
 443         /* the line has just been processed */
 444         begin_of_line = end_of_line + 1;
 445     }
 446
 447     if (begin_of_line > ib->buf) {
 448         /* move the remaining data to the beginning */
 449         ib->used -= begin_of_line - ib->buf;
 450         memmove(ib->buf, begin_of_line, ib->used);
 451         ib->buf[ib->used] = '\0';
 452
 453     }
 454
 455     return RETURN_SUCCESS;
 456
 457 }
 458
 459 int real_time_under_monitoring(void)
 460 {
 461     return nb_rt > 0;
 462 }
 463
 464 /*
 465  * monitor the set of registered file descriptors for pending input
 466  */
 467 int monitor_input(Grace *grace, Input_buffer *tbl, int tblsize, int no_wait)
 468 {
 469
 470     Input_buffer *ib;
 471     fd_set rfds;
 472     int remaining;
 473     struct timeval timeout;
 474     int highest, first_time, retsel;
 475
 476     /* we don't want to get stuck here, we memorize the start date
 477        and will check we do not exceed our allowed time slice */
 478     gettimeofday(&read_begin, NULL);
 479     first_time    = 1;
 480     retsel        = 1;
 481     while (((time_spent() < tbl->delay) || first_time) && retsel > 0) {
 482
 483         /* register all the monitored descriptors */
 484         highest = -1;
 485         FD_ZERO(&rfds);
 486         for (ib = tbl; ib < tbl + tblsize; ib++) {
 487             if (ib->fd >= 0) {
 488                 FD_SET(ib->fd, &rfds);
 489                 if (ib->fd > highest) {
 490                     highest = ib->fd;
 491                 }
 492             }
 493         }
 494
 495         if (highest < 0) {
 496             /* there's nothing to do */
 497             return RETURN_SUCCESS;
 498         }
 499
 500         if (no_wait) {
 501             /* just check for available data without waiting */
 502             remaining = 0;
 503         } else {
 504             /* wait until data or end of time slice arrive */
 505             remaining = tbl->delay - time_spent();
 506             if (remaining < 0) {
 507                 remaining = 0;
 508             }
 509         }
 510         timeout.tv_sec = remaining / 1000;
 511         timeout.tv_usec = 1000l * (remaining % 1000);
 512         retsel = select(highest + 1, &rfds, NULL, NULL, &timeout);
 513
 514         for (ib = tbl;
 515              ((time_spent() < tbl->delay) || first_time) && ib < tbl + tblsize;
 516              ib++) {
 517             if (ib->fd >= 0 && FD_ISSET(ib->fd, &rfds)) {
 518                 /* there is pending input */
 519                 if (read_real_time_lines(ib) != RETURN_SUCCESS
 520                     || process_complete_lines(ib) != RETURN_SUCCESS) {
 521                     return RETURN_FAILURE;
 522                 }
 523
 524                 if (ib->zeros >= 5) {
 525                     /* we were told five times something happened, but
 526                        never got any byte : we assume the pipe (or
 527                        whatever) has been closed by the peer */
 528                     if (ib->reopen) {
 529                         /* we should reset the input buffer, in case
 530                            the peer also reopens it */
 531                         if (reopen_real_time_input(grace, ib) != RETURN_SUCCESS) {
 532                             return RETURN_FAILURE;
 533                         }
 534                     } else {
 535                         unregister_real_time_input(ib->name);
 536                     }
 537
 538                     /* we have changed the table, we should end the loop */
 539                     break;
 540                 }
 541             }
 542         }
 543
 544         /* after one pass, we obey timeout */
 545         first_time = 0;
 546     }
 547
 548     return RETURN_SUCCESS;
 549 }
 550
 551 /*
 552  * read a line increasing buffer as necessary
 553  */
 554 static int read_long_line(FILE * fp, char **linebuf, int *buflen)
 555 {
 556     char *cursor;
 557     int  available;
 558     int  nbread, retval;
 559
 560     cursor    = *linebuf;
 561     available = *buflen;
 562     retval    = RETURN_FAILURE;
 563     do {
 564         /* do we have enough space to store the characters ? */
 565         if (available < 2) {
 566             if (expand_line_buffer(linebuf, buflen, &cursor)
 567                 != RETURN_SUCCESS) {
 568                 return RETURN_FAILURE;
 569             }
 570         }
 571         available = (int)(*linebuf-cursor) + *buflen;
 572
 573         /* read as much as possible */
 574         if (grace_fgets(cursor, available, fp) == NULL) {
 575             return retval;
 576         }
 577         nbread = strlen(cursor);
 578         if (nbread < 1) {
 579             return retval;
 580         } else {
 581             retval = RETURN_SUCCESS;
 582         }
 583
 584         /* prepare next read */
 585         cursor    += nbread;
 586         available -= nbread;
 587
 588     } while (*(cursor - 1) != '\n');
 589
 590     return retval;
 591 }
 592
 593
 594 /* open a file for write */
 595 FILE *grace_openw(Grace *grace, char *fn)
 596 {
 597     struct stat statb;
 598     char buf[GR_MAXPATHLEN + 50];
 599     FILE *retval;
 600
 601     if (!fn || !fn[0]) {
 602         errmsg("No file name given");
 603         return NULL;
 604     } else if (strcmp(fn, "-") == 0 || strcmp(fn, "stdout") == 0) {
 605         return stdout;
 606     } else {
 607         if (stat(fn, &statb) == 0) {
 608             /* check to make sure this is a file and not a dir */
 609             if (S_ISREG(statb.st_mode)) {
 610                 sprintf(buf, "Overwrite %s?", fn);
 611                 if (!yesno(buf, NULL, NULL, NULL)) {
 612                     return NULL;
 613                 }
 614             } else {
 615                 sprintf(buf, "%s is not a regular file!", fn);
 616                 errmsg(buf);
 617                 return NULL;
 618             }
 619         }
 620         retval = filter_write(grace, fn);
 621         if (!retval) {
 622             sprintf(buf, "Can't write to file %s, check permissions!", fn);
 623             errmsg(buf);
 624         }
 625         return retval;
 626     }
 627 }
 628
 629 char *grace_path(Grace *grace, char *fn)
 630 {
 631     static char buf[GR_MAXPATHLEN];
 632     struct stat statb;
 633
 634     if (fn == NULL) {
 635         return NULL;
 636     } else {
 637         strcpy(buf, fn);
 638
 639         switch (fn[0]) {
 640         case '/':
 641         case '\0':
 642             return buf;
 643             break;
 644         case '~':
 645             expand_tilde(grace, buf);
 646             return buf;
 647             break;
 648         case '.':
 649             switch (fn[1]) {
 650             case '/':
 651                 return buf;
 652                 break;
 653             case '.':
 654                 if (fn[2] == '/') {
 655                     return buf;
 656                 }
 657                 break;
 658             }
 659         }
 660         /* if we arrived here, the path is relative */
 661         if (stat(buf, &statb) == 0) {
 662             /* ok, we found it */
 663             return buf;
 664         }
 665
 666         /* second try: in .grace/ in the current dir */
 667         strcpy(buf, ".grace/");
 668         strcat(buf, fn);
 669         if (stat(buf, &statb) == 0) {
 670             return buf;
 671         }
 672
 673         /* third try: in .grace/ in the $HOME dir */
 674         strcpy(buf, get_userhome(grace));
 675         strcat(buf, ".grace/");
 676         strcat(buf, fn);
 677         if (stat(buf, &statb) == 0) {
 678             return buf;
 679         }
 680
 681         /* the last attempt: in $GRACE_HOME */
 682         strcpy(buf, get_grace_home(grace));
 683         strcat(buf, "/");
 684         strcat(buf, fn);
 685         if (stat(buf, &statb) == 0) {
 686             return buf;
 687         }
 688
 689         /* giving up... */
 690         strcpy(buf, fn);
 691         return buf;
 692     }
 693 }
 694
 695 char *grace_path2(Grace *grace, const char *prefix, char *fn)
 696 {
 697     char *s, *bufp;
 698
 699     s = copy_string(NULL, prefix);
 700     if (!s) {
 701         return NULL;
 702     }
 703
 704     if (s[strlen(s) - 1] != '/') {
 705         s = concat_strings(s, "/");
 706     }
 707
 708     s = concat_strings(s, fn);
 709
 710     bufp = grace_path(grace, s);
 711
 712     xfree(s);
 713
 714     return bufp;
 715 }
 716
 717 char *grace_exe_path(Grace *grace, char *fn)
 718 {
 719     static char buf[GR_MAXPATHLEN];
 720     char *cp;
 721
 722     if (fn == NULL) {
 723         return NULL;
 724     } else {
 725         cp = strchr(fn, ' ');
 726         if (cp == NULL) {
 727             return exe_path_translate(grace_path(grace, fn));
 728         } else {
 729             strcpy(buf, fn);
 730             buf[cp - fn] = '\0';
 731             strcpy(buf, grace_path(grace, buf));
 732             strcat(buf, " ");
 733             strcat(buf, cp);
 734             return exe_path_translate(buf);
 735         }
 736     }
 737 }
 738
 739 /* open a file for read */
 740 FILE *grace_openr(Grace *grace, char *fn, int src)
 741 {
 742     struct stat statb;
 743     char *tfn;
 744     char buf[GR_MAXPATHLEN + 50];
 745
 746     if (!fn || !fn[0]) {
 747         errmsg("No file name given");
 748         return NULL;
 749     }
 750     switch (src) {
 751     case SOURCE_DISK:
 752         tfn = grace_path(grace, fn);
 753         if (strcmp(tfn, "-") == 0 || strcmp(tfn, "stdin") == 0) {
 754             return stdin;
 755         } else if (stat(tfn, &statb)) {
 756             sprintf(buf, "Can't stat file %s", tfn);
 757             errmsg(buf);
 758             return NULL;
 759         /* check to make sure this is a file and not a dir */
 760         } else if (!S_ISREG(statb.st_mode)) {
 761             sprintf(buf, "%s is not a regular file", tfn);
 762             errmsg(buf);
 763             return NULL;
 764         } else {
 765             return filter_read(grace, tfn);
 766         }
 767         break;
 768     case SOURCE_PIPE:
 769         tfn = grace_exe_path(grace, fn);
 770         return popen(tfn, "r");
 771         break;
 772     default:
 773         errmsg("Wrong call to grace_openr()");
 774         return NULL;
 775     }
 776 }
 777
 778 /*
 779  * close either a pipe or a file pointer
 780  *
 781  */
 782 void grace_close(FILE *fp)
 783 {
 784     if (fp == stdin || fp == stderr || fp == stdout) {
 785         return;
 786     }
 787     if (pclose(fp) == -1) {
 788         fclose(fp);
 789     }
 790 }
 791
 792 static int uniread(Quark *pr, FILE *fp, int load_type, char *label)
 793 {
 794     int nrows, ncols, nncols, nscols, nncols_req;
 795     int *formats = NULL;
 796     int breakon, readerror;
 797     ss_data ssd;
 798     char *s, tbuf[128];
 799     char *linebuf=NULL;
 800     int linelen=0;   /* a misleading name ... */
 801     int linecount;
 802
 803     linecount = 0;
 804     readerror = 0;
 805     nrows = 0;
 806
 807     breakon = TRUE;
 808
 809     memset(&ssd, 0, sizeof(ssd));
 810
 811     while (read_long_line(fp, &linebuf, &linelen) == RETURN_SUCCESS) {
 812         linecount++;
 813         s = linebuf;
 814         while (*s == ' ' || *s == '\t' || *s == '\n') {
 815             s++;
 816         }
 817         /* skip comments */
 818         if (*s == '#') {
 819             continue;
 820         }
 821         /*   command     end-of-set      EOL   */
 822         if (*s == '@' || *s == '&' || *s == '\0') {
 823             /* a data break line */
 824             if (breakon != TRUE) {
 825                 /* free excessive storage */
 826                 realloc_ss_data(&ssd, nrows);
 827
 828                 /* store accumulated data in set(s) */
 829                 if (store_data(pr, &ssd, load_type) != RETURN_SUCCESS) {
 830                     xfree(linebuf);
 831                     return RETURN_FAILURE;
 832                 }
 833
 834                 /* reset state registers */
 835                 nrows = 0;
 836                 readerror = 0;
 837                 breakon = TRUE;
 838             }
 839             if (*s == '@') {
 840                 scanner(s + 1);
 841                 continue;
 842             }
 843         } else {
 844             if (breakon) {
 845                 /* parse the data line */
 846                 XCFREE(formats);
 847                 if (parse_ss_row(pr, s, &nncols, &nscols, &formats) != RETURN_SUCCESS) {
 848                     errmsg("Can't parse data");
 849                     xfree(linebuf);
 850                     return RETURN_FAILURE;
 851                 }
 852
 853                 if (load_type == LOAD_SINGLE) {
 854                     RunTime *rt = rt_from_quark(pr);
 855                     nncols_req = settype_cols(rt->curtype);
 856                     if (nncols_req <= nncols) {
 857                         nncols = nncols_req;
 858                     } else if (nncols_req == nncols + 1) {
 859                         /* X from index, OK */
 860                         ;
 861                     } else {
 862                         errmsg("Column count incorrect");
 863                         xfree(linebuf);
 864                         return RETURN_FAILURE;
 865                     }
 866                 }
 867
 868                 ncols = nncols + nscols;
 869
 870                 /* init the data storage */
 871                 if (init_ss_data(&ssd, ncols, formats, label)
 872                     != RETURN_SUCCESS) {
 873                     errmsg("Malloc failed in uniread()");
 874                     xfree(linebuf);
 875                     return RETURN_FAILURE;
 876                 }
 877
 878                 breakon = FALSE;
 879             }
 880             if (nrows % BUFSIZE == 0) {
 881                 if (realloc_ss_data(&ssd, nrows + BUFSIZE) != RETURN_SUCCESS) {
 882                     errmsg("Malloc failed in uniread()");
 883                     free_ss_data(&ssd);
 884                     xfree(linebuf);
 885                     return RETURN_FAILURE;
 886                 }
 887             }
 888
 889             if (insert_data_row(pr, &ssd, nrows, s) != RETURN_SUCCESS) {
 890                 sprintf(tbuf, "Error parsing line %d, skipped", linecount);
 891                 errmsg(tbuf);
 892                 readerror++;
 893                 if (readerror > MAXERR) {
 894                     if (yesno("Lots of errors, abort?", NULL, NULL, NULL)) {
 895                         free_ss_data(&ssd);
 896                         xfree(linebuf);
 897                         return RETURN_FAILURE;
 898                     } else {
 899                         readerror = 0;
 900                     }
 901                 }
 902             } else {
 903                 nrows++;
 904             }
 905         }
 906     }
 907
 908     if (nrows > 0) {
 909         /* free excessive storage */
 910         realloc_ss_data(&ssd, nrows);
 911
 912         /* store accumulated data in set(s) */
 913         if (store_data(pr, &ssd, load_type) != RETURN_SUCCESS) {
 914             xfree(linebuf);
 915             return RETURN_FAILURE;
 916         }
 917     }
 918
 919     xfree(linebuf);
 920     xfree(formats);
 921     return RETURN_SUCCESS;
 922 }
 923
 924
 925 int getdata(Quark *pr, char *fn, int src, int load_type)
 926 {
 927     FILE *fp;
 928     int retval;
 929     int save_version, cur_version;
 930     Grace *grace = grace_from_quark(pr);
 931
 932     fp = grace_openr(grace, fn, src);
 933     if (fp == NULL) {
 934         return RETURN_FAILURE;
 935     }
 936
 937     save_version = project_get_version_id(pr);
 938     project_set_version_id(pr, 0);
 939
 940     parser_state_reset(pr);
 941
 942     retval = uniread(pr, fp, load_type, fn);
 943
 944     grace_close(fp);
 945
 946     cur_version = project_get_version_id(pr);
 947     if (cur_version != 0) {
 948         /* a complete project */
 949         project_postprocess(pr);
 950     } else if (load_type != LOAD_BLOCK) {
 951         /* just a few sets */
 952         autoscale_graph(graph_get_current(pr), grace->rt->autoscale_onread);
 953     }
 954     project_set_version_id(pr, save_version);
 955
 956     return retval;
 957 }
 958
 959
 960 /*
 961  * read data to the set from a file overriding the current contents
 962  */
 963 int update_set_from_file(Quark *pset)
 964 {
 965     int retval;
 966     Dataset *dsp;
 967
 968     dsp = set_get_dataset(pset);
 969
 970     if (!dsp) {
 971         retval = RETURN_FAILURE;
 972     } else {
 973         FILE *fp;
 974         RunTime *rt = rt_from_quark(pset);
 975
 976         fp = grace_openr(grace_from_quark(pset), dsp->hotfile, dsp->hotsrc);
 977
 978         killsetdata(pset);
 979         rt->curtype = set_get_type(pset);
 980         retval = uniread(get_parent_project(pset), fp, LOAD_SINGLE, dsp->hotfile);
 981
 982         grace_close(fp);
 983     }
 984
 985     return retval;
 986 }
 987
 988
 989 void outputset(Quark *pset, char *fname, char *dformat)
 990 {
 991     FILE *cp;
 992
 993     if ((cp = grace_openw(grace_from_quark(pset), fname)) == NULL) {
 994         return;
 995     } else {
 996         write_set(pset, cp, dformat);
 997         grace_close(cp);
 998     }
 999 }
1000
1001 /*
1002  * write out a set
1003  */
1004 int write_set(Quark *pset, FILE *cp, char *format)
1005 {
1006     int i, n, col, ncols;
1007     double *x[MAX_SET_COLS];
1008     char **s;
1009
1010     if (cp == NULL) {
1011         return RETURN_FAILURE;
1012     }
1013
1014     if (is_set_dataless(pset) == FALSE) {
1015         n = set_get_length(pset);
1016         ncols = set_get_ncols(pset);
1017         for (col = 0; col < ncols; col++) {
1018             x[col] = set_get_col(pset, col);
1019         }
1020         s = set_get_strings(pset);
1021
1022         if (format == NULL) {
1023             format = project_get_sformat(get_parent_project(pset));
1024         }
1025
1026         for (i = 0; i < n; i++) {
1027             for (col = 0; col < ncols; col++) {
1028                 if (col != 0) {
1029                     fputs(" ", cp);
1030                 }
1031                 fprintf(cp, format, x[col][i]);
1032             }
1033             if (s != NULL) {
1034                 fprintf(cp, " \"%s\"", escapequotes(s[i]));
1035             }
1036             fputs("\n", cp);
1037         }
1038         fprintf(cp, "\n");
1039     }
1040
1041     return RETURN_SUCCESS;
1042 }
1043
1044 extern int load_xgr_project(Grace *grace, char *fn);
1045
1046 int load_agr_project(Grace *grace, char *fn)
1047 {
1048     Quark *project = project_new(grace->rt->qfactory);
1049
1050     grace->rt->print_file[0] = '\0';
1051     grace->rt->curtype = SET_XY;
1052
1053     if (getdata(project, fn, SOURCE_DISK, LOAD_SINGLE) == RETURN_SUCCESS) {
1054         return grace_set_project(grace, project);
1055     } else {
1056         parser_state_reset(grace->project);
1057         quark_free(project);
1058
1059         return RETURN_FAILURE;
1060     }
1061 }
1062
1063 int load_project_file(Grace *grace, char *fn, int as_template)
1064 {
1065     Quark *gr;
1066     int retval;
1067     Quark **graphs;
1068     int i, ngraphs;
1069
1070     if (grace->project &&
1071         quark_dirtystate_get(grace->project) &&
1072         !yesno("Abandon unsaved changes?", NULL, NULL, NULL)) {
1073         return RETURN_FAILURE;
1074     }
1075
1076     /* A temporary hack */
1077     if (fn && strstr(fn, ".xgr")) {
1078         retval = load_xgr_project(grace, fn);
1079     } else {
1080         retval = load_agr_project(grace, fn);
1081     }
1082
1083     if (retval == RETURN_SUCCESS) {
1084         char *tfn;
1085         struct stat statb;
1086         time_t mtime;
1087         static char buf[GR_MAXPATHLEN];
1088         char *bufp;
1089
1090         if (as_template == FALSE) {
1091             project_set_docname(grace->project, fn);
1092         }
1093
1094         /* Set idstr = basename */
1095         strcpy(buf, mybasename(fn));
1096         bufp = strrchr(buf, '.');
1097         if (bufp) {
1098             *(bufp) = '\0';
1099         }
1100         quark_idstr_set(grace->project, buf);
1101
1102         /* Set timestamp */
1103         tfn = grace_path(grace, fn);
1104         if (tfn && !stat(tfn, &statb)) {
1105             mtime = statb.st_mtime;
1106         } else {
1107             /* Probably, piped */
1108             time(&mtime);
1109         }
1110         project_update_timestamp(grace->project, &mtime);
1111
1112         /* Clear dirtystate */
1113         quark_dirtystate_set(grace->project, FALSE);
1114     }
1115
1116     /* try to switch to the first active graph */
1117     ngraphs = project_get_graphs(grace->project, &graphs);
1118     for (i = 0; i < ngraphs; i++) {
1119         gr = graphs[i];
1120         if (select_graph(gr) == RETURN_SUCCESS) {
1121             break;
1122         }
1123     }
1124     xfree(graphs);
1125
1126 #ifndef NONE_GUI
1127     update_all();
1128 #endif
1129     return retval;
1130 }
1131
1132 int load_project(Grace *grace, char *fn)
1133 {
1134     return load_project_file(grace, fn, FALSE);
1135 }
1136
1137 int new_project(Grace *grace, char *template)
1138 {
1139     int retval;
1140     char *s;
1141
1142     if (is_empty_string(template)) {
1143         retval = load_project_file(grace, "templates/Default.xgr", TRUE);
1144     } else if (template[0] == '/') {
1145         retval = load_project_file(grace, template, TRUE);
1146     } else {
1147         s = xmalloc(strlen("templates/") + strlen(template) + 1);
1148         if (s == NULL) {
1149             retval = RETURN_FAILURE;
1150         } else {
1151             sprintf(s, "templates/%s", template);
1152             retval = load_project_file(grace, s, TRUE);
1153             xfree(s);
1154         }
1155     }
1156
1157     return retval;
1158 }
1159
1160 #ifdef HAVE_NETCDF
1161
1162 /*
1163  * Read scaling attribute such ass add_offset or scale_factor
1164  * with default value
1165 */
1166
1167 double read_scale_attr(int cdfid, int varid, char *attrname, double default_value) {
1168   double attribute;
1169   int status;
1170   nc_type attr_type;
1171   size_t length;
1172   status =  nc_inq_att(cdfid, varid, attrname, &attr_type, &length);
1173   if((status!=NC_NOERR) || (length != 1)) {
1174     return default_value;
1175   }
1176   status= nc_get_att_double (cdfid, varid, attrname, &attribute);
1177   if(status!=NC_NOERR) {
1178     return default_value;
1179   }
1180   return attribute;
1181 }
1182
1183
1184 /*
1185  * read a variable from netcdf file into a set in graph gr
1186  * xvar and yvar are the names for x, y in the netcdf file resp.
1187  * return 0 on fail, return 1 if success.
1188  *
1189  * if xvar == NULL, then load the index of the point to x
1190  *
1191  */
1192 int readnetcdf(Quark *pset,
1193                char *netcdfname,
1194                char *xvar,
1195                char *yvar,
1196                int nstart,
1197                int nstop,
1198                int nstride)
1199 {
1200     Quark *gr = get_parent_graph(pset);
1201     int cdfid;                  /* netCDF id */
1202     int i, n;
1203     double *x, *y;
1204     float *xf, *yf;
1205     short *xs, *ys;
1206     long *xl, *yl;
1207     char *xb,*yb;
1208     char buf[256];
1209
1210     /* variable ids */
1211     int x_id = -1, y_id;
1212
1213     /* variable shapes */
1214     long start[2];
1215     long count[2];
1216     /* scaling attributes */
1217     double scale_factor, add_offset;
1218
1219     nc_type xdatatype = 0;
1220     nc_type ydatatype = 0;
1221     int xndims, xdim[10], xnatts;
1222     int yndims, ydim[10], ynatts;
1223     long nx, ny;
1224
1225     RunTime *rt = rt_from_quark(pset);
1226
1227     ncopts = 0;                 /* no crash on error */
1228
1229 /*
1230  * open the netcdf file and locate the variable to read
1231  */
1232     if ((cdfid = ncopen(netcdfname, NC_NOWRITE)) == -1) {
1233         errmsg("Can't open file.");
1234         return 0;
1235     }
1236     if (xvar != NULL) {
1237         if ((x_id = ncvarid(cdfid, xvar)) == -1) {
1238             char ebuf[256];
1239             sprintf(ebuf, "readnetcdf(): No such variable %s for X", xvar);
1240             errmsg(ebuf);
1241             return 0;
1242         }
1243         ncvarinq(cdfid, x_id, NULL, &xdatatype, &xndims, xdim, &xnatts);
1244         ncdiminq(cdfid, xdim[0], NULL, &nx);
1245         if (xndims != 1) {
1246             errmsg("Number of dimensions for X must be 1.");
1247             return 0;
1248         }
1249     }
1250     if ((y_id = ncvarid(cdfid, yvar)) == -1) {
1251         char ebuf[256];
1252         sprintf(ebuf, "readnetcdf(): No such variable %s for Y", yvar);
1253         errmsg(ebuf);
1254         return 0;
1255     }
1256     ncvarinq(cdfid, y_id, NULL, &ydatatype, &yndims, ydim, &ynatts);
1257     ncdiminq(cdfid, ydim[0], NULL, &ny);
1258     if (yndims != 1) {
1259         errmsg("Number of dimensions for Y must be 1.");
1260         return 0;
1261     }
1262     if (xvar != NULL) {
1263         n = nx < ny ? nx : ny;
1264     } else {
1265         n = ny;
1266     }
1267     if (n <= 0) {
1268         errmsg("Length of dimension == 0.");
1269         return 0;
1270     }
1271 /*
1272  * allocate for this set
1273  */
1274     x = xcalloc(n, SIZEOF_DOUBLE);
1275     y = xcalloc(n, SIZEOF_DOUBLE);
1276     if (x == NULL || y == NULL) {
1277         XCFREE(x);
1278         XCFREE(y);
1279         ncclose(cdfid);
1280         return 0;
1281     }
1282     start[0] = 0;
1283     count[0] = n;               /* This will retrieve whole file, modify
1284                                  * these values to get subset. This will only
1285                                  * work for single-dimension vars.  You need
1286                                  * to add dims to start & count for
1287                                  * multi-dimensional. */
1288
1289 /*
1290  * read the variables from the netcdf file
1291  */
1292     if (xvar != NULL) {
1293       scale_factor = read_scale_attr(cdfid,x_id, "scale_factor", 1.0);
1294       add_offset = read_scale_attr(cdfid,x_id, "add_offset", 0.0);
1295 /* TODO should check for other data types here */
1296 /* TODO should check for NULL on the xcallocs() */
1297 /* TODO making assumptions about the sizes of shorts and longs */
1298         switch (xdatatype) {
1299         case NC_BYTE:
1300             xb = xcalloc(n, 1);
1301             /* Theoretically I should add macro to autoconf
1302              file to figure out sizeof(byte)
1303             */
1304             ncvarget(cdfid, x_id, start, count, (void *) xb);
1305             for (i = 0; i < n; i++) {
1306                 x[i] = scale_factor*xb[i]+add_offset;
1307             }
1308             xfree(xb);
1309             break;
1310         case NC_SHORT:
1311             xs = xcalloc(n, SIZEOF_SHORT);
1312             ncvarget(cdfid, x_id, start, count, (void *) xs);
1313             for (i = 0; i < n; i++) {
1314                 x[i] = x[i] = scale_factor*xs[i]+add_offset;
1315             }
1316             xfree(xs);
1317             break;
1318         case NC_LONG:
1319             xl = xcalloc(n, SIZEOF_LONG);
1320             ncvarget(cdfid, x_id, start, count, (void *) xl);
1321             for (i = 0; i < n; i++) {
1322                 x[i] = scale_factor*xl[i]+add_offset;
1323             }
1324             xfree(xl);
1325             break;
1326         case NC_FLOAT:
1327             xf = xcalloc(n, SIZEOF_FLOAT);
1328             ncvarget(cdfid, x_id, start, count, (void *) xf);
1329             for (i = 0; i < n; i++) {
1330                 x[i] = scale_factor*xf[i]+add_offset;
1331             }
1332             xfree(xf);
1333             break;
1334         case NC_DOUBLE:
1335             ncvarget(cdfid, x_id, start, count, (void *) x);
1336             for (i = 0; i < n; i++) {
1337                 x[i] = scale_factor*x[i]+add_offset;
1338             }
1339             break;
1340         default:
1341             errmsg("Data type not supported");
1342             XCFREE(x);
1343             XCFREE(y);
1344             ncclose(cdfid);
1345             return 0;
1346             break;
1347         }
1348     } else {                    /* just load index */
1349         for (i = 0; i < n; i++) {
1350             x[i] = i + 1;
1351         }
1352     }
1353     scale_factor = read_scale_attr(cdfid,y_id, "scale_factor", 1.0);
1354     add_offset = read_scale_attr(cdfid,y_id, "add_offset", 0.0);
1355     switch (ydatatype) {
1356     case NC_BYTE:
1357       yb = xcalloc(n, 1);
1358       ncvarget(cdfid, y_id, start, count, (void *) yb);
1359       for (i = 0; i < n; i++) {
1360         y[i] = scale_factor*yb[i]+add_offset;
1361       }
1362       xfree(yb);
1363       break;
1364
1365     case NC_SHORT:
1366         ys = xcalloc(n, SIZEOF_SHORT);
1367         ncvarget(cdfid, y_id, start, count, (void *) ys);
1368         for (i = 0; i < n; i++) {
1369             y[i] = scale_factor*ys[i]+add_offset;
1370         }
1371         break;
1372     case NC_LONG:
1373         yl = xcalloc(n, SIZEOF_LONG);
1374         ncvarget(cdfid, y_id, start, count, (void *) yl);
1375         for (i = 0; i < n; i++) {
1376             y[i] = scale_factor*yl[i]+add_offset;
1377         }
1378         break;
1379     case NC_FLOAT:
1380 /* TODO should check for NULL here */
1381         yf = xcalloc(n, SIZEOF_FLOAT);
1382         ncvarget(cdfid, y_id, start, count, (void *) yf);
1383         for (i = 0; i < n; i++) {
1384             y[i] = scale_factor*yf[i]+add_offset;
1385         }
1386         xfree(yf);
1387         break;
1388     case NC_DOUBLE:
1389         ncvarget(cdfid, y_id, start, count, (void *) y);
1390         for (i = 0; i < n; i++) {
1391             y[i] = scale_factor*y[i]+add_offset;
1392         }
1393         break;
1394     default:
1395         errmsg("Data type not supported");
1396         XCFREE(x);
1397         XCFREE(y);
1398         ncclose(cdfid);
1399         return 0;
1400         break;
1401     }
1402     ncclose(cdfid);
1403
1404 /*
1405  * initialize stuff for the newly created set
1406  */
1407     set_set_type(pset, SET_XY);
1408     set_set_col(pset, 0, x, n);
1409     set_set_col(pset, 1, y, n);
1410
1411     sprintf(buf, "File %s x = %s y = %s", netcdfname, xvar == NULL ? "Index" : xvar, yvar);
1412     set_set_comment(pset, buf);
1413
1414     autoscale_graph(gr, rt->autoscale_onread);
1415
1416     return 1;
1417 }
1418
1419 int write_netcdf(Quark *pr, char *fname)
1420 {
1421     Quark **graphs;
1422     int i, ngraphs;
1423     Quark *gr;
1424     char buf[512];
1425     int ncid;                   /* netCDF id */
1426     int j;
1427     /* dimension ids */
1428     int n_dim;
1429     /* variable ids */
1430     int x_id, y_id;
1431     int dims[1];
1432     long len[1];
1433     long it = 0;
1434     double *x, *y, x1, x2, y1, y2;
1435     RunTime *rt = rt_from_quark(pr);
1436     ncid = nccreate(fname, NC_CLOBBER);
1437     ncattput(ncid, NC_GLOBAL, "Contents", NC_CHAR, 11, (void *) "grace sets");
1438     ngraphs = project_get_graphs(pr, &graphs);
1439     for (i = 0; i < ngraphs; i++) {
1440         gr = graphs[i];
1441         if (gr) {
1442             Quark **psets;
1443             int nsets = get_descendant_sets(gr, &psets);
1444             for (j = 0; j < nsets; j++) {
1445                 Quark *pset = psets[j];
1446                 if (!is_set_dataless(pset)) {
1447                     char s[64];
1448
1449                     sprintf(buf, "comment");
1450                     ncattput(ncid, NC_GLOBAL, buf, NC_CHAR,
1451                     strlen(set_get_comment(pset)), (void *) set_get_comment(pset));
1452
1453                     sprintf(buf, "type");
1454                     strcpy(s, set_types(rt, set_get_type(pset)));
1455                     ncattput(ncid, NC_GLOBAL, buf, NC_CHAR, strlen(s), (void *) s);
1456
1457                     sprintf(buf, "n");
1458                     n_dim = ncdimdef(ncid, buf, set_get_length(pset));
1459                     dims[0] = n_dim;
1460                     getsetminmax(&pset, 1, &x1, &x2, &y1, &y2);
1461                     sprintf(buf, "x");
1462                     x_id = ncvardef(ncid, buf, NC_DOUBLE, 1, dims);
1463                     ncattput(ncid, x_id, "min", NC_DOUBLE, 1, (void *) &x1);
1464                     ncattput(ncid, x_id, "max", NC_DOUBLE, 1, (void *) &x2);
1465                     dims[0] = n_dim;
1466                     sprintf(buf, "y");
1467                     y_id = ncvardef(ncid, buf, NC_DOUBLE, 1, dims);
1468                     ncattput(ncid, y_id, "min", NC_DOUBLE, 1, (void *) &y1);
1469                     ncattput(ncid, y_id, "max", NC_DOUBLE, 1, (void *) &y2);
1470                 }
1471             }
1472             xfree(psets);
1473         }
1474     }
1475     ncendef(ncid);
1476     ncclose(ncid);
1477     if ((ncid = ncopen(fname, NC_WRITE)) == -1) {
1478         errmsg("Can't open file.");
1479         xfree(graphs);
1480         return 1;
1481     }
1482     for (i = 0; i < ngraphs; i++) {
1483         gr = graphs[i];
1484         if (gr) {
1485             Quark **psets;
1486             int nsets = get_descendant_sets(gr, &psets);
1487             for (j = 0; j < nsets; j++) {
1488                 Quark *pset = psets[j];
1489                 if (!is_set_dataless(pset)) {
1490                     len[0] = set_get_length(pset);
1491                     x = getx(pset);
1492                     y = gety(pset);
1493                     sprintf(buf, "x");
1494                     x_id = ncvarid(ncid, buf);
1495                     sprintf(buf, "y");
1496                     y_id = ncvarid(ncid, buf);
1497                     ncvarput(ncid, x_id, &it, len, (void *) x);
1498                     ncvarput(ncid, y_id, &it, len, (void *) y);
1499                 }
1500             }
1501             xfree(psets);
1502         }
1503     }
1504
1505     xfree(graphs);
1506     ncclose(ncid);
1507     return 0;
1508 }
1509
1510 #endif                          /* HAVE_NETCDF */