src/gromacs/utility/futil.cpp

   1 /*
   2  * This file is part of the GROMACS molecular simulation package.
   3  *
   4  * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
   5  * Copyright (c) 2001-2004, The GROMACS development team.
   6  * Copyright (c) 2013,2014,2015,2016,2017,2018,2019, by the GROMACS development team, led by
   7  * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
   8  * and including many others, as listed in the AUTHORS file in the
   9  * top-level source directory and at http://www.gromacs.org.
  10  *
  11  * GROMACS is free software; you can redistribute it and/or
  12  * modify it under the terms of the GNU Lesser General Public License
  13  * as published by the Free Software Foundation; either version 2.1
  14  * of the License, or (at your option) any later version.
  15  *
  16  * GROMACS is distributed in the hope that it will be useful,
  17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  19  * Lesser General Public License for more details.
  20  *
  21  * You should have received a copy of the GNU Lesser General Public
  22  * License along with GROMACS; if not, see
  23  * http://www.gnu.org/licenses, or write to the Free Software Foundation,
  24  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  25  *
  26  * If you want to redistribute modifications to GROMACS, please
  27  * consider that scientific software is very special. Version
  28  * control is crucial - bugs must be traceable. We will be happy to
  29  * consider code for inclusion in the official distribution, but
  30  * derived work must not be called official GROMACS. Details are found
  31  * in the README & COPYING files - if they are missing, get the
  32  * official version at http://www.gromacs.org.
  33  *
  34  * To help us fund GROMACS development, we humbly ask that you cite
  35  * the research papers on the package. Check out http://www.gromacs.org.
  36  */
  37 #include "gmxpre.h"
  38
  39 #include "futil.h"
  40
  41 #include "config.h"
  42
  43 #include <cerrno>
  44 #include <cstdio>
  45 #include <cstdlib>
  46 #include <cstring>
  47
  48 #include <tuple>
  49
  50 #include <fcntl.h>
  51 #include <sys/stat.h>
  52 #include <sys/types.h>
  53
  54 #ifdef HAVE_UNISTD_H
  55 #include <unistd.h>
  56 #endif
  57 #if GMX_NATIVE_WINDOWS
  58 #include <direct.h>   // For _chdir() and _getcwd()
  59 #include <io.h>
  60 #include <windows.h>
  61 #endif
  62
  63 #include "gromacs/utility/cstringutil.h"
  64 #include "gromacs/utility/datafilefinder.h"
  65 #include "gromacs/utility/dir_separator.h"
  66 #include "gromacs/utility/exceptions.h"
  67 #include "gromacs/utility/fatalerror.h"
  68 #include "gromacs/utility/mutex.h"
  69 #include "gromacs/utility/path.h"
  70 #include "gromacs/utility/programcontext.h"
  71 #include "gromacs/utility/smalloc.h"
  72 #include "gromacs/utility/stringutil.h"
  73
  74 /* we keep a linked list of all files opened through pipes (i.e.
  75    compressed or .gzipped files. This way we can distinguish between them
  76    without having to change the semantics of reading from/writing to files)
  77  */
  78 typedef struct t_pstack {
  79     FILE            *fp;
  80     struct t_pstack *prev;
  81 } t_pstack;
  82
  83 static t_pstack    *pstack           = nullptr;
  84 static bool         bUnbuffered      = false;
  85 static int          s_maxBackupCount = 0;
  86
  87 /* this linked list is an intrinsically globally shared object, so we have
  88    to protect it with mutexes */
  89 static gmx::Mutex pstack_mutex;
  90
  91 using Lock = gmx::lock_guard<gmx::Mutex>;
  92
  93 namespace gmx
  94 {
  95 namespace
  96 {
  97 //! Global library file finder; stores the object set with setLibraryFileFinder().
  98 const DataFileFinder *g_libFileFinder;
  99 //! Default library file finder if nothing is set.
 100 const DataFileFinder  g_defaultLibFileFinder;
 101 }   // namespace
 102
 103 const DataFileFinder &getLibraryFileFinder()
 104 {
 105     if (g_libFileFinder != nullptr)
 106     {
 107         return *g_libFileFinder;
 108     }
 109     return g_defaultLibFileFinder;
 110 }
 111
 112 void setLibraryFileFinder(const DataFileFinder *finder)
 113 {
 114     g_libFileFinder = finder;
 115 }
 116
 117 } // namespace gmx
 118
 119 void gmx_disable_file_buffering()
 120 {
 121     bUnbuffered = true;
 122 }
 123
 124 void gmx_set_max_backup_count(int count)
 125 {
 126     if (count < 0)
 127     {
 128         const char *env = getenv("GMX_MAXBACKUP");
 129         if (env != nullptr)
 130         {
 131             // TODO: Check that the value is converted properly.
 132             count = strtol(env, nullptr, 10);
 133             if (count < 0)
 134             {
 135                 count = 0;
 136             }
 137         }
 138         else
 139         {
 140             // Use a reasonably low value for countmax; we might
 141             // generate 4-5 files in each round, and we don't
 142             // want to hit directory limits of 1024 or 2048 files.
 143             count = 99;
 144         }
 145     }
 146     s_maxBackupCount = count;
 147 }
 148
 149 static void push_ps(FILE *fp)
 150 {
 151     t_pstack *ps;
 152
 153     Lock      pstackLock(pstack_mutex);
 154
 155     snew(ps, 1);
 156     ps->fp   = fp;
 157     ps->prev = pstack;
 158     pstack   = ps;
 159 }
 160
 161 #if GMX_FAHCORE
 162 /* don't use pipes!*/
 163 #define popen fah_fopen
 164 #define pclose fah_fclose
 165 #define SKIP_FFOPS 1
 166 #else
 167 #ifdef gmx_ffclose
 168 #undef gmx_ffclose
 169 #endif
 170 #if (!HAVE_PIPES && !defined(__native_client__))
 171 static FILE *popen(const char *nm, const char *mode)
 172 {
 173     gmx_impl("Sorry no pipes...");
 174
 175     return NULL;
 176 }
 177
 178 static int pclose(FILE *fp)
 179 {
 180     gmx_impl("Sorry no pipes...");
 181
 182     return 0;
 183 }
 184 #endif /* !HAVE_PIPES && !defined(__native_client__) */
 185 #endif /* GMX_FAHCORE */
 186
 187 int gmx_ffclose(FILE *fp)
 188 {
 189 #ifdef SKIP_FFOPS
 190     return fclose(fp);
 191 #else
 192     t_pstack *ps, *tmp;
 193     int       ret = 0;
 194
 195     Lock      pstackLock(pstack_mutex);
 196
 197     ps = pstack;
 198     if (ps == nullptr)
 199     {
 200         if (fp != nullptr)
 201         {
 202             ret = fclose(fp);
 203         }
 204     }
 205     else if (ps->fp == fp)
 206     {
 207         if (fp != nullptr)
 208         {
 209             ret = pclose(fp);
 210         }
 211         pstack = pstack->prev;
 212         sfree(ps);
 213     }
 214     else
 215     {
 216         while ((ps->prev != nullptr) && (ps->prev->fp != fp))
 217         {
 218             ps = ps->prev;
 219         }
 220         if ((ps->prev != nullptr) && ps->prev->fp == fp)
 221         {
 222             if (ps->prev->fp != nullptr)
 223             {
 224                 ret = pclose(ps->prev->fp);
 225             }
 226             tmp      = ps->prev;
 227             ps->prev = ps->prev->prev;
 228             sfree(tmp);
 229         }
 230         else
 231         {
 232             if (fp != nullptr)
 233             {
 234                 ret = fclose(fp);
 235             }
 236         }
 237     }
 238
 239     return ret;
 240 #endif
 241 }
 242
 243
 244 void frewind(FILE *fp)
 245 {
 246     Lock      pstackLock(pstack_mutex);
 247
 248     t_pstack *ps = pstack;
 249     while (ps != nullptr)
 250     {
 251         if (ps->fp == fp)
 252         {
 253             fprintf(stderr, "Cannot rewind compressed file!\n");
 254             return;
 255         }
 256         ps = ps->prev;
 257     }
 258     rewind(fp);
 259 }
 260
 261 int gmx_fseek(FILE *stream, gmx_off_t offset, int whence)
 262 {
 263 #if HAVE_FSEEKO
 264     return fseeko(stream, offset, whence);
 265 #else
 266 #if HAVE__FSEEKI64
 267     return _fseeki64(stream, offset, whence);
 268 #else
 269     return fseek(stream, offset, whence);
 270 #endif
 271 #endif
 272 }
 273
 274 gmx_off_t gmx_ftell(FILE *stream)
 275 {
 276 #if HAVE_FSEEKO
 277     return ftello(stream);
 278 #else
 279 #if HAVE__FSEEKI64
 280 #ifndef __MINGW32__
 281     return _ftelli64(stream);
 282 #else
 283     return ftello64(stream);
 284 #endif
 285 #else
 286     return ftell(stream);
 287 #endif
 288 #endif
 289 }
 290
 291 int gmx_truncate(const std::string &filename, gmx_off_t length)
 292 {
 293 #if GMX_NATIVE_WINDOWS
 294     FILE *fp = fopen(filename.c_str(), "rb+");
 295     if (fp == NULL)
 296     {
 297         return -1;
 298     }
 299 #ifdef _MSC_VER
 300     int rc = _chsize_s(fileno(fp), length);
 301 #else
 302     int rc = _chsize(fileno(fp), length);
 303 #endif
 304     fclose(fp);
 305     return rc;
 306 #else
 307     return truncate(filename.c_str(), length);
 308 #endif
 309 }
 310
 311 static FILE *uncompress(const std::string &fn, const char *mode)
 312 {
 313     FILE       *fp;
 314     std::string buf = "uncompress -c < " + fn;
 315     fprintf(stderr, "Going to execute '%s'\n", buf.c_str());
 316     if ((fp = popen(buf.c_str(), mode)) == nullptr)
 317     {
 318         gmx_open(fn);
 319     }
 320     push_ps(fp);
 321
 322     return fp;
 323 }
 324
 325 static FILE *gunzip(const std::string &fn, const char *mode)
 326 {
 327     FILE       *fp;
 328     std::string buf = "gunzip -c < ";
 329     buf += fn;
 330     fprintf(stderr, "Going to execute '%s'\n", buf.c_str());
 331     if ((fp = popen(buf.c_str(), mode)) == nullptr)
 332     {
 333         gmx_open(fn);
 334     }
 335     push_ps(fp);
 336
 337     return fp;
 338 }
 339
 340 gmx_bool gmx_fexist(const std::string &fname)
 341 {
 342     FILE *test;
 343
 344     if (fname.empty())
 345     {
 346         return FALSE;
 347     }
 348     test = fopen(fname.c_str(), "r");
 349     if (test == nullptr)
 350     {
 351         /*Windows doesn't allow fopen of directory - so we need to check this seperately */
 352         #if GMX_NATIVE_WINDOWS
 353         DWORD attr = GetFileAttributes(fname.c_str());
 354         return (attr != INVALID_FILE_ATTRIBUTES) && (attr & FILE_ATTRIBUTE_DIRECTORY);
 355         #else
 356         return FALSE;
 357         #endif
 358     }
 359     else
 360     {
 361         fclose(test);
 362         return TRUE;
 363     }
 364 }
 365
 366 static std::string backup_fn(const std::string &file)
 367 {
 368     int          count = 1;
 369
 370     std::string  directory = gmx::Path::getParentPath(file);
 371     std::string  fn        = gmx::Path::getFilename(file);
 372     std::string  buf;
 373     if (directory.empty())
 374     {
 375         directory = ".";
 376     }
 377     do
 378     {
 379         buf = gmx::formatString("%s/#%s.%d#", directory.c_str(), fn.c_str(), count);
 380         count++;
 381     }
 382     while ((count <= s_maxBackupCount) && gmx_fexist(buf));
 383
 384     /* Arbitrarily bail out */
 385     if (count > s_maxBackupCount)
 386     {
 387         /* TODO: The error message is only accurate for code that starts with
 388          * Gromacs command-line interface. */
 389         gmx_fatal(FARGS, "Won't make more than %d backups of %s for you.\n"
 390                   "The env.var. GMX_MAXBACKUP controls this maximum, -1 disables backups.",
 391                   s_maxBackupCount, fn.c_str());
 392     }
 393
 394     return buf;
 395 }
 396
 397 void make_backup(const std::string &name)
 398 {
 399     if (s_maxBackupCount <= 0)
 400     {
 401         return;
 402     }
 403     if (gmx_fexist(name))
 404     {
 405         auto backup = backup_fn(name);
 406         if (rename(name.c_str(), backup.c_str()) == 0)
 407         {
 408             fprintf(stderr, "\nBack Off! I just backed up %s to %s\n",
 409                     name.c_str(), backup.c_str());
 410         }
 411         else
 412         {
 413             fprintf(stderr, "\nSorry couldn't backup %s to %s\n", name.c_str(), backup.c_str());
 414         }
 415     }
 416 }
 417
 418 FILE *gmx_ffopen(const std::string &file, const char *mode)
 419 {
 420 #ifdef SKIP_FFOPS
 421     return fopen(file, mode);
 422 #else
 423     FILE    *ff = nullptr;
 424     gmx_bool bRead;
 425     int      bs;
 426
 427     if (file.empty())
 428     {
 429         return nullptr;
 430     }
 431
 432     if (mode[0] == 'w')
 433     {
 434         make_backup(file);
 435     }
 436
 437     bRead = (mode[0] == 'r' && mode[1] != '+');
 438     if (!bRead || gmx_fexist(file))
 439     {
 440         if ((ff = fopen(file.c_str(), mode)) == nullptr)
 441         {
 442             gmx_file(file);
 443         }
 444         /* Check whether we should be using buffering (default) or not
 445          * (for debugging)
 446          */
 447         const char *bufsize = nullptr;
 448         if (bUnbuffered || ((bufsize = getenv("GMX_LOG_BUFFER")) != nullptr))
 449         {
 450             /* Check whether to use completely unbuffered */
 451             if (bUnbuffered)
 452             {
 453                 bs = 0;
 454             }
 455             else
 456             {
 457                 bs = strtol(bufsize, nullptr, 10);
 458             }
 459             if (bs <= 0)
 460             {
 461                 setbuf(ff, nullptr);
 462             }
 463             else
 464             {
 465                 char *ptr;
 466                 snew(ptr, bs+8);
 467                 if (setvbuf(ff, ptr, _IOFBF, bs) != 0)
 468                 {
 469                     gmx_file("Buffering File");
 470                 }
 471             }
 472         }
 473     }
 474     else
 475     {
 476         std::string compressedFileName = file;
 477         compressedFileName += ".Z";
 478         if (gmx_fexist(compressedFileName))
 479         {
 480             ff = uncompress(compressedFileName, mode);
 481         }
 482         else
 483         {
 484             compressedFileName  = file;
 485             compressedFileName += ".gz";
 486             if (gmx_fexist(compressedFileName))
 487             {
 488                 ff = gunzip(compressedFileName, mode);
 489             }
 490             else
 491             {
 492                 gmx_file(file);
 493             }
 494         }
 495     }
 496     return ff;
 497 #endif
 498 }
 499
 500 namespace gmx
 501 {
 502
 503 std::string findLibraryFile(const std::string &filename, bool bAddCWD, bool bFatal)
 504 {
 505     std::string result;
 506     try
 507     {
 508         const DataFileFinder &finder = getLibraryFileFinder();
 509         result = finder.findFile(DataFileOptions(filename)
 510                                      .includeCurrentDir(bAddCWD)
 511                                      .throwIfNotFound(bFatal));
 512     }
 513     GMX_CATCH_ALL_AND_EXIT_WITH_FATAL_ERROR;
 514     return result;
 515 }
 516
 517 std::string findLibraryFile(const char *filename, bool bAddCWD, bool bFatal)
 518 {
 519     return findLibraryFile(std::string(filename), bAddCWD, bFatal);
 520 }
 521
 522 FilePtr openLibraryFile(const std::string &filename, bool bAddCWD, bool bFatal)
 523 {
 524     FilePtr fp;
 525     try
 526     {
 527         const DataFileFinder &finder = getLibraryFileFinder();
 528         fp = finder.openFile(DataFileOptions(filename)
 529                                  .includeCurrentDir(bAddCWD)
 530                                  .throwIfNotFound(bFatal));
 531     }
 532     GMX_CATCH_ALL_AND_EXIT_WITH_FATAL_ERROR;
 533     return fp;
 534 }
 535
 536 FilePtr openLibraryFile(const char *filename, bool bAddCWD, bool bFatal)
 537 {
 538     return openLibraryFile(std::string(filename), bAddCWD, bFatal);
 539 }
 540
 541 } // namespace gmx
 542
 543 /*! \brief Use mkstemp (or similar function to make a new temporary
 544  * file and (on non-Windows systems) return a file descriptor to it.
 545  *
 546  * \todo Use std::string and std::vector<char>. */
 547 static int makeTemporaryFilename(char *buf)
 548 {
 549     int len;
 550
 551     if ((len = strlen(buf)) < 7)
 552     {
 553         gmx_fatal(FARGS, "Buf passed to gmx_tmpnam must be at least 7 bytes long");
 554     }
 555     for (int i = len-6; (i < len); i++)
 556     {
 557         buf[i] = 'X';
 558     }
 559     /* mktemp is dangerous and we should use mkstemp instead, but
 560      * since windows doesnt support it we have to separate the cases.
 561      * 20090307: mktemp deprecated, use iso c++ _mktemp instead.
 562      */
 563     int fd;
 564 #if GMX_NATIVE_WINDOWS
 565     _mktemp(buf);
 566     if (buf == NULL)
 567     {
 568         gmx_fatal(FARGS, "Error creating temporary file %s: %s", buf,
 569                   strerror(errno));
 570     }
 571     fd = 0;
 572 #else
 573     fd = mkstemp(buf);
 574
 575     if (fd < 0)
 576     {
 577         gmx_fatal(FARGS, "Error creating temporary file %s: %s", buf,
 578                   strerror(errno));
 579     }
 580 #endif
 581     return fd;
 582 }
 583 // TODO use std::string
 584 void gmx_tmpnam(char *buf)
 585 {
 586     int fd = makeTemporaryFilename(buf);
 587 #if !GMX_NATIVE_WINDOWS
 588     close(fd);
 589 #endif
 590 }
 591
 592 // TODO use std::string
 593 FILE *gmx_fopen_temporary(char *buf)
 594 {
 595     FILE *fpout = nullptr;
 596     int   fd    = makeTemporaryFilename(buf);
 597
 598 #if GMX_NATIVE_WINDOWS
 599     if ((fpout = fopen(buf, "w")) == NULL)
 600     {
 601         gmx_fatal(FARGS, "Cannot open temporary file %s", buf);
 602     }
 603 #else
 604     if ((fpout = fdopen(fd, "w")) == nullptr)
 605     {
 606         gmx_fatal(FARGS, "Cannot open temporary file %s", buf);
 607     }
 608 #endif
 609
 610     return fpout;
 611 }
 612
 613 int gmx_file_rename(const char *oldname, const char *newname)
 614 {
 615 #if !GMX_NATIVE_WINDOWS
 616     /* under unix, rename() is atomic (at least, it should be). */
 617     return rename(oldname, newname);
 618 #else
 619     if (MoveFileEx(oldname, newname,
 620                    MOVEFILE_REPLACE_EXISTING|MOVEFILE_WRITE_THROUGH))
 621     {
 622         return 0;
 623     }
 624     else
 625     {
 626         return 1;
 627     }
 628 #endif
 629 }
 630
 631 int gmx_file_copy(const char *oldname, const char *newname, gmx_bool copy_if_empty)
 632 {
 633     gmx::FilePtr in(fopen(oldname, "rb"));
 634     if (!in)
 635     {
 636         return 1;
 637     }
 638
 639     /* If we don't copy when empty, we postpone opening the file
 640        until we're actually ready to write. */
 641     gmx::FilePtr out;
 642     if (copy_if_empty)
 643     {
 644         out.reset(fopen(newname, "wb"));
 645         if (!out)
 646         {
 647             return 1;
 648         }
 649     }
 650
 651     /* the full copy buffer size: */
 652     constexpr int     FILECOPY_BUFSIZE = 1<<16;
 653     std::vector<char> buf(FILECOPY_BUFSIZE);
 654
 655     while (!feof(in.get()))
 656     {
 657         size_t nread;
 658
 659         nread = fread(buf.data(), sizeof(char), FILECOPY_BUFSIZE, in.get());
 660         if (nread > 0)
 661         {
 662             size_t ret;
 663             if (!out)
 664             {
 665                 /* so this is where we open when copy_if_empty is false:
 666                    here we know we read something. */
 667                 out.reset(fopen(newname, "wb"));
 668                 if (!out)
 669                 {
 670                     return 1;
 671                 }
 672             }
 673             ret = fwrite(buf.data(), sizeof(char), nread, out.get());
 674             if (ret != nread)
 675             {
 676                 return 1;
 677             }
 678         }
 679         if (ferror(in.get()))
 680         {
 681             return 1;
 682         }
 683     }
 684     return 0;
 685 }
 686
 687
 688 int gmx_fsync(FILE *fp)
 689 {
 690     int rc = 0;
 691
 692 #if GMX_FAHCORE
 693     /* the fahcore defines its own os-independent fsync */
 694     rc = fah_fsync(fp);
 695 #else /* GMX_FAHCORE */
 696     {
 697         int fn;
 698
 699         /* get the file number */
 700 #if HAVE_FILENO
 701         fn = fileno(fp);
 702 #elif HAVE__FILENO
 703         fn = _fileno(fp);
 704 #else
 705         fn = -1;
 706 #endif
 707
 708         /* do the actual fsync */
 709         if (fn >= 0)
 710         {
 711 #if HAVE_FSYNC
 712             rc = fsync(fn);
 713 #elif HAVE__COMMIT
 714             rc = _commit(fn);
 715 #endif
 716         }
 717     }
 718 #endif /* GMX_FAHCORE */
 719
 720     /* We check for these error codes this way because POSIX requires them
 721        to be defined, and using anything other than macros is unlikely: */
 722 #ifdef EINTR
 723     /* we don't want to report an error just because fsync() caught a signal.
 724        For our purposes, we can just ignore this. */
 725     if (rc && errno == EINTR)
 726     {
 727         rc = 0;
 728     }
 729 #endif
 730 #ifdef EINVAL
 731     /* we don't want to report an error just because we tried to fsync()
 732        stdout, a socket or a pipe. */
 733     if (rc && errno == EINVAL)
 734     {
 735         rc = 0;
 736     }
 737 #endif
 738     return rc;
 739 }
 740
 741 void gmx_chdir(const char *directory)
 742 {
 743 #if GMX_NATIVE_WINDOWS
 744     int rc = _chdir(directory);
 745 #else
 746     int rc = chdir(directory);
 747 #endif
 748     if (rc != 0)
 749     {
 750         gmx_fatal(FARGS, "Cannot change directory to '%s'. Reason: %s",
 751                   directory, strerror(errno));
 752     }
 753 }
 754
 755 void gmx_getcwd(char *buffer, size_t size)
 756 {
 757 #if GMX_NATIVE_WINDOWS
 758     char *pdum = _getcwd(buffer, size);
 759 #else
 760     char *pdum = getcwd(buffer, size);
 761 #endif
 762     if (pdum == nullptr)
 763     {
 764         gmx_fatal(FARGS, "Cannot get working directory. Reason: %s",
 765                   strerror(errno));
 766     }
 767 }