Merge branch release-2016
[gromacs.git] / src / gromacs / fileio / gmxfio.cpp
blob485abae3a9a74447acce2f0cff6477b173726acf
1 /*
2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
5 * Copyright (c) 2001-2004, The GROMACS development team.
6 * Copyright (c) 2013,2014,2015,2016,2017, by the GROMACS development team, led by
7 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
8 * and including many others, as listed in the AUTHORS file in the
9 * top-level source directory and at http://www.gromacs.org.
11 * GROMACS is free software; you can redistribute it and/or
12 * modify it under the terms of the GNU Lesser General Public License
13 * as published by the Free Software Foundation; either version 2.1
14 * of the License, or (at your option) any later version.
16 * GROMACS is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 * Lesser General Public License for more details.
21 * You should have received a copy of the GNU Lesser General Public
22 * License along with GROMACS; if not, see
23 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
24 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
26 * If you want to redistribute modifications to GROMACS, please
27 * consider that scientific software is very special. Version
28 * control is crucial - bugs must be traceable. We will be happy to
29 * consider code for inclusion in the official distribution, but
30 * derived work must not be called official GROMACS. Details are found
31 * in the README & COPYING files - if they are missing, get the
32 * official version at http://www.gromacs.org.
34 * To help us fund GROMACS development, we humbly ask that you cite
35 * the research papers on the package. Check out http://www.gromacs.org.
37 #include "gmxpre.h"
39 #include "gmxfio.h"
41 #include "config.h"
43 #include <cerrno>
44 #include <cstdio>
45 #include <cstring>
47 #if HAVE_IO_H
48 #include <io.h>
49 #endif
50 #ifdef HAVE_UNISTD_H
51 #include <unistd.h>
52 #endif
54 #include "thread_mpi/threads.h"
56 #include "gromacs/fileio/filetypes.h"
57 #include "gromacs/fileio/md5.h"
58 #include "gromacs/utility/fatalerror.h"
59 #include "gromacs/utility/futil.h"
60 #include "gromacs/utility/mutex.h"
61 #include "gromacs/utility/smalloc.h"
63 #include "gmxfio-impl.h"
65 /* This is the new improved and thread safe version of gmxfio. */
69 /* the list of open files is a linked list, with a dummy element at its head;
70 it is initialized when the first file is opened. */
71 static t_fileio *open_files = nullptr;
74 /* this mutex locks the open_files structure so that no two threads can
75 modify it.
77 For now, we use this as a coarse grained lock on all file
78 insertion/deletion operations because it makes avoiding deadlocks
79 easier, and adds almost no overhead: the only overhead is during
80 opening and closing of files, or during global operations like
81 iterating along all open files. All these cases should be rare
82 during the simulation. */
83 static gmx::Mutex open_file_mutex;
85 using Lock = gmx::lock_guard<gmx::Mutex>;
87 /******************************************************************
89 * Internal functions:
91 ******************************************************************/
93 static int gmx_fio_int_flush(t_fileio* fio)
95 int rc = 0;
97 if (fio->fp)
99 rc = fflush(fio->fp);
102 return rc;
105 /* lock the mutex associated with this fio. This needs to be done for every
106 type of access to the fio's elements. */
107 void gmx_fio_lock(t_fileio *fio)
109 tMPI_Lock_lock(&(fio->mtx));
111 /* unlock the mutex associated with this fio. */
112 void gmx_fio_unlock(t_fileio *fio)
114 tMPI_Lock_unlock(&(fio->mtx));
117 /* make a dummy head element, assuming we locked everything. */
118 static void gmx_fio_make_dummy(void)
120 if (!open_files)
122 snew(open_files, 1);
123 open_files->fp = nullptr;
124 open_files->fn = nullptr;
125 open_files->next = open_files;
126 open_files->prev = open_files;
127 tMPI_Lock_init(&(open_files->mtx));
137 /***********************************************************************
139 * FILE LIST OPERATIONS
141 ***********************************************************************/
144 /* insert a new t_fileio into the list */
145 static void gmx_fio_insert(t_fileio *fio)
147 t_fileio *prev;
148 Lock openFilesLock(open_file_mutex);
149 gmx_fio_make_dummy();
151 /* and lock the fio we got and the list's head **/
152 gmx_fio_lock(fio);
153 gmx_fio_lock(open_files);
154 prev = open_files->prev;
155 /* lock the element after the current one */
156 if (prev != open_files)
158 gmx_fio_lock(prev);
161 /* now do the actual insertion: */
162 fio->next = open_files;
163 open_files->prev = fio;
164 prev->next = fio;
165 fio->prev = prev;
167 /* now unlock all our locks */
168 if (prev != open_files)
170 gmx_fio_unlock(prev);
172 gmx_fio_unlock(open_files);
173 gmx_fio_unlock(fio);
176 /* remove a t_fileio into the list. We assume the fio is locked, and we leave
177 it locked.
178 NOTE: We also assume that the open_file_mutex has been locked */
179 static void gmx_fio_remove(t_fileio *fio)
181 /* lock prev, because we're changing it */
182 gmx_fio_lock(fio->prev);
184 /* now set the prev's pointer */
185 fio->prev->next = fio->next;
186 gmx_fio_unlock(fio->prev);
188 /* with the next ptr, we can simply lock while the original was locked */
189 gmx_fio_lock(fio->next);
190 fio->next->prev = fio->prev;
191 gmx_fio_unlock(fio->next);
193 /* and make sure we point nowhere in particular */
194 fio->next = fio->prev = fio;
198 /* get the first open file, or NULL if there is none.
199 Returns a locked fio. Assumes open_files_mutex is locked. */
200 static t_fileio *gmx_fio_get_first(void)
202 t_fileio *ret;
204 gmx_fio_make_dummy();
206 gmx_fio_lock(open_files);
207 ret = open_files->next;
210 /* check whether there were any to begin with */
211 if (ret == open_files)
213 /* after this, the open_file pointer should never change */
214 ret = nullptr;
216 else
218 gmx_fio_lock(open_files->next);
220 gmx_fio_unlock(open_files);
223 return ret;
226 /* get the next open file, or NULL if there is none.
227 Unlocks the previous fio and locks the next one.
228 Assumes open_file_mutex is locked. */
229 static t_fileio *gmx_fio_get_next(t_fileio *fio)
231 t_fileio *ret;
233 ret = fio->next;
234 /* check if that was the last one */
235 if (fio->next == open_files)
237 ret = nullptr;
239 else
241 gmx_fio_lock(ret);
243 gmx_fio_unlock(fio);
245 return ret;
248 /* Stop looping through the open_files. Assumes open_file_mutex is locked. */
249 static void gmx_fio_stop_getting_next(t_fileio *fio)
251 gmx_fio_unlock(fio);
257 /*****************************************************************
259 * EXPORTED SECTION
261 *****************************************************************/
262 t_fileio *gmx_fio_open(const char *fn, const char *mode)
264 t_fileio *fio = nullptr;
265 char newmode[5];
266 gmx_bool bRead, bReadWrite;
268 /* sanitize the mode string */
269 if (std::strncmp(mode, "r+", 2) == 0)
271 std::strcpy(newmode, "r+");
273 else if (mode[0] == 'r')
275 std::strcpy(newmode, "r");
277 else if (strncmp(mode, "w+", 2) == 0)
279 std::strcpy(newmode, "w+");
281 else if (mode[0] == 'w')
283 std::strcpy(newmode, "w");
285 else if (strncmp(mode, "a+", 2) == 0)
287 std::strcpy(newmode, "a+");
289 else if (mode[0] == 'a')
291 std::strcpy(newmode, "a");
293 else
295 gmx_fatal(FARGS, "DEATH HORROR in gmx_fio_open, mode is '%s'", mode);
298 /* Check if it should be opened as a binary file */
299 if (!ftp_is_text(fn2ftp(fn)))
301 strcat(newmode, "b");
304 snew(fio, 1);
305 tMPI_Lock_init(&(fio->mtx));
306 bRead = (newmode[0] == 'r' && newmode[1] != '+');
307 bReadWrite = (newmode[1] == '+');
308 fio->fp = nullptr;
309 fio->xdr = nullptr;
310 if (fn)
312 if (fn2ftp(fn) == efTNG)
314 gmx_incons("gmx_fio_open may not be used to open TNG files");
316 fio->iFTP = fn2ftp(fn);
317 fio->fn = gmx_strdup(fn);
319 fio->fp = gmx_ffopen(fn, newmode);
320 /* If this file type is in the list of XDR files, open it like that */
321 if (ftp_is_xdr(fio->iFTP))
323 /* determine the XDR direction */
324 if (newmode[0] == 'w' || newmode[0] == 'a')
326 fio->xdrmode = XDR_ENCODE;
328 else
330 fio->xdrmode = XDR_DECODE;
332 snew(fio->xdr, 1);
333 xdrstdio_create(fio->xdr, fio->fp, fio->xdrmode);
336 /* for appending seek to end of file to make sure ftell gives correct position
337 * important for checkpointing */
338 if (newmode[0] == 'a')
340 gmx_fseek(fio->fp, 0, SEEK_END);
343 else
345 gmx_fatal(FARGS, "Cannot open file with NULL filename string");
348 fio->bRead = bRead;
349 fio->bReadWrite = bReadWrite;
350 fio->bDouble = (sizeof(real) == sizeof(double));
352 /* and now insert this file into the list of open files. */
353 gmx_fio_insert(fio);
354 return fio;
357 static int gmx_fio_close_locked(t_fileio *fio)
359 int rc = 0;
361 if (fio->xdr != nullptr)
363 xdr_destroy(fio->xdr);
364 sfree(fio->xdr);
367 if (fio->fp != nullptr)
369 rc = gmx_ffclose(fio->fp); /* fclose returns 0 if happy */
373 return rc;
376 int gmx_fio_close(t_fileio *fio)
378 int rc = 0;
380 Lock openFilesLock(open_file_mutex);
382 gmx_fio_lock(fio);
383 /* first remove it from the list */
384 gmx_fio_remove(fio);
385 rc = gmx_fio_close_locked(fio);
386 gmx_fio_unlock(fio);
388 sfree(fio->fn);
389 sfree(fio);
391 return rc;
394 /* close only fp but keep FIO entry. */
395 int gmx_fio_fp_close(t_fileio *fio)
397 int rc = 0;
398 gmx_fio_lock(fio);
399 if (fio->xdr == nullptr)
401 rc = gmx_ffclose(fio->fp); /* fclose returns 0 if happy */
402 fio->fp = nullptr;
404 gmx_fio_unlock(fio);
406 return rc;
409 FILE * gmx_fio_fopen(const char *fn, const char *mode)
411 FILE *ret;
412 t_fileio *fio;
414 fio = gmx_fio_open(fn, mode);
415 gmx_fio_lock(fio);
416 ret = fio->fp;
417 gmx_fio_unlock(fio);
419 return ret;
422 int gmx_fio_fclose(FILE *fp)
424 t_fileio *cur;
425 int rc = -1;
427 Lock openFilesLock(open_file_mutex);
428 cur = gmx_fio_get_first();
429 while (cur)
431 if (cur->fp == fp)
433 rc = gmx_fio_close_locked(cur);
434 gmx_fio_remove(cur);
435 gmx_fio_stop_getting_next(cur);
436 sfree(cur->fn);
437 sfree(cur);
438 break;
440 cur = gmx_fio_get_next(cur);
443 return rc;
446 /* internal variant of get_file_md5 that operates on a locked file */
447 static int gmx_fio_int_get_file_md5(t_fileio *fio, gmx_off_t offset,
448 unsigned char digest[])
450 /*1MB: large size important to catch almost identical files */
451 #define CPT_CHK_LEN 1048576
452 md5_state_t state;
453 unsigned char *buf;
454 gmx_off_t read_len;
455 gmx_off_t seek_offset;
456 int ret = -1;
458 seek_offset = offset - CPT_CHK_LEN;
459 if (seek_offset < 0)
461 seek_offset = 0;
463 read_len = offset - seek_offset;
466 if (fio->fp && fio->bReadWrite)
468 ret = gmx_fseek(fio->fp, seek_offset, SEEK_SET);
469 if (ret)
471 gmx_fseek(fio->fp, 0, SEEK_END);
474 if (ret) /*either no fp, not readwrite, or fseek not successful */
476 return -1;
479 snew(buf, CPT_CHK_LEN);
480 /* the read puts the file position back to offset */
481 if ((gmx_off_t)fread(buf, 1, read_len, fio->fp) != read_len)
483 /* not fatal: md5sum check to prevent overwriting files
484 * works (less safe) without
485 * */
486 if (ferror(fio->fp))
488 fprintf(stderr, "\nTrying to get md5sum: %s: %s\n", fio->fn,
489 strerror(errno));
491 else if (feof(fio->fp))
494 * For long runs that checkpoint frequently but write e.g. logs
495 * infrequently we don't want to issue lots of warnings before we
496 * have written anything to the log.
498 if (0)
500 fprintf(stderr, "\nTrying to get md5sum: EOF: %s\n", fio->fn);
503 else
505 fprintf(
506 stderr,
507 "\nTrying to get md5sum: Unknown reason for short read: %s\n",
508 fio->fn);
511 gmx_fseek(fio->fp, 0, SEEK_END);
513 ret = -1;
515 gmx_fseek(fio->fp, 0, SEEK_END); /*is already at end, but under windows
516 it gives problems otherwise*/
518 if (debug)
520 fprintf(debug, "chksum %s readlen %ld\n", fio->fn, (long int)read_len);
523 if (!ret)
525 gmx_md5_init(&state);
526 gmx_md5_append(&state, buf, read_len);
527 gmx_md5_finish(&state, digest);
528 ret = read_len;
530 sfree(buf);
531 return ret;
536 * fio: file to compute md5 for
537 * offset: starting pointer of region to use for md5
538 * digest: return array of md5 sum
540 int gmx_fio_get_file_md5(t_fileio *fio, gmx_off_t offset,
541 unsigned char digest[])
543 int ret;
545 gmx_fio_lock(fio);
546 ret = gmx_fio_int_get_file_md5(fio, offset, digest);
547 gmx_fio_unlock(fio);
549 return ret;
552 /* The fio_mutex should ALWAYS be locked when this function is called */
553 static int gmx_fio_int_get_file_position(t_fileio *fio, gmx_off_t *offset)
555 /* Flush the file, so we are sure it is written */
556 if (gmx_fio_int_flush(fio))
558 char buf[STRLEN];
559 sprintf(
560 buf,
561 "Cannot write file '%s'; maybe you are out of disk space?",
562 fio->fn);
563 gmx_file(buf);
566 /* We cannot count on XDR being able to write 64-bit integers,
567 so separate into high/low 32-bit values.
568 In case the filesystem has 128-bit offsets we only care
569 about the first 64 bits - we'll have to fix
570 this when exabyte-size output files are common...
572 *offset = gmx_ftell(fio->fp);
574 return 0;
577 int gmx_fio_get_output_file_positions(gmx_file_position_t **p_outputfiles,
578 int *p_nfiles)
580 int nfiles, nalloc;
581 gmx_file_position_t * outputfiles;
582 t_fileio *cur;
584 nfiles = 0;
586 /* pre-allocate 100 files */
587 nalloc = 100;
588 snew(outputfiles, nalloc);
590 Lock openFilesLock(open_file_mutex);
591 cur = gmx_fio_get_first();
592 while (cur)
594 /* Skip the checkpoint files themselves, since they could be open when
595 we call this routine... */
596 if (!cur->bRead && cur->iFTP != efCPT)
598 /* This is an output file currently open for writing, add it */
599 if (nfiles == nalloc)
601 nalloc += 100;
602 srenew(outputfiles, nalloc);
605 std::strncpy(outputfiles[nfiles].filename, cur->fn, STRLEN - 1);
607 /* Get the file position */
608 gmx_fio_int_get_file_position(cur, &outputfiles[nfiles].offset);
609 #ifndef GMX_FAHCORE
610 outputfiles[nfiles].chksum_size
611 = gmx_fio_int_get_file_md5(cur,
612 outputfiles[nfiles].offset,
613 outputfiles[nfiles].chksum);
614 #endif
615 nfiles++;
618 cur = gmx_fio_get_next(cur);
620 *p_nfiles = nfiles;
621 *p_outputfiles = outputfiles;
623 return 0;
627 char *gmx_fio_getname(t_fileio *fio)
629 char *ret;
630 gmx_fio_lock(fio);
631 ret = fio->fn;
632 gmx_fio_unlock(fio);
634 return ret;
637 int gmx_fio_getftp(t_fileio* fio)
639 int ret;
641 gmx_fio_lock(fio);
642 ret = fio->iFTP;
643 gmx_fio_unlock(fio);
645 return ret;
648 void gmx_fio_rewind(t_fileio* fio)
650 gmx_fio_lock(fio);
652 if (fio->xdr)
654 xdr_destroy(fio->xdr);
655 frewind(fio->fp);
656 xdrstdio_create(fio->xdr, fio->fp, fio->xdrmode);
658 else
660 frewind(fio->fp);
662 gmx_fio_unlock(fio);
666 int gmx_fio_flush(t_fileio* fio)
668 int ret;
670 gmx_fio_lock(fio);
671 ret = gmx_fio_int_flush(fio);
672 gmx_fio_unlock(fio);
674 return ret;
679 static int gmx_fio_int_fsync(t_fileio *fio)
681 int rc = 0;
683 if (fio->fp)
685 rc = gmx_fsync(fio->fp);
687 return rc;
691 int gmx_fio_fsync(t_fileio *fio)
693 int rc;
695 gmx_fio_lock(fio);
696 rc = gmx_fio_int_fsync(fio);
697 gmx_fio_unlock(fio);
699 return rc;
704 t_fileio *gmx_fio_all_output_fsync(void)
706 t_fileio *ret = nullptr;
707 t_fileio *cur;
709 Lock openFilesLock(open_file_mutex);
710 cur = gmx_fio_get_first();
711 while (cur)
713 if (!cur->bRead)
715 /* if any of them fails, return failure code */
716 int rc = gmx_fio_int_fsync(cur);
717 if (rc != 0 && !ret)
719 ret = cur;
722 cur = gmx_fio_get_next(cur);
725 /* in addition, we force these to be written out too, if they're being
726 redirected. We don't check for errors because errors most likely mean
727 that they're not redirected. */
728 fflush(stdout);
729 fflush(stderr);
730 #if HAVE_FSYNC
731 /* again, fahcore defines HAVE_FSYNC and fsync() */
732 fsync(STDOUT_FILENO);
733 fsync(STDERR_FILENO);
734 #endif
736 return ret;
740 gmx_off_t gmx_fio_ftell(t_fileio* fio)
742 gmx_off_t ret = 0;
744 gmx_fio_lock(fio);
745 if (fio->fp)
747 ret = gmx_ftell(fio->fp);
749 gmx_fio_unlock(fio);
750 return ret;
753 int gmx_fio_seek(t_fileio* fio, gmx_off_t fpos)
755 int rc;
757 gmx_fio_lock(fio);
758 if (fio->fp)
760 rc = gmx_fseek(fio->fp, fpos, SEEK_SET);
762 else
764 gmx_file(fio->fn);
765 rc = -1;
767 gmx_fio_unlock(fio);
768 return rc;
771 FILE *gmx_fio_getfp(t_fileio *fio)
773 FILE *ret = nullptr;
775 gmx_fio_lock(fio);
776 if (fio->fp)
778 ret = fio->fp;
780 gmx_fio_unlock(fio);
781 return ret;
784 gmx_bool gmx_fio_getread(t_fileio* fio)
786 gmx_bool ret;
788 gmx_fio_lock(fio);
789 ret = fio->bRead;
790 gmx_fio_unlock(fio);
792 return ret;
795 int xtc_seek_time(t_fileio *fio, real time, int natoms, gmx_bool bSeekForwardOnly)
797 int ret;
799 gmx_fio_lock(fio);
800 ret = xdr_xtc_seek_time(time, fio->fp, fio->xdr, natoms, bSeekForwardOnly);
801 gmx_fio_unlock(fio);
803 return ret;