From e7c01c7781dcdfeed5a65047937ee53a7132deba Mon Sep 17 00:00:00 2001 From: Andrew Borodin Date: Wed, 6 Apr 2016 13:19:59 +0300 Subject: [PATCH] Ticket #2193: use Coreutils way to define buffer size to copy file. m4.include/stat-size.m4, lib/stat-size.h: get these files from Gnulib. src/filemanager/ioblksize.h: get this file from Coreutils. Signed-off-by: Andrew Borodin --- acinclude.m4 | 1 + configure.ac | 1 + lib/Makefile.am | 1 + lib/stat-size.h | 111 ++++++++++++++++++++++++++++++++++++++++++++ m4.include/stat-size.m4 | 14 ++++++ src/filemanager/Makefile.am | 1 + src/filemanager/ioblksize.h | 84 +++++++++++++++++++++++++++++++++ 7 files changed, 213 insertions(+) create mode 100644 lib/stat-size.h create mode 100644 m4.include/stat-size.m4 create mode 100644 src/filemanager/ioblksize.h diff --git a/acinclude.m4 b/acinclude.m4 index 861ca2261..fcdf6b28a 100644 --- a/acinclude.m4 +++ b/acinclude.m4 @@ -4,6 +4,7 @@ m4_include([m4.include/dx_doxygen.m4]) m4_include([m4.include/mc-cflags.m4]) m4_include([m4.include/mc-check-search-type.m4]) m4_include([m4.include/mode_t.m4]) +m4_include([m4.include/stat-size.m4]) m4_include([m4.include/ls-mntd-fs.m4]) m4_include([m4.include/fstypename.m4]) m4_include([m4.include/fsusage.m4]) diff --git a/configure.ac b/configure.ac index d0d8358cc..3c7ca181f 100644 --- a/configure.ac +++ b/configure.ac @@ -183,6 +183,7 @@ AC_TYPE_UID_T AC_STRUCT_ST_BLOCKS AC_CHECK_MEMBERS([struct stat.st_blksize, struct stat.st_rdev]) +gl_STAT_SIZE AH_TEMPLATE([sig_atomic_t], [/* Define to `int' if doesn't define.]) diff --git a/lib/Makefile.am b/lib/Makefile.am index 8a419280b..0c08e7ddd 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -41,6 +41,7 @@ libmc_la_SOURCES = \ lock.c lock.h \ serialize.c serialize.h \ shell.c shell.h \ + stat-size.h \ timefmt.c timefmt.h \ timer.c timer.h diff --git a/lib/stat-size.h b/lib/stat-size.h new file mode 100644 index 000000000..16f39a923 --- /dev/null +++ b/lib/stat-size.h @@ -0,0 +1,111 @@ +/* macros useful in interpreting size-related values in struct stat. + Copyright (C) 1989, 1991-2016 Free Software Foundation, Inc. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . */ + +/* *INDENT-OFF* */ +/* + Macros defined by this file (s is an rvalue of type struct stat): + + DEV_BSIZE: The device blocksize. But use ST_NBLOCKSIZE instead. + ST_BLKSIZE(s): Preferred (in the sense of best performance) I/O blocksize + for the file, in bytes. + ST_NBLOCKS(s): Number of blocks in the file, including indirect blocks. + ST_NBLOCKSIZE: Size of blocks used when calculating ST_NBLOCKS. + */ +/* *INDENT-ON* */ + +#ifndef STAT_SIZE_H +#define STAT_SIZE_H + +/* sys/param.h may define DEV_BSIZE */ +#if HAVE_SYS_PARAM_H +#include +#endif + + +/* Get or fake the disk device blocksize. + Usually defined by sys/param.h (if at all). */ +#if !defined DEV_BSIZE && defined BSIZE +#define DEV_BSIZE BSIZE +#endif +#if !defined DEV_BSIZE && defined BBSIZE /* SGI sys/param.h */ +#define DEV_BSIZE BBSIZE +#endif +#ifndef DEV_BSIZE +#define DEV_BSIZE 4096 +#endif + + + +/* Extract or fake data from a 'struct stat'. + ST_BLKSIZE: Preferred I/O blocksize for the file, in bytes. + ST_NBLOCKS: Number of blocks in the file, including indirect blocks. + ST_NBLOCKSIZE: Size of blocks used when calculating ST_NBLOCKS. */ +#ifndef HAVE_STRUCT_STAT_ST_BLOCKS +#define ST_BLKSIZE(statbuf) DEV_BSIZE + /* coreutils' fileblocks.c also uses BSIZE. */ +#if defined _POSIX_SOURCE || !defined BSIZE +#define ST_NBLOCKS(statbuf) \ + ((statbuf).st_size / ST_NBLOCKSIZE + ((statbuf).st_size % ST_NBLOCKSIZE != 0)) +#else + /* This definition calls st_blocks, which is in the fileblocks module. */ +#define ST_NBLOCKS(statbuf) \ + (S_ISREG ((statbuf).st_mode) || S_ISDIR ((statbuf).st_mode) ? \ + st_blocks ((statbuf).st_size) : 0) +#endif +#else +/* Some systems, like Sequents, return st_blksize of 0 on pipes. + Also, when running 'rsh hpux11-system cat any-file', cat would + determine that the output stream had an st_blksize of 2147421096. + Conversely st_blksize can be 2 GiB (or maybe even larger) with XFS + on 64-bit hosts. Somewhat arbitrarily, limit the "optimal" block + size to SIZE_MAX / 8 + 1. (Dividing SIZE_MAX by only 4 wouldn't + suffice, since "cat" sometimes multiplies the result by 4.) If + anyone knows of a system for which this limit is too small, please + report it as a bug in this code. */ +#define ST_BLKSIZE(statbuf) ((0 < (statbuf).st_blksize \ + && (statbuf).st_blksize <= ((size_t)-1) / 8 + 1) \ + ? (statbuf).st_blksize : DEV_BSIZE) +#if defined hpux || defined __hpux__ || defined __hpux + /* HP-UX counts st_blocks in 1024-byte units. + This loses when mixing HP-UX and BSD file systems with NFS. */ +#define ST_NBLOCKSIZE 1024 +#else /* !hpux */ +#if defined _AIX && defined _I386 + /* AIX PS/2 counts st_blocks in 4K units. */ +#define ST_NBLOCKSIZE (4 * 1024) +#else +#if defined _CRAY +#define ST_NBLOCKS(statbuf) \ + (S_ISREG ((statbuf).st_mode) || S_ISDIR ((statbuf).st_mode) \ + ? (statbuf).st_blocks * ST_BLKSIZE (statbuf) / ST_NBLOCKSIZE : 0) +#endif +#endif +#endif +#endif + +#ifndef ST_NBLOCKS +#define ST_NBLOCKS(statbuf) ((statbuf).st_blocks) +#endif + +#ifndef ST_NBLOCKSIZE +#ifdef S_BLKSIZE +#define ST_NBLOCKSIZE S_BLKSIZE +#else +#define ST_NBLOCKSIZE 512 +#endif +#endif + +#endif /* STAT_SIZE_H */ diff --git a/m4.include/stat-size.m4 b/m4.include/stat-size.m4 new file mode 100644 index 000000000..86d409864 --- /dev/null +++ b/m4.include/stat-size.m4 @@ -0,0 +1,14 @@ +#serial 1 + +# Copyright (C) 2011-2016 Free Software Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +AC_DEFUN([gl_STAT_SIZE], +[ + # Don't call AC_STRUCT_ST_BLOCKS because it causes bugs. Details at + # http://lists.gnu.org/archive/html/bug-gnulib/2011-06/msg00051.html + AC_CHECK_HEADERS_ONCE([sys/param.h]) +]) diff --git a/src/filemanager/Makefile.am b/src/filemanager/Makefile.am index 7188dc500..b593b7025 100644 --- a/src/filemanager/Makefile.am +++ b/src/filemanager/Makefile.am @@ -17,6 +17,7 @@ libmcfilemanager_la_SOURCES = \ find.c find.h \ hotlist.c hotlist.h \ info.c info.h \ + ioblksize.h \ layout.c layout.h \ midnight.h midnight.c \ mountlist.c mountlist.h \ diff --git a/src/filemanager/ioblksize.h b/src/filemanager/ioblksize.h new file mode 100644 index 000000000..05a08bc81 --- /dev/null +++ b/src/filemanager/ioblksize.h @@ -0,0 +1,84 @@ +/* I/O block size definitions for coreutils + Copyright (C) 1989-2016 Free Software Foundation, Inc. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . */ + +/* Include this file _after_ system headers if possible. */ + +/* sys/stat.h will already have been included by system.h. */ +#include "lib/stat-size.h" + +/* *INDENT-OFF* */ + +/* As of May 2014, 128KiB is determined to be the minimium + blksize to best minimize system call overhead. + This can be tested with this script: + + for i in $(seq 0 10); do + bs=$((1024*2**$i)) + printf "%7s=" $bs + timeout --foreground -sINT 2 \ + dd bs=$bs if=/dev/zero of=/dev/null 2>&1 \ + | sed -n 's/.* \([0-9.]* [GM]B\/s\)/\1/p' + done + + With the results shown for these systems: + system #1: 1.7GHz pentium-m with 400MHz DDR2 RAM, arch=i686 + system #2: 2.1GHz i3-2310M with 1333MHz DDR3 RAM, arch=x86_64 + system #3: 3.2GHz i7-970 with 1333MHz DDR3, arch=x86_64 + system #4: 2.20GHz Xeon E5-2660 with 1333MHz DDR3, arch=x86_64 + system #5: 2.30GHz i7-3615QM with 1600MHz DDR3, arch=x86_64 + system #6: 1.30GHz i5-4250U with 1-channel 1600MHz DDR3, arch=x86_64 + system #7: 3.55GHz IBM,8231-E2B with 1066MHz DDR3, POWER7 revision 2.1 + + per-system transfer rate (GB/s) + blksize #1 #2 #3 #4 #5 #6 #7 + ------------------------------------------------------------------------ + 1024 .73 1.7 2.6 .64 1.0 2.5 1.3 + 2048 1.3 3.0 4.4 1.2 2.0 4.4 2.5 + 4096 2.4 5.1 6.5 2.3 3.7 7.4 4.8 + 8192 3.5 7.3 8.5 4.0 6.0 10.4 9.2 + 16384 3.9 9.4 10.1 6.3 8.3 13.3 16.8 + 32768 5.2 9.9 11.1 8.1 10.7 13.2 28.0 + 65536 5.3 11.2 12.0 10.6 12.8 16.1 41.4 + 131072 5.5 11.8 12.3 12.1 14.0 16.7 54.8 + 262144 5.7 11.6 12.5 12.3 14.7 16.4 40.0 + 524288 5.7 11.4 12.5 12.1 14.7 15.5 34.5 + 1048576 5.8 11.4 12.6 12.2 14.9 15.7 36.5 + + + Note that this is to minimize system call overhead. + Other values may be appropriate to minimize file system + or disk overhead. For example on my current GNU/Linux system + the readahead setting is 128KiB which was read using: + + file="." + device=$(df --output=source --local "$file" | tail -n1) + echo $(( $(blockdev --getra $device) * 512 )) + + However there isn't a portable way to get the above. + In the future we could use the above method if available + and default to io_blksize() if not. + */ + + +enum { IO_BUFSIZE = 128 * 1024 }; + +/* *INDENT-ON* */ + +static inline size_t +io_blksize (struct stat sb) +{ + return MAX (IO_BUFSIZE, ST_BLKSIZE (sb)); +} -- 2.11.4.GIT