fs/read_write.c

   1 /*
   2  *  linux/fs/read_write.c
   3  *
   4  *  Copyright (C) 1991, 1992  Linus Torvalds
   5  */
   6
   7 #include <linux/slab.h>
   8 #include <linux/stat.h>
   9 #include <linux/fcntl.h>
  10 #include <linux/file.h>
  11 #include <linux/uio.h>
  12 #include <linux/smp_lock.h>
  13 #include <linux/fsnotify.h>
  14 #include <linux/security.h>
  15 #include <linux/module.h>
  16 #include <linux/syscalls.h>
  17 #include <linux/pagemap.h>
  18 #include <linux/splice.h>
  19 #include "read_write.h"
  20
  21 #include <asm/uaccess.h>
  22 #include <asm/unistd.h>
  23
  24 const struct file_operations generic_ro_fops = {
  25         .llseek         = generic_file_llseek,
  26         .read           = do_sync_read,
  27         .aio_read       = generic_file_aio_read,
  28         .mmap           = generic_file_readonly_mmap,
  29         .splice_read    = generic_file_splice_read,
  30 };
  31
  32 EXPORT_SYMBOL(generic_ro_fops);
  33
  34 /**
  35  * generic_file_llseek_unlocked - lockless generic llseek implementation
  36  * @file:       file structure to seek on
  37  * @offset:     file offset to seek to
  38  * @origin:     type of seek
  39  *
  40  * Updates the file offset to the value specified by @offset and @origin.
  41  * Locking must be provided by the caller.
  42  */
  43 loff_t
  44 generic_file_llseek_unlocked(struct file *file, loff_t offset, int origin)
  45 {
  46         struct inode *inode = file->f_mapping->host;
  47
  48         switch (origin) {
  49         case SEEK_END:
  50                 offset += inode->i_size;
  51                 break;
  52         case SEEK_CUR:
  53                 /*
  54                  * Here we special-case the lseek(fd, 0, SEEK_CUR)
  55                  * position-querying operation.  Avoid rewriting the "same"
  56                  * f_pos value back to the file because a concurrent read(),
  57                  * write() or lseek() might have altered it
  58                  */
  59                 if (offset == 0)
  60                         return file->f_pos;
  61                 offset += file->f_pos;
  62                 break;
  63         }
  64
  65         if (offset < 0 || offset > inode->i_sb->s_maxbytes)
  66                 return -EINVAL;
  67
  68         /* Special lock needed here? */
  69         if (offset != file->f_pos) {
  70                 file->f_pos = offset;
  71                 file->f_version = 0;
  72         }
  73
  74         return offset;
  75 }
  76 EXPORT_SYMBOL(generic_file_llseek_unlocked);
  77
  78 /**
  79  * generic_file_llseek - generic llseek implementation for regular files
  80  * @file:       file structure to seek on
  81  * @offset:     file offset to seek to
  82  * @origin:     type of seek
  83  *
  84  * This is a generic implemenation of ->llseek useable for all normal local
  85  * filesystems.  It just updates the file offset to the value specified by
  86  * @offset and @origin under i_mutex.
  87  */
  88 loff_t generic_file_llseek(struct file *file, loff_t offset, int origin)
  89 {
  90         loff_t rval;
  91
  92         mutex_lock(&file->f_dentry->d_inode->i_mutex);
  93         rval = generic_file_llseek_unlocked(file, offset, origin);
  94         mutex_unlock(&file->f_dentry->d_inode->i_mutex);
  95
  96         return rval;
  97 }
  98 EXPORT_SYMBOL(generic_file_llseek);
  99
 100 loff_t no_llseek(struct file *file, loff_t offset, int origin)
 101 {
 102         return -ESPIPE;
 103 }
 104 EXPORT_SYMBOL(no_llseek);
 105
 106 loff_t default_llseek(struct file *file, loff_t offset, int origin)
 107 {
 108         loff_t retval;
 109
 110         lock_kernel();
 111         switch (origin) {
 112                 case SEEK_END:
 113                         offset += i_size_read(file->f_path.dentry->d_inode);
 114                         break;
 115                 case SEEK_CUR:
 116                         if (offset == 0) {
 117                                 retval = file->f_pos;
 118                                 goto out;
 119                         }
 120                         offset += file->f_pos;
 121         }
 122         retval = -EINVAL;
 123         if (offset >= 0) {
 124                 if (offset != file->f_pos) {
 125                         file->f_pos = offset;
 126                         file->f_version = 0;
 127                 }
 128                 retval = offset;
 129         }
 130 out:
 131         unlock_kernel();
 132         return retval;
 133 }
 134 EXPORT_SYMBOL(default_llseek);
 135
 136 loff_t vfs_llseek(struct file *file, loff_t offset, int origin)
 137 {
 138         loff_t (*fn)(struct file *, loff_t, int);
 139
 140         fn = no_llseek;
 141         if (file->f_mode & FMODE_LSEEK) {
 142                 fn = default_llseek;
 143                 if (file->f_op && file->f_op->llseek)
 144                         fn = file->f_op->llseek;
 145         }
 146         return fn(file, offset, origin);
 147 }
 148 EXPORT_SYMBOL(vfs_llseek);
 149
 150 SYSCALL_DEFINE3(lseek, unsigned int, fd, off_t, offset, unsigned int, origin)
 151 {
 152         off_t retval;
 153         struct file * file;
 154         int fput_needed;
 155
 156         retval = -EBADF;
 157         file = fget_light(fd, &fput_needed);
 158         if (!file)
 159                 goto bad;
 160
 161         retval = -EINVAL;
 162         if (origin <= SEEK_MAX) {
 163                 loff_t res = vfs_llseek(file, offset, origin);
 164                 retval = res;
 165                 if (res != (loff_t)retval)
 166                         retval = -EOVERFLOW;    /* LFS: should only happen on 32 bit platforms */
 167         }
 168         fput_light(file, fput_needed);
 169 bad:
 170         return retval;
 171 }
 172
 173 #ifdef __ARCH_WANT_SYS_LLSEEK
 174 SYSCALL_DEFINE5(llseek, unsigned int, fd, unsigned long, offset_high,
 175                 unsigned long, offset_low, loff_t __user *, result,
 176                 unsigned int, origin)
 177 {
 178         int retval;
 179         struct file * file;
 180         loff_t offset;
 181         int fput_needed;
 182
 183         retval = -EBADF;
 184         file = fget_light(fd, &fput_needed);
 185         if (!file)
 186                 goto bad;
 187
 188         retval = -EINVAL;
 189         if (origin > SEEK_MAX)
 190                 goto out_putf;
 191
 192         offset = vfs_llseek(file, ((loff_t) offset_high << 32) | offset_low,
 193                         origin);
 194
 195         retval = (int)offset;
 196         if (offset >= 0) {
 197                 retval = -EFAULT;
 198                 if (!copy_to_user(result, &offset, sizeof(offset)))
 199                         retval = 0;
 200         }
 201 out_putf:
 202         fput_light(file, fput_needed);
 203 bad:
 204         return retval;
 205 }
 206 #endif
 207
 208 /*
 209  * rw_verify_area doesn't like huge counts. We limit
 210  * them to something that fits in "int" so that others
 211  * won't have to do range checks all the time.
 212  */
 213 #define MAX_RW_COUNT (INT_MAX & PAGE_CACHE_MASK)
 214
 215 int rw_verify_area(int read_write, struct file *file, loff_t *ppos, size_t count)
 216 {
 217         struct inode *inode;
 218         loff_t pos;
 219         int retval = -EINVAL;
 220
 221         inode = file->f_path.dentry->d_inode;
 222         if (unlikely((ssize_t) count < 0))
 223                 return retval;
 224         pos = *ppos;
 225         if (unlikely((pos < 0) || (loff_t) (pos + count) < 0))
 226                 return retval;
 227
 228         if (unlikely(inode->i_flock && mandatory_lock(inode))) {
 229                 retval = locks_mandatory_area(
 230                         read_write == READ ? FLOCK_VERIFY_READ : FLOCK_VERIFY_WRITE,
 231                         inode, file, pos, count);
 232                 if (retval < 0)
 233                         return retval;
 234         }
 235         retval = security_file_permission(file,
 236                                 read_write == READ ? MAY_READ : MAY_WRITE);
 237         if (retval)
 238                 return retval;
 239         return count > MAX_RW_COUNT ? MAX_RW_COUNT : count;
 240 }
 241
 242 static void wait_on_retry_sync_kiocb(struct kiocb *iocb)
 243 {
 244         set_current_state(TASK_UNINTERRUPTIBLE);
 245         if (!kiocbIsKicked(iocb))
 246                 schedule();
 247         else
 248                 kiocbClearKicked(iocb);
 249         __set_current_state(TASK_RUNNING);
 250 }
 251
 252 ssize_t do_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos)
 253 {
 254         struct iovec iov = { .iov_base = buf, .iov_len = len };
 255         struct kiocb kiocb;
 256         ssize_t ret;
 257
 258         init_sync_kiocb(&kiocb, filp);
 259         kiocb.ki_pos = *ppos;
 260         kiocb.ki_left = len;
 261         kiocb.ki_nbytes = len;
 262
 263         for (;;) {
 264                 ret = filp->f_op->aio_read(&kiocb, &iov, 1, kiocb.ki_pos);
 265                 if (ret != -EIOCBRETRY)
 266                         break;
 267                 wait_on_retry_sync_kiocb(&kiocb);
 268         }
 269
 270         if (-EIOCBQUEUED == ret)
 271                 ret = wait_on_sync_kiocb(&kiocb);
 272         *ppos = kiocb.ki_pos;
 273         return ret;
 274 }
 275
 276 EXPORT_SYMBOL(do_sync_read);
 277
 278 ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos)
 279 {
 280         ssize_t ret;
 281
 282         if (!(file->f_mode & FMODE_READ))
 283                 return -EBADF;
 284         if (!file->f_op || (!file->f_op->read && !file->f_op->aio_read))
 285                 return -EINVAL;
 286         if (unlikely(!access_ok(VERIFY_WRITE, buf, count)))
 287                 return -EFAULT;
 288
 289         ret = rw_verify_area(READ, file, pos, count);
 290         if (ret >= 0) {
 291                 count = ret;
 292                 if (file->f_op->read)
 293                         ret = file->f_op->read(file, buf, count, pos);
 294                 else
 295                         ret = do_sync_read(file, buf, count, pos);
 296                 if (ret > 0) {
 297                         fsnotify_access(file->f_path.dentry);
 298                         add_rchar(current, ret);
 299                 }
 300                 inc_syscr(current);
 301         }
 302
 303         return ret;
 304 }
 305
 306 EXPORT_SYMBOL(vfs_read);
 307
 308 ssize_t do_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos)
 309 {
 310         struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = len };
 311         struct kiocb kiocb;
 312         ssize_t ret;
 313
 314         init_sync_kiocb(&kiocb, filp);
 315         kiocb.ki_pos = *ppos;
 316         kiocb.ki_left = len;
 317         kiocb.ki_nbytes = len;
 318
 319         for (;;) {
 320                 ret = filp->f_op->aio_write(&kiocb, &iov, 1, kiocb.ki_pos);
 321                 if (ret != -EIOCBRETRY)
 322                         break;
 323                 wait_on_retry_sync_kiocb(&kiocb);
 324         }
 325
 326         if (-EIOCBQUEUED == ret)
 327                 ret = wait_on_sync_kiocb(&kiocb);
 328         *ppos = kiocb.ki_pos;
 329         return ret;
 330 }
 331
 332 EXPORT_SYMBOL(do_sync_write);
 333
 334 ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_t *pos)
 335 {
 336         ssize_t ret;
 337
 338         if (!(file->f_mode & FMODE_WRITE))
 339                 return -EBADF;
 340         if (!file->f_op || (!file->f_op->write && !file->f_op->aio_write))
 341                 return -EINVAL;
 342         if (unlikely(!access_ok(VERIFY_READ, buf, count)))
 343                 return -EFAULT;
 344
 345         ret = rw_verify_area(WRITE, file, pos, count);
 346         if (ret >= 0) {
 347                 count = ret;
 348                 if (file->f_op->write)
 349                         ret = file->f_op->write(file, buf, count, pos);
 350                 else
 351                         ret = do_sync_write(file, buf, count, pos);
 352                 if (ret > 0) {
 353                         fsnotify_modify(file->f_path.dentry);
 354                         add_wchar(current, ret);
 355                 }
 356                 inc_syscw(current);
 357         }
 358
 359         return ret;
 360 }
 361
 362 EXPORT_SYMBOL(vfs_write);
 363
 364 static inline loff_t file_pos_read(struct file *file)
 365 {
 366         return file->f_pos;
 367 }
 368
 369 static inline void file_pos_write(struct file *file, loff_t pos)
 370 {
 371         file->f_pos = pos;
 372 }
 373
 374 SYSCALL_DEFINE3(read, unsigned int, fd, char __user *, buf, size_t, count)
 375 {
 376         struct file *file;
 377         ssize_t ret = -EBADF;
 378         int fput_needed;
 379
 380         file = fget_light(fd, &fput_needed);
 381         if (file) {
 382                 loff_t pos = file_pos_read(file);
 383                 ret = vfs_read(file, buf, count, &pos);
 384                 file_pos_write(file, pos);
 385                 fput_light(file, fput_needed);
 386         }
 387
 388         return ret;
 389 }
 390
 391 SYSCALL_DEFINE3(write, unsigned int, fd, const char __user *, buf,
 392                 size_t, count)
 393 {
 394         struct file *file;
 395         ssize_t ret = -EBADF;
 396         int fput_needed;
 397
 398         file = fget_light(fd, &fput_needed);
 399         if (file) {
 400                 loff_t pos = file_pos_read(file);
 401                 ret = vfs_write(file, buf, count, &pos);
 402                 file_pos_write(file, pos);
 403                 fput_light(file, fput_needed);
 404         }
 405
 406         return ret;
 407 }
 408
 409 SYSCALL_DEFINE(pread64)(unsigned int fd, char __user *buf,
 410                         size_t count, loff_t pos)
 411 {
 412         struct file *file;
 413         ssize_t ret = -EBADF;
 414         int fput_needed;
 415
 416         if (pos < 0)
 417                 return -EINVAL;
 418
 419         file = fget_light(fd, &fput_needed);
 420         if (file) {
 421                 ret = -ESPIPE;
 422                 if (file->f_mode & FMODE_PREAD)
 423                         ret = vfs_read(file, buf, count, &pos);
 424                 fput_light(file, fput_needed);
 425         }
 426
 427         return ret;
 428 }
 429 #ifdef CONFIG_HAVE_SYSCALL_WRAPPERS
 430 asmlinkage long SyS_pread64(long fd, long buf, long count, loff_t pos)
 431 {
 432         return SYSC_pread64((unsigned int) fd, (char __user *) buf,
 433                             (size_t) count, pos);
 434 }
 435 SYSCALL_ALIAS(sys_pread64, SyS_pread64);
 436 #endif
 437
 438 SYSCALL_DEFINE(pwrite64)(unsigned int fd, const char __user *buf,
 439                          size_t count, loff_t pos)
 440 {
 441         struct file *file;
 442         ssize_t ret = -EBADF;
 443         int fput_needed;
 444
 445         if (pos < 0)
 446                 return -EINVAL;
 447
 448         file = fget_light(fd, &fput_needed);
 449         if (file) {
 450                 ret = -ESPIPE;
 451                 if (file->f_mode & FMODE_PWRITE)
 452                         ret = vfs_write(file, buf, count, &pos);
 453                 fput_light(file, fput_needed);
 454         }
 455
 456         return ret;
 457 }
 458 #ifdef CONFIG_HAVE_SYSCALL_WRAPPERS
 459 asmlinkage long SyS_pwrite64(long fd, long buf, long count, loff_t pos)
 460 {
 461         return SYSC_pwrite64((unsigned int) fd, (const char __user *) buf,
 462                              (size_t) count, pos);
 463 }
 464 SYSCALL_ALIAS(sys_pwrite64, SyS_pwrite64);
 465 #endif
 466
 467 /*
 468  * Reduce an iovec's length in-place.  Return the resulting number of segments
 469  */
 470 unsigned long iov_shorten(struct iovec *iov, unsigned long nr_segs, size_t to)
 471 {
 472         unsigned long seg = 0;
 473         size_t len = 0;
 474
 475         while (seg < nr_segs) {
 476                 seg++;
 477                 if (len + iov->iov_len >= to) {
 478                         iov->iov_len = to - len;
 479                         break;
 480                 }
 481                 len += iov->iov_len;
 482                 iov++;
 483         }
 484         return seg;
 485 }
 486 EXPORT_SYMBOL(iov_shorten);
 487
 488 ssize_t do_sync_readv_writev(struct file *filp, const struct iovec *iov,
 489                 unsigned long nr_segs, size_t len, loff_t *ppos, iov_fn_t fn)
 490 {
 491         struct kiocb kiocb;
 492         ssize_t ret;
 493
 494         init_sync_kiocb(&kiocb, filp);
 495         kiocb.ki_pos = *ppos;
 496         kiocb.ki_left = len;
 497         kiocb.ki_nbytes = len;
 498
 499         for (;;) {
 500                 ret = fn(&kiocb, iov, nr_segs, kiocb.ki_pos);
 501                 if (ret != -EIOCBRETRY)
 502                         break;
 503                 wait_on_retry_sync_kiocb(&kiocb);
 504         }
 505
 506         if (ret == -EIOCBQUEUED)
 507                 ret = wait_on_sync_kiocb(&kiocb);
 508         *ppos = kiocb.ki_pos;
 509         return ret;
 510 }
 511
 512 /* Do it by hand, with file-ops */
 513 ssize_t do_loop_readv_writev(struct file *filp, struct iovec *iov,
 514                 unsigned long nr_segs, loff_t *ppos, io_fn_t fn)
 515 {
 516         struct iovec *vector = iov;
 517         ssize_t ret = 0;
 518
 519         while (nr_segs > 0) {
 520                 void __user *base;
 521                 size_t len;
 522                 ssize_t nr;
 523
 524                 base = vector->iov_base;
 525                 len = vector->iov_len;
 526                 vector++;
 527                 nr_segs--;
 528
 529                 nr = fn(filp, base, len, ppos);
 530
 531                 if (nr < 0) {
 532                         if (!ret)
 533                                 ret = nr;
 534                         break;
 535                 }
 536                 ret += nr;
 537                 if (nr != len)
 538                         break;
 539         }
 540
 541         return ret;
 542 }
 543
 544 /* A write operation does a read from user space and vice versa */
 545 #define vrfy_dir(type) ((type) == READ ? VERIFY_WRITE : VERIFY_READ)
 546
 547 ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector,
 548                               unsigned long nr_segs, unsigned long fast_segs,
 549                               struct iovec *fast_pointer,
 550                               struct iovec **ret_pointer)
 551   {
 552         unsigned long seg;
 553         ssize_t ret;
 554         struct iovec *iov = fast_pointer;
 555
 556         /*
 557          * SuS says "The readv() function *may* fail if the iovcnt argument
 558          * was less than or equal to 0, or greater than {IOV_MAX}.  Linux has
 559          * traditionally returned zero for zero segments, so...
 560          */
 561         if (nr_segs == 0) {
 562                 ret = 0;
 563                 goto out;
 564         }
 565
 566         /*
 567          * First get the "struct iovec" from user memory and
 568          * verify all the pointers
 569          */
 570         if (nr_segs > UIO_MAXIOV) {
 571                 ret = -EINVAL;
 572                 goto out;
 573         }
 574         if (nr_segs > fast_segs) {
 575                 iov = kmalloc(nr_segs*sizeof(struct iovec), GFP_KERNEL);
 576                 if (iov == NULL) {
 577                         ret = -ENOMEM;
 578                         goto out;
 579                 }
 580         }
 581         if (copy_from_user(iov, uvector, nr_segs*sizeof(*uvector))) {
 582                 ret = -EFAULT;
 583                 goto out;
 584         }
 585
 586         /*
 587          * According to the Single Unix Specification we should return EINVAL
 588          * if an element length is < 0 when cast to ssize_t or if the
 589          * total length would overflow the ssize_t return value of the
 590          * system call.
 591          */
 592         ret = 0;
 593         for (seg = 0; seg < nr_segs; seg++) {
 594                 void __user *buf = iov[seg].iov_base;
 595                 ssize_t len = (ssize_t)iov[seg].iov_len;
 596
 597                 /* see if we we're about to use an invalid len or if
 598                  * it's about to overflow ssize_t */
 599                 if (len < 0 || (ret + len < ret)) {
 600                         ret = -EINVAL;
 601                         goto out;
 602                 }
 603                 if (unlikely(!access_ok(vrfy_dir(type), buf, len))) {
 604                         ret = -EFAULT;
 605                         goto out;
 606                 }
 607
 608                 ret += len;
 609         }
 610 out:
 611         *ret_pointer = iov;
 612         return ret;
 613 }
 614
 615 static ssize_t do_readv_writev(int type, struct file *file,
 616                                const struct iovec __user * uvector,
 617                                unsigned long nr_segs, loff_t *pos)
 618 {
 619         size_t tot_len;
 620         struct iovec iovstack[UIO_FASTIOV];
 621         struct iovec *iov = iovstack;
 622         ssize_t ret;
 623         io_fn_t fn;
 624         iov_fn_t fnv;
 625
 626         if (!file->f_op) {
 627                 ret = -EINVAL;
 628                 goto out;
 629         }
 630
 631         ret = rw_copy_check_uvector(type, uvector, nr_segs,
 632                         ARRAY_SIZE(iovstack), iovstack, &iov);
 633         if (ret <= 0)
 634                 goto out;
 635
 636         tot_len = ret;
 637         ret = rw_verify_area(type, file, pos, tot_len);
 638         if (ret < 0)
 639                 goto out;
 640
 641         fnv = NULL;
 642         if (type == READ) {
 643                 fn = file->f_op->read;
 644                 fnv = file->f_op->aio_read;
 645         } else {
 646                 fn = (io_fn_t)file->f_op->write;
 647                 fnv = file->f_op->aio_write;
 648         }
 649
 650         if (fnv)
 651                 ret = do_sync_readv_writev(file, iov, nr_segs, tot_len,
 652                                                 pos, fnv);
 653         else
 654                 ret = do_loop_readv_writev(file, iov, nr_segs, pos, fn);
 655
 656 out:
 657         if (iov != iovstack)
 658                 kfree(iov);
 659         if ((ret + (type == READ)) > 0) {
 660                 if (type == READ)
 661                         fsnotify_access(file->f_path.dentry);
 662                 else
 663                         fsnotify_modify(file->f_path.dentry);
 664         }
 665         return ret;
 666 }
 667
 668 ssize_t vfs_readv(struct file *file, const struct iovec __user *vec,
 669                   unsigned long vlen, loff_t *pos)
 670 {
 671         if (!(file->f_mode & FMODE_READ))
 672                 return -EBADF;
 673         if (!file->f_op || (!file->f_op->aio_read && !file->f_op->read))
 674                 return -EINVAL;
 675
 676         return do_readv_writev(READ, file, vec, vlen, pos);
 677 }
 678
 679 EXPORT_SYMBOL(vfs_readv);
 680
 681 ssize_t vfs_writev(struct file *file, const struct iovec __user *vec,
 682                    unsigned long vlen, loff_t *pos)
 683 {
 684         if (!(file->f_mode & FMODE_WRITE))
 685                 return -EBADF;
 686         if (!file->f_op || (!file->f_op->aio_write && !file->f_op->write))
 687                 return -EINVAL;
 688
 689         return do_readv_writev(WRITE, file, vec, vlen, pos);
 690 }
 691
 692 EXPORT_SYMBOL(vfs_writev);
 693
 694 SYSCALL_DEFINE3(readv, unsigned long, fd, const struct iovec __user *, vec,
 695                 unsigned long, vlen)
 696 {
 697         struct file *file;
 698         ssize_t ret = -EBADF;
 699         int fput_needed;
 700
 701         file = fget_light(fd, &fput_needed);
 702         if (file) {
 703                 loff_t pos = file_pos_read(file);
 704                 ret = vfs_readv(file, vec, vlen, &pos);
 705                 file_pos_write(file, pos);
 706                 fput_light(file, fput_needed);
 707         }
 708
 709         if (ret > 0)
 710                 add_rchar(current, ret);
 711         inc_syscr(current);
 712         return ret;
 713 }
 714
 715 SYSCALL_DEFINE3(writev, unsigned long, fd, const struct iovec __user *, vec,
 716                 unsigned long, vlen)
 717 {
 718         struct file *file;
 719         ssize_t ret = -EBADF;
 720         int fput_needed;
 721
 722         file = fget_light(fd, &fput_needed);
 723         if (file) {
 724                 loff_t pos = file_pos_read(file);
 725                 ret = vfs_writev(file, vec, vlen, &pos);
 726                 file_pos_write(file, pos);
 727                 fput_light(file, fput_needed);
 728         }
 729
 730         if (ret > 0)
 731                 add_wchar(current, ret);
 732         inc_syscw(current);
 733         return ret;
 734 }
 735
 736 static inline loff_t pos_from_hilo(unsigned long high, unsigned long low)
 737 {
 738 #define HALF_LONG_BITS (BITS_PER_LONG / 2)
 739         return (((loff_t)high << HALF_LONG_BITS) << HALF_LONG_BITS) | low;
 740 }
 741
 742 SYSCALL_DEFINE5(preadv, unsigned long, fd, const struct iovec __user *, vec,
 743                 unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
 744 {
 745         loff_t pos = pos_from_hilo(pos_h, pos_l);
 746         struct file *file;
 747         ssize_t ret = -EBADF;
 748         int fput_needed;
 749
 750         if (pos < 0)
 751                 return -EINVAL;
 752
 753         file = fget_light(fd, &fput_needed);
 754         if (file) {
 755                 ret = -ESPIPE;
 756                 if (file->f_mode & FMODE_PREAD)
 757                         ret = vfs_readv(file, vec, vlen, &pos);
 758                 fput_light(file, fput_needed);
 759         }
 760
 761         if (ret > 0)
 762                 add_rchar(current, ret);
 763         inc_syscr(current);
 764         return ret;
 765 }
 766
 767 SYSCALL_DEFINE5(pwritev, unsigned long, fd, const struct iovec __user *, vec,
 768                 unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
 769 {
 770         loff_t pos = pos_from_hilo(pos_h, pos_l);
 771         struct file *file;
 772         ssize_t ret = -EBADF;
 773         int fput_needed;
 774
 775         if (pos < 0)
 776                 return -EINVAL;
 777
 778         file = fget_light(fd, &fput_needed);
 779         if (file) {
 780                 ret = -ESPIPE;
 781                 if (file->f_mode & FMODE_PWRITE)
 782                         ret = vfs_writev(file, vec, vlen, &pos);
 783                 fput_light(file, fput_needed);
 784         }
 785
 786         if (ret > 0)
 787                 add_wchar(current, ret);
 788         inc_syscw(current);
 789         return ret;
 790 }
 791
 792 static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
 793                            size_t count, loff_t max)
 794 {
 795         struct file * in_file, * out_file;
 796         struct inode * in_inode, * out_inode;
 797         loff_t pos;
 798         ssize_t retval;
 799         int fput_needed_in, fput_needed_out, fl;
 800
 801         /*
 802          * Get input file, and verify that it is ok..
 803          */
 804         retval = -EBADF;
 805         in_file = fget_light(in_fd, &fput_needed_in);
 806         if (!in_file)
 807                 goto out;
 808         if (!(in_file->f_mode & FMODE_READ))
 809                 goto fput_in;
 810         retval = -ESPIPE;
 811         if (!ppos)
 812                 ppos = &in_file->f_pos;
 813         else
 814                 if (!(in_file->f_mode & FMODE_PREAD))
 815                         goto fput_in;
 816         retval = rw_verify_area(READ, in_file, ppos, count);
 817         if (retval < 0)
 818                 goto fput_in;
 819         count = retval;
 820
 821         /*
 822          * Get output file, and verify that it is ok..
 823          */
 824         retval = -EBADF;
 825         out_file = fget_light(out_fd, &fput_needed_out);
 826         if (!out_file)
 827                 goto fput_in;
 828         if (!(out_file->f_mode & FMODE_WRITE))
 829                 goto fput_out;
 830         retval = -EINVAL;
 831         in_inode = in_file->f_path.dentry->d_inode;
 832         out_inode = out_file->f_path.dentry->d_inode;
 833         retval = rw_verify_area(WRITE, out_file, &out_file->f_pos, count);
 834         if (retval < 0)
 835                 goto fput_out;
 836         count = retval;
 837
 838         if (!max)
 839                 max = min(in_inode->i_sb->s_maxbytes, out_inode->i_sb->s_maxbytes);
 840
 841         pos = *ppos;
 842         if (unlikely(pos + count > max)) {
 843                 retval = -EOVERFLOW;
 844                 if (pos >= max)
 845                         goto fput_out;
 846                 count = max - pos;
 847         }
 848
 849         fl = 0;
 850 #if 0
 851         /*
 852          * We need to debate whether we can enable this or not. The
 853          * man page documents EAGAIN return for the output at least,
 854          * and the application is arguably buggy if it doesn't expect
 855          * EAGAIN on a non-blocking file descriptor.
 856          */
 857         if (in_file->f_flags & O_NONBLOCK)
 858                 fl = SPLICE_F_NONBLOCK;
 859 #endif
 860         retval = do_splice_direct(in_file, ppos, out_file, count, fl);
 861
 862         if (retval > 0) {
 863                 add_rchar(current, retval);
 864                 add_wchar(current, retval);
 865         }
 866
 867         inc_syscr(current);
 868         inc_syscw(current);
 869         if (*ppos > max)
 870                 retval = -EOVERFLOW;
 871
 872 fput_out:
 873         fput_light(out_file, fput_needed_out);
 874 fput_in:
 875         fput_light(in_file, fput_needed_in);
 876 out:
 877         return retval;
 878 }
 879
 880 SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd, off_t __user *, offset, size_t, count)
 881 {
 882         loff_t pos;
 883         off_t off;
 884         ssize_t ret;
 885
 886         if (offset) {
 887                 if (unlikely(get_user(off, offset)))
 888                         return -EFAULT;
 889                 pos = off;
 890                 ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
 891                 if (unlikely(put_user(pos, offset)))
 892                         return -EFAULT;
 893                 return ret;
 894         }
 895
 896         return do_sendfile(out_fd, in_fd, NULL, count, 0);
 897 }
 898
 899 SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd, loff_t __user *, offset, size_t, count)
 900 {
 901         loff_t pos;
 902         ssize_t ret;
 903
 904         if (offset) {
 905                 if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
 906                         return -EFAULT;
 907                 ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
 908                 if (unlikely(put_user(pos, offset)))
 909                         return -EFAULT;
 910                 return ret;
 911         }
 912
 913         return do_sendfile(out_fd, in_fd, NULL, count, 0);
 914 }