fs/read_write.c

   1 /*
   2  *  linux/fs/read_write.c
   3  *
   4  *  Copyright (C) 1991, 1992  Linus Torvalds
   5  */
   6
   7 #include <linux/slab.h>
   8 #include <linux/stat.h>
   9 #include <linux/fcntl.h>
  10 #include <linux/file.h>
  11 #include <linux/uio.h>
  12 #include <linux/smp_lock.h>
  13 #include <linux/fsnotify.h>
  14 #include <linux/security.h>
  15 #include <linux/module.h>
  16 #include <linux/syscalls.h>
  17 #include <linux/pagemap.h>
  18 #include <linux/splice.h>
  19 #include "read_write.h"
  20
  21 #include <asm/uaccess.h>
  22 #include <asm/unistd.h>
  23
  24 const struct file_operations generic_ro_fops = {
  25         .llseek         = generic_file_llseek,
  26         .read           = do_sync_read,
  27         .aio_read       = generic_file_aio_read,
  28         .mmap           = generic_file_readonly_mmap,
  29         .splice_read    = generic_file_splice_read,
  30 };
  31
  32 EXPORT_SYMBOL(generic_ro_fops);
  33
  34 /**
  35  * generic_file_llseek_unlocked - lockless generic llseek implementation
  36  * @file:       file structure to seek on
  37  * @offset:     file offset to seek to
  38  * @origin:     type of seek
  39  *
  40  * Updates the file offset to the value specified by @offset and @origin.
  41  * Locking must be provided by the caller.
  42  */
  43 loff_t
  44 generic_file_llseek_unlocked(struct file *file, loff_t offset, int origin)
  45 {
  46         struct inode *inode = file->f_mapping->host;
  47
  48         switch (origin) {
  49         case SEEK_END:
  50                 offset += inode->i_size;
  51                 break;
  52         case SEEK_CUR:
  53                 offset += file->f_pos;
  54                 break;
  55         }
  56
  57         if (offset < 0 || offset > inode->i_sb->s_maxbytes)
  58                 return -EINVAL;
  59
  60         /* Special lock needed here? */
  61         if (offset != file->f_pos) {
  62                 file->f_pos = offset;
  63                 file->f_version = 0;
  64         }
  65
  66         return offset;
  67 }
  68 EXPORT_SYMBOL(generic_file_llseek_unlocked);
  69
  70 /**
  71  * generic_file_llseek - generic llseek implementation for regular files
  72  * @file:       file structure to seek on
  73  * @offset:     file offset to seek to
  74  * @origin:     type of seek
  75  *
  76  * This is a generic implemenation of ->llseek useable for all normal local
  77  * filesystems.  It just updates the file offset to the value specified by
  78  * @offset and @origin under i_mutex.
  79  */
  80 loff_t generic_file_llseek(struct file *file, loff_t offset, int origin)
  81 {
  82         loff_t rval;
  83
  84         mutex_lock(&file->f_dentry->d_inode->i_mutex);
  85         rval = generic_file_llseek_unlocked(file, offset, origin);
  86         mutex_unlock(&file->f_dentry->d_inode->i_mutex);
  87
  88         return rval;
  89 }
  90 EXPORT_SYMBOL(generic_file_llseek);
  91
  92 loff_t no_llseek(struct file *file, loff_t offset, int origin)
  93 {
  94         return -ESPIPE;
  95 }
  96 EXPORT_SYMBOL(no_llseek);
  97
  98 loff_t default_llseek(struct file *file, loff_t offset, int origin)
  99 {
 100         loff_t retval;
 101
 102         lock_kernel();
 103         switch (origin) {
 104                 case SEEK_END:
 105                         offset += i_size_read(file->f_path.dentry->d_inode);
 106                         break;
 107                 case SEEK_CUR:
 108                         offset += file->f_pos;
 109         }
 110         retval = -EINVAL;
 111         if (offset >= 0) {
 112                 if (offset != file->f_pos) {
 113                         file->f_pos = offset;
 114                         file->f_version = 0;
 115                 }
 116                 retval = offset;
 117         }
 118         unlock_kernel();
 119         return retval;
 120 }
 121 EXPORT_SYMBOL(default_llseek);
 122
 123 loff_t vfs_llseek(struct file *file, loff_t offset, int origin)
 124 {
 125         loff_t (*fn)(struct file *, loff_t, int);
 126
 127         fn = no_llseek;
 128         if (file->f_mode & FMODE_LSEEK) {
 129                 fn = default_llseek;
 130                 if (file->f_op && file->f_op->llseek)
 131                         fn = file->f_op->llseek;
 132         }
 133         return fn(file, offset, origin);
 134 }
 135 EXPORT_SYMBOL(vfs_llseek);
 136
 137 SYSCALL_DEFINE3(lseek, unsigned int, fd, off_t, offset, unsigned int, origin)
 138 {
 139         off_t retval;
 140         struct file * file;
 141         int fput_needed;
 142
 143         retval = -EBADF;
 144         file = fget_light(fd, &fput_needed);
 145         if (!file)
 146                 goto bad;
 147
 148         retval = -EINVAL;
 149         if (origin <= SEEK_MAX) {
 150                 loff_t res = vfs_llseek(file, offset, origin);
 151                 retval = res;
 152                 if (res != (loff_t)retval)
 153                         retval = -EOVERFLOW;    /* LFS: should only happen on 32 bit platforms */
 154         }
 155         fput_light(file, fput_needed);
 156 bad:
 157         return retval;
 158 }
 159
 160 #ifdef __ARCH_WANT_SYS_LLSEEK
 161 SYSCALL_DEFINE5(llseek, unsigned int, fd, unsigned long, offset_high,
 162                 unsigned long, offset_low, loff_t __user *, result,
 163                 unsigned int, origin)
 164 {
 165         int retval;
 166         struct file * file;
 167         loff_t offset;
 168         int fput_needed;
 169
 170         retval = -EBADF;
 171         file = fget_light(fd, &fput_needed);
 172         if (!file)
 173                 goto bad;
 174
 175         retval = -EINVAL;
 176         if (origin > SEEK_MAX)
 177                 goto out_putf;
 178
 179         offset = vfs_llseek(file, ((loff_t) offset_high << 32) | offset_low,
 180                         origin);
 181
 182         retval = (int)offset;
 183         if (offset >= 0) {
 184                 retval = -EFAULT;
 185                 if (!copy_to_user(result, &offset, sizeof(offset)))
 186                         retval = 0;
 187         }
 188 out_putf:
 189         fput_light(file, fput_needed);
 190 bad:
 191         return retval;
 192 }
 193 #endif
 194
 195 /*
 196  * rw_verify_area doesn't like huge counts. We limit
 197  * them to something that fits in "int" so that others
 198  * won't have to do range checks all the time.
 199  */
 200 #define MAX_RW_COUNT (INT_MAX & PAGE_CACHE_MASK)
 201
 202 int rw_verify_area(int read_write, struct file *file, loff_t *ppos, size_t count)
 203 {
 204         struct inode *inode;
 205         loff_t pos;
 206         int retval = -EINVAL;
 207
 208         inode = file->f_path.dentry->d_inode;
 209         if (unlikely((ssize_t) count < 0))
 210                 return retval;
 211         pos = *ppos;
 212         if (unlikely((pos < 0) || (loff_t) (pos + count) < 0))
 213                 return retval;
 214
 215         if (unlikely(inode->i_flock && mandatory_lock(inode))) {
 216                 retval = locks_mandatory_area(
 217                         read_write == READ ? FLOCK_VERIFY_READ : FLOCK_VERIFY_WRITE,
 218                         inode, file, pos, count);
 219                 if (retval < 0)
 220                         return retval;
 221         }
 222         retval = security_file_permission(file,
 223                                 read_write == READ ? MAY_READ : MAY_WRITE);
 224         if (retval)
 225                 return retval;
 226         return count > MAX_RW_COUNT ? MAX_RW_COUNT : count;
 227 }
 228
 229 static void wait_on_retry_sync_kiocb(struct kiocb *iocb)
 230 {
 231         set_current_state(TASK_UNINTERRUPTIBLE);
 232         if (!kiocbIsKicked(iocb))
 233                 schedule();
 234         else
 235                 kiocbClearKicked(iocb);
 236         __set_current_state(TASK_RUNNING);
 237 }
 238
 239 ssize_t do_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos)
 240 {
 241         struct iovec iov = { .iov_base = buf, .iov_len = len };
 242         struct kiocb kiocb;
 243         ssize_t ret;
 244
 245         init_sync_kiocb(&kiocb, filp);
 246         kiocb.ki_pos = *ppos;
 247         kiocb.ki_left = len;
 248
 249         for (;;) {
 250                 ret = filp->f_op->aio_read(&kiocb, &iov, 1, kiocb.ki_pos);
 251                 if (ret != -EIOCBRETRY)
 252                         break;
 253                 wait_on_retry_sync_kiocb(&kiocb);
 254         }
 255
 256         if (-EIOCBQUEUED == ret)
 257                 ret = wait_on_sync_kiocb(&kiocb);
 258         *ppos = kiocb.ki_pos;
 259         return ret;
 260 }
 261
 262 EXPORT_SYMBOL(do_sync_read);
 263
 264 ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos)
 265 {
 266         ssize_t ret;
 267
 268         if (!(file->f_mode & FMODE_READ))
 269                 return -EBADF;
 270         if (!file->f_op || (!file->f_op->read && !file->f_op->aio_read))
 271                 return -EINVAL;
 272         if (unlikely(!access_ok(VERIFY_WRITE, buf, count)))
 273                 return -EFAULT;
 274
 275         ret = rw_verify_area(READ, file, pos, count);
 276         if (ret >= 0) {
 277                 count = ret;
 278                 if (file->f_op->read)
 279                         ret = file->f_op->read(file, buf, count, pos);
 280                 else
 281                         ret = do_sync_read(file, buf, count, pos);
 282                 if (ret > 0) {
 283                         fsnotify_access(file->f_path.dentry);
 284                         add_rchar(current, ret);
 285                 }
 286                 inc_syscr(current);
 287         }
 288
 289         return ret;
 290 }
 291
 292 EXPORT_SYMBOL(vfs_read);
 293
 294 ssize_t do_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos)
 295 {
 296         struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = len };
 297         struct kiocb kiocb;
 298         ssize_t ret;
 299
 300         init_sync_kiocb(&kiocb, filp);
 301         kiocb.ki_pos = *ppos;
 302         kiocb.ki_left = len;
 303
 304         for (;;) {
 305                 ret = filp->f_op->aio_write(&kiocb, &iov, 1, kiocb.ki_pos);
 306                 if (ret != -EIOCBRETRY)
 307                         break;
 308                 wait_on_retry_sync_kiocb(&kiocb);
 309         }
 310
 311         if (-EIOCBQUEUED == ret)
 312                 ret = wait_on_sync_kiocb(&kiocb);
 313         *ppos = kiocb.ki_pos;
 314         return ret;
 315 }
 316
 317 EXPORT_SYMBOL(do_sync_write);
 318
 319 ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_t *pos)
 320 {
 321         ssize_t ret;
 322
 323         if (!(file->f_mode & FMODE_WRITE))
 324                 return -EBADF;
 325         if (!file->f_op || (!file->f_op->write && !file->f_op->aio_write))
 326                 return -EINVAL;
 327         if (unlikely(!access_ok(VERIFY_READ, buf, count)))
 328                 return -EFAULT;
 329
 330         ret = rw_verify_area(WRITE, file, pos, count);
 331         if (ret >= 0) {
 332                 count = ret;
 333                 if (file->f_op->write)
 334                         ret = file->f_op->write(file, buf, count, pos);
 335                 else
 336                         ret = do_sync_write(file, buf, count, pos);
 337                 if (ret > 0) {
 338                         fsnotify_modify(file->f_path.dentry);
 339                         add_wchar(current, ret);
 340                 }
 341                 inc_syscw(current);
 342         }
 343
 344         return ret;
 345 }
 346
 347 EXPORT_SYMBOL(vfs_write);
 348
 349 static inline loff_t file_pos_read(struct file *file)
 350 {
 351         return file->f_pos;
 352 }
 353
 354 static inline void file_pos_write(struct file *file, loff_t pos)
 355 {
 356         file->f_pos = pos;
 357 }
 358
 359 SYSCALL_DEFINE3(read, unsigned int, fd, char __user *, buf, size_t, count)
 360 {
 361         struct file *file;
 362         ssize_t ret = -EBADF;
 363         int fput_needed;
 364
 365         file = fget_light(fd, &fput_needed);
 366         if (file) {
 367                 loff_t pos = file_pos_read(file);
 368                 ret = vfs_read(file, buf, count, &pos);
 369                 file_pos_write(file, pos);
 370                 fput_light(file, fput_needed);
 371         }
 372
 373         return ret;
 374 }
 375
 376 SYSCALL_DEFINE3(write, unsigned int, fd, const char __user *, buf,
 377                 size_t, count)
 378 {
 379         struct file *file;
 380         ssize_t ret = -EBADF;
 381         int fput_needed;
 382
 383         file = fget_light(fd, &fput_needed);
 384         if (file) {
 385                 loff_t pos = file_pos_read(file);
 386                 ret = vfs_write(file, buf, count, &pos);
 387                 file_pos_write(file, pos);
 388                 fput_light(file, fput_needed);
 389         }
 390
 391         return ret;
 392 }
 393
 394 SYSCALL_DEFINE(pread64)(unsigned int fd, char __user *buf,
 395                         size_t count, loff_t pos)
 396 {
 397         struct file *file;
 398         ssize_t ret = -EBADF;
 399         int fput_needed;
 400
 401         if (pos < 0)
 402                 return -EINVAL;
 403
 404         file = fget_light(fd, &fput_needed);
 405         if (file) {
 406                 ret = -ESPIPE;
 407                 if (file->f_mode & FMODE_PREAD)
 408                         ret = vfs_read(file, buf, count, &pos);
 409                 fput_light(file, fput_needed);
 410         }
 411
 412         return ret;
 413 }
 414 #ifdef CONFIG_HAVE_SYSCALL_WRAPPERS
 415 asmlinkage long SyS_pread64(long fd, long buf, long count, loff_t pos)
 416 {
 417         return SYSC_pread64((unsigned int) fd, (char __user *) buf,
 418                             (size_t) count, pos);
 419 }
 420 SYSCALL_ALIAS(sys_pread64, SyS_pread64);
 421 #endif
 422
 423 SYSCALL_DEFINE(pwrite64)(unsigned int fd, const char __user *buf,
 424                          size_t count, loff_t pos)
 425 {
 426         struct file *file;
 427         ssize_t ret = -EBADF;
 428         int fput_needed;
 429
 430         if (pos < 0)
 431                 return -EINVAL;
 432
 433         file = fget_light(fd, &fput_needed);
 434         if (file) {
 435                 ret = -ESPIPE;
 436                 if (file->f_mode & FMODE_PWRITE)
 437                         ret = vfs_write(file, buf, count, &pos);
 438                 fput_light(file, fput_needed);
 439         }
 440
 441         return ret;
 442 }
 443 #ifdef CONFIG_HAVE_SYSCALL_WRAPPERS
 444 asmlinkage long SyS_pwrite64(long fd, long buf, long count, loff_t pos)
 445 {
 446         return SYSC_pwrite64((unsigned int) fd, (const char __user *) buf,
 447                              (size_t) count, pos);
 448 }
 449 SYSCALL_ALIAS(sys_pwrite64, SyS_pwrite64);
 450 #endif
 451
 452 /*
 453  * Reduce an iovec's length in-place.  Return the resulting number of segments
 454  */
 455 unsigned long iov_shorten(struct iovec *iov, unsigned long nr_segs, size_t to)
 456 {
 457         unsigned long seg = 0;
 458         size_t len = 0;
 459
 460         while (seg < nr_segs) {
 461                 seg++;
 462                 if (len + iov->iov_len >= to) {
 463                         iov->iov_len = to - len;
 464                         break;
 465                 }
 466                 len += iov->iov_len;
 467                 iov++;
 468         }
 469         return seg;
 470 }
 471 EXPORT_SYMBOL(iov_shorten);
 472
 473 ssize_t do_sync_readv_writev(struct file *filp, const struct iovec *iov,
 474                 unsigned long nr_segs, size_t len, loff_t *ppos, iov_fn_t fn)
 475 {
 476         struct kiocb kiocb;
 477         ssize_t ret;
 478
 479         init_sync_kiocb(&kiocb, filp);
 480         kiocb.ki_pos = *ppos;
 481         kiocb.ki_left = len;
 482         kiocb.ki_nbytes = len;
 483
 484         for (;;) {
 485                 ret = fn(&kiocb, iov, nr_segs, kiocb.ki_pos);
 486                 if (ret != -EIOCBRETRY)
 487                         break;
 488                 wait_on_retry_sync_kiocb(&kiocb);
 489         }
 490
 491         if (ret == -EIOCBQUEUED)
 492                 ret = wait_on_sync_kiocb(&kiocb);
 493         *ppos = kiocb.ki_pos;
 494         return ret;
 495 }
 496
 497 /* Do it by hand, with file-ops */
 498 ssize_t do_loop_readv_writev(struct file *filp, struct iovec *iov,
 499                 unsigned long nr_segs, loff_t *ppos, io_fn_t fn)
 500 {
 501         struct iovec *vector = iov;
 502         ssize_t ret = 0;
 503
 504         while (nr_segs > 0) {
 505                 void __user *base;
 506                 size_t len;
 507                 ssize_t nr;
 508
 509                 base = vector->iov_base;
 510                 len = vector->iov_len;
 511                 vector++;
 512                 nr_segs--;
 513
 514                 nr = fn(filp, base, len, ppos);
 515
 516                 if (nr < 0) {
 517                         if (!ret)
 518                                 ret = nr;
 519                         break;
 520                 }
 521                 ret += nr;
 522                 if (nr != len)
 523                         break;
 524         }
 525
 526         return ret;
 527 }
 528
 529 /* A write operation does a read from user space and vice versa */
 530 #define vrfy_dir(type) ((type) == READ ? VERIFY_WRITE : VERIFY_READ)
 531
 532 ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector,
 533                               unsigned long nr_segs, unsigned long fast_segs,
 534                               struct iovec *fast_pointer,
 535                               struct iovec **ret_pointer)
 536   {
 537         unsigned long seg;
 538         ssize_t ret;
 539         struct iovec *iov = fast_pointer;
 540
 541         /*
 542          * SuS says "The readv() function *may* fail if the iovcnt argument
 543          * was less than or equal to 0, or greater than {IOV_MAX}.  Linux has
 544          * traditionally returned zero for zero segments, so...
 545          */
 546         if (nr_segs == 0) {
 547                 ret = 0;
 548                 goto out;
 549         }
 550
 551         /*
 552          * First get the "struct iovec" from user memory and
 553          * verify all the pointers
 554          */
 555         if (nr_segs > UIO_MAXIOV) {
 556                 ret = -EINVAL;
 557                 goto out;
 558         }
 559         if (nr_segs > fast_segs) {
 560                 iov = kmalloc(nr_segs*sizeof(struct iovec), GFP_KERNEL);
 561                 if (iov == NULL) {
 562                         ret = -ENOMEM;
 563                         goto out;
 564                 }
 565         }
 566         if (copy_from_user(iov, uvector, nr_segs*sizeof(*uvector))) {
 567                 ret = -EFAULT;
 568                 goto out;
 569         }
 570
 571         /*
 572          * According to the Single Unix Specification we should return EINVAL
 573          * if an element length is < 0 when cast to ssize_t or if the
 574          * total length would overflow the ssize_t return value of the
 575          * system call.
 576          */
 577         ret = 0;
 578         for (seg = 0; seg < nr_segs; seg++) {
 579                 void __user *buf = iov[seg].iov_base;
 580                 ssize_t len = (ssize_t)iov[seg].iov_len;
 581
 582                 /* see if we we're about to use an invalid len or if
 583                  * it's about to overflow ssize_t */
 584                 if (len < 0 || (ret + len < ret)) {
 585                         ret = -EINVAL;
 586                         goto out;
 587                 }
 588                 if (unlikely(!access_ok(vrfy_dir(type), buf, len))) {
 589                         ret = -EFAULT;
 590                         goto out;
 591                 }
 592
 593                 ret += len;
 594         }
 595 out:
 596         *ret_pointer = iov;
 597         return ret;
 598 }
 599
 600 static ssize_t do_readv_writev(int type, struct file *file,
 601                                const struct iovec __user * uvector,
 602                                unsigned long nr_segs, loff_t *pos)
 603 {
 604         size_t tot_len;
 605         struct iovec iovstack[UIO_FASTIOV];
 606         struct iovec *iov = iovstack;
 607         ssize_t ret;
 608         io_fn_t fn;
 609         iov_fn_t fnv;
 610
 611         if (!file->f_op) {
 612                 ret = -EINVAL;
 613                 goto out;
 614         }
 615
 616         ret = rw_copy_check_uvector(type, uvector, nr_segs,
 617                         ARRAY_SIZE(iovstack), iovstack, &iov);
 618         if (ret <= 0)
 619                 goto out;
 620
 621         tot_len = ret;
 622         ret = rw_verify_area(type, file, pos, tot_len);
 623         if (ret < 0)
 624                 goto out;
 625
 626         fnv = NULL;
 627         if (type == READ) {
 628                 fn = file->f_op->read;
 629                 fnv = file->f_op->aio_read;
 630         } else {
 631                 fn = (io_fn_t)file->f_op->write;
 632                 fnv = file->f_op->aio_write;
 633         }
 634
 635         if (fnv)
 636                 ret = do_sync_readv_writev(file, iov, nr_segs, tot_len,
 637                                                 pos, fnv);
 638         else
 639                 ret = do_loop_readv_writev(file, iov, nr_segs, pos, fn);
 640
 641 out:
 642         if (iov != iovstack)
 643                 kfree(iov);
 644         if ((ret + (type == READ)) > 0) {
 645                 if (type == READ)
 646                         fsnotify_access(file->f_path.dentry);
 647                 else
 648                         fsnotify_modify(file->f_path.dentry);
 649         }
 650         return ret;
 651 }
 652
 653 ssize_t vfs_readv(struct file *file, const struct iovec __user *vec,
 654                   unsigned long vlen, loff_t *pos)
 655 {
 656         if (!(file->f_mode & FMODE_READ))
 657                 return -EBADF;
 658         if (!file->f_op || (!file->f_op->aio_read && !file->f_op->read))
 659                 return -EINVAL;
 660
 661         return do_readv_writev(READ, file, vec, vlen, pos);
 662 }
 663
 664 EXPORT_SYMBOL(vfs_readv);
 665
 666 ssize_t vfs_writev(struct file *file, const struct iovec __user *vec,
 667                    unsigned long vlen, loff_t *pos)
 668 {
 669         if (!(file->f_mode & FMODE_WRITE))
 670                 return -EBADF;
 671         if (!file->f_op || (!file->f_op->aio_write && !file->f_op->write))
 672                 return -EINVAL;
 673
 674         return do_readv_writev(WRITE, file, vec, vlen, pos);
 675 }
 676
 677 EXPORT_SYMBOL(vfs_writev);
 678
 679 SYSCALL_DEFINE3(readv, unsigned long, fd, const struct iovec __user *, vec,
 680                 unsigned long, vlen)
 681 {
 682         struct file *file;
 683         ssize_t ret = -EBADF;
 684         int fput_needed;
 685
 686         file = fget_light(fd, &fput_needed);
 687         if (file) {
 688                 loff_t pos = file_pos_read(file);
 689                 ret = vfs_readv(file, vec, vlen, &pos);
 690                 file_pos_write(file, pos);
 691                 fput_light(file, fput_needed);
 692         }
 693
 694         if (ret > 0)
 695                 add_rchar(current, ret);
 696         inc_syscr(current);
 697         return ret;
 698 }
 699
 700 SYSCALL_DEFINE3(writev, unsigned long, fd, const struct iovec __user *, vec,
 701                 unsigned long, vlen)
 702 {
 703         struct file *file;
 704         ssize_t ret = -EBADF;
 705         int fput_needed;
 706
 707         file = fget_light(fd, &fput_needed);
 708         if (file) {
 709                 loff_t pos = file_pos_read(file);
 710                 ret = vfs_writev(file, vec, vlen, &pos);
 711                 file_pos_write(file, pos);
 712                 fput_light(file, fput_needed);
 713         }
 714
 715         if (ret > 0)
 716                 add_wchar(current, ret);
 717         inc_syscw(current);
 718         return ret;
 719 }
 720
 721 static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
 722                            size_t count, loff_t max)
 723 {
 724         struct file * in_file, * out_file;
 725         struct inode * in_inode, * out_inode;
 726         loff_t pos;
 727         ssize_t retval;
 728         int fput_needed_in, fput_needed_out, fl;
 729
 730         /*
 731          * Get input file, and verify that it is ok..
 732          */
 733         retval = -EBADF;
 734         in_file = fget_light(in_fd, &fput_needed_in);
 735         if (!in_file)
 736                 goto out;
 737         if (!(in_file->f_mode & FMODE_READ))
 738                 goto fput_in;
 739         retval = -EINVAL;
 740         in_inode = in_file->f_path.dentry->d_inode;
 741         if (!in_inode)
 742                 goto fput_in;
 743         if (!in_file->f_op || !in_file->f_op->splice_read)
 744                 goto fput_in;
 745         retval = -ESPIPE;
 746         if (!ppos)
 747                 ppos = &in_file->f_pos;
 748         else
 749                 if (!(in_file->f_mode & FMODE_PREAD))
 750                         goto fput_in;
 751         retval = rw_verify_area(READ, in_file, ppos, count);
 752         if (retval < 0)
 753                 goto fput_in;
 754         count = retval;
 755
 756         /*
 757          * Get output file, and verify that it is ok..
 758          */
 759         retval = -EBADF;
 760         out_file = fget_light(out_fd, &fput_needed_out);
 761         if (!out_file)
 762                 goto fput_in;
 763         if (!(out_file->f_mode & FMODE_WRITE))
 764                 goto fput_out;
 765         retval = -EINVAL;
 766         if (!out_file->f_op || !out_file->f_op->sendpage)
 767                 goto fput_out;
 768         out_inode = out_file->f_path.dentry->d_inode;
 769         retval = rw_verify_area(WRITE, out_file, &out_file->f_pos, count);
 770         if (retval < 0)
 771                 goto fput_out;
 772         count = retval;
 773
 774         if (!max)
 775                 max = min(in_inode->i_sb->s_maxbytes, out_inode->i_sb->s_maxbytes);
 776
 777         pos = *ppos;
 778         retval = -EINVAL;
 779         if (unlikely(pos < 0))
 780                 goto fput_out;
 781         if (unlikely(pos + count > max)) {
 782                 retval = -EOVERFLOW;
 783                 if (pos >= max)
 784                         goto fput_out;
 785                 count = max - pos;
 786         }
 787
 788         fl = 0;
 789 #if 0
 790         /*
 791          * We need to debate whether we can enable this or not. The
 792          * man page documents EAGAIN return for the output at least,
 793          * and the application is arguably buggy if it doesn't expect
 794          * EAGAIN on a non-blocking file descriptor.
 795          */
 796         if (in_file->f_flags & O_NONBLOCK)
 797                 fl = SPLICE_F_NONBLOCK;
 798 #endif
 799         retval = do_splice_direct(in_file, ppos, out_file, count, fl);
 800
 801         if (retval > 0) {
 802                 add_rchar(current, retval);
 803                 add_wchar(current, retval);
 804         }
 805
 806         inc_syscr(current);
 807         inc_syscw(current);
 808         if (*ppos > max)
 809                 retval = -EOVERFLOW;
 810
 811 fput_out:
 812         fput_light(out_file, fput_needed_out);
 813 fput_in:
 814         fput_light(in_file, fput_needed_in);
 815 out:
 816         return retval;
 817 }
 818
 819 SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd, off_t __user *, offset, size_t, count)
 820 {
 821         loff_t pos;
 822         off_t off;
 823         ssize_t ret;
 824
 825         if (offset) {
 826                 if (unlikely(get_user(off, offset)))
 827                         return -EFAULT;
 828                 pos = off;
 829                 ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
 830                 if (unlikely(put_user(pos, offset)))
 831                         return -EFAULT;
 832                 return ret;
 833         }
 834
 835         return do_sendfile(out_fd, in_fd, NULL, count, 0);
 836 }
 837
 838 SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd, loff_t __user *, offset, size_t, count)
 839 {
 840         loff_t pos;
 841         ssize_t ret;
 842
 843         if (offset) {
 844                 if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
 845                         return -EFAULT;
 846                 ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
 847                 if (unlikely(put_user(pos, offset)))
 848                         return -EFAULT;
 849                 return ret;
 850         }
 851
 852         return do_sendfile(out_fd, in_fd, NULL, count, 0);
 853 }