fs/eventfd.c

   1 /*
   2  *  fs/eventfd.c
   3  *
   4  *  Copyright (C) 2007  Davide Libenzi <davidel@xmailserver.org>
   5  *
   6  */
   7
   8 #include <linux/file.h>
   9 #include <linux/poll.h>
  10 #include <linux/init.h>
  11 #include <linux/fs.h>
  12 #include <linux/sched.h>
  13 #include <linux/kernel.h>
  14 #include <linux/slab.h>
  15 #include <linux/list.h>
  16 #include <linux/spinlock.h>
  17 #include <linux/anon_inodes.h>
  18 #include <linux/syscalls.h>
  19 #include <linux/export.h>
  20 #include <linux/kref.h>
  21 #include <linux/eventfd.h>
  22
  23 struct eventfd_ctx {
  24         struct kref kref;
  25         wait_queue_head_t wqh;
  26         /*
  27          * Every time that a write(2) is performed on an eventfd, the
  28          * value of the __u64 being written is added to "count" and a
  29          * wakeup is performed on "wqh". A read(2) will return the "count"
  30          * value to userspace, and will reset "count" to zero. The kernel
  31          * side eventfd_signal() also, adds to the "count" counter and
  32          * issue a wakeup.
  33          */
  34         __u64 count;
  35         unsigned int flags;
  36 };
  37
  38 /**
  39  * eventfd_signal - Adds @n to the eventfd counter.
  40  * @ctx: [in] Pointer to the eventfd context.
  41  * @n: [in] Value of the counter to be added to the eventfd internal counter.
  42  *          The value cannot be negative.
  43  *
  44  * This function is supposed to be called by the kernel in paths that do not
  45  * allow sleeping. In this function we allow the counter to reach the ULLONG_MAX
  46  * value, and we signal this as overflow condition by returining a POLLERR
  47  * to poll(2).
  48  *
  49  * Returns the amount by which the counter was incrememnted.  This will be less
  50  * than @n if the counter has overflowed.
  51  */
  52 __u64 eventfd_signal(struct eventfd_ctx *ctx, __u64 n)
  53 {
  54         unsigned long flags;
  55
  56         spin_lock_irqsave(&ctx->wqh.lock, flags);
  57         if (ULLONG_MAX - ctx->count < n)
  58                 n = ULLONG_MAX - ctx->count;
  59         ctx->count += n;
  60         if (waitqueue_active(&ctx->wqh))
  61                 wake_up_locked_poll(&ctx->wqh, POLLIN);
  62         spin_unlock_irqrestore(&ctx->wqh.lock, flags);
  63
  64         return n;
  65 }
  66 EXPORT_SYMBOL_GPL(eventfd_signal);
  67
  68 static void eventfd_free_ctx(struct eventfd_ctx *ctx)
  69 {
  70         kfree(ctx);
  71 }
  72
  73 static void eventfd_free(struct kref *kref)
  74 {
  75         struct eventfd_ctx *ctx = container_of(kref, struct eventfd_ctx, kref);
  76
  77         eventfd_free_ctx(ctx);
  78 }
  79
  80 /**
  81  * eventfd_ctx_get - Acquires a reference to the internal eventfd context.
  82  * @ctx: [in] Pointer to the eventfd context.
  83  *
  84  * Returns: In case of success, returns a pointer to the eventfd context.
  85  */
  86 struct eventfd_ctx *eventfd_ctx_get(struct eventfd_ctx *ctx)
  87 {
  88         kref_get(&ctx->kref);
  89         return ctx;
  90 }
  91 EXPORT_SYMBOL_GPL(eventfd_ctx_get);
  92
  93 /**
  94  * eventfd_ctx_put - Releases a reference to the internal eventfd context.
  95  * @ctx: [in] Pointer to eventfd context.
  96  *
  97  * The eventfd context reference must have been previously acquired either
  98  * with eventfd_ctx_get() or eventfd_ctx_fdget().
  99  */
 100 void eventfd_ctx_put(struct eventfd_ctx *ctx)
 101 {
 102         kref_put(&ctx->kref, eventfd_free);
 103 }
 104 EXPORT_SYMBOL_GPL(eventfd_ctx_put);
 105
 106 static int eventfd_release(struct inode *inode, struct file *file)
 107 {
 108         struct eventfd_ctx *ctx = file->private_data;
 109
 110         wake_up_poll(&ctx->wqh, POLLHUP);
 111         eventfd_ctx_put(ctx);
 112         return 0;
 113 }
 114
 115 static unsigned int eventfd_poll(struct file *file, poll_table *wait)
 116 {
 117         struct eventfd_ctx *ctx = file->private_data;
 118         unsigned int events = 0;
 119         unsigned long flags;
 120
 121         poll_wait(file, &ctx->wqh, wait);
 122
 123         spin_lock_irqsave(&ctx->wqh.lock, flags);
 124         if (ctx->count > 0)
 125                 events |= POLLIN;
 126         if (ctx->count == ULLONG_MAX)
 127                 events |= POLLERR;
 128         if (ULLONG_MAX - 1 > ctx->count)
 129                 events |= POLLOUT;
 130         spin_unlock_irqrestore(&ctx->wqh.lock, flags);
 131
 132         return events;
 133 }
 134
 135 static void eventfd_ctx_do_read(struct eventfd_ctx *ctx, __u64 *cnt)
 136 {
 137         *cnt = (ctx->flags & EFD_SEMAPHORE) ? 1 : ctx->count;
 138         ctx->count -= *cnt;
 139 }
 140
 141 /**
 142  * eventfd_ctx_remove_wait_queue - Read the current counter and removes wait queue.
 143  * @ctx: [in] Pointer to eventfd context.
 144  * @wait: [in] Wait queue to be removed.
 145  * @cnt: [out] Pointer to the 64-bit counter value.
 146  *
 147  * Returns %0 if successful, or the following error codes:
 148  *
 149  * -EAGAIN      : The operation would have blocked.
 150  *
 151  * This is used to atomically remove a wait queue entry from the eventfd wait
 152  * queue head, and read/reset the counter value.
 153  */
 154 int eventfd_ctx_remove_wait_queue(struct eventfd_ctx *ctx, wait_queue_t *wait,
 155                                   __u64 *cnt)
 156 {
 157         unsigned long flags;
 158
 159         spin_lock_irqsave(&ctx->wqh.lock, flags);
 160         eventfd_ctx_do_read(ctx, cnt);
 161         __remove_wait_queue(&ctx->wqh, wait);
 162         if (*cnt != 0 && waitqueue_active(&ctx->wqh))
 163                 wake_up_locked_poll(&ctx->wqh, POLLOUT);
 164         spin_unlock_irqrestore(&ctx->wqh.lock, flags);
 165
 166         return *cnt != 0 ? 0 : -EAGAIN;
 167 }
 168 EXPORT_SYMBOL_GPL(eventfd_ctx_remove_wait_queue);
 169
 170 /**
 171  * eventfd_ctx_read - Reads the eventfd counter or wait if it is zero.
 172  * @ctx: [in] Pointer to eventfd context.
 173  * @no_wait: [in] Different from zero if the operation should not block.
 174  * @cnt: [out] Pointer to the 64-bit counter value.
 175  *
 176  * Returns %0 if successful, or the following error codes:
 177  *
 178  * -EAGAIN      : The operation would have blocked but @no_wait was non-zero.
 179  * -ERESTARTSYS : A signal interrupted the wait operation.
 180  *
 181  * If @no_wait is zero, the function might sleep until the eventfd internal
 182  * counter becomes greater than zero.
 183  */
 184 ssize_t eventfd_ctx_read(struct eventfd_ctx *ctx, int no_wait, __u64 *cnt)
 185 {
 186         ssize_t res;
 187         DECLARE_WAITQUEUE(wait, current);
 188
 189         spin_lock_irq(&ctx->wqh.lock);
 190         *cnt = 0;
 191         res = -EAGAIN;
 192         if (ctx->count > 0)
 193                 res = 0;
 194         else if (!no_wait) {
 195                 __add_wait_queue(&ctx->wqh, &wait);
 196                 for (;;) {
 197                         set_current_state(TASK_INTERRUPTIBLE);
 198                         if (ctx->count > 0) {
 199                                 res = 0;
 200                                 break;
 201                         }
 202                         if (signal_pending(current)) {
 203                                 res = -ERESTARTSYS;
 204                                 break;
 205                         }
 206                         spin_unlock_irq(&ctx->wqh.lock);
 207                         schedule();
 208                         spin_lock_irq(&ctx->wqh.lock);
 209                 }
 210                 __remove_wait_queue(&ctx->wqh, &wait);
 211                 __set_current_state(TASK_RUNNING);
 212         }
 213         if (likely(res == 0)) {
 214                 eventfd_ctx_do_read(ctx, cnt);
 215                 if (waitqueue_active(&ctx->wqh))
 216                         wake_up_locked_poll(&ctx->wqh, POLLOUT);
 217         }
 218         spin_unlock_irq(&ctx->wqh.lock);
 219
 220         return res;
 221 }
 222 EXPORT_SYMBOL_GPL(eventfd_ctx_read);
 223
 224 static ssize_t eventfd_read(struct file *file, char __user *buf, size_t count,
 225                             loff_t *ppos)
 226 {
 227         struct eventfd_ctx *ctx = file->private_data;
 228         ssize_t res;
 229         __u64 cnt;
 230
 231         if (count < sizeof(cnt))
 232                 return -EINVAL;
 233         res = eventfd_ctx_read(ctx, file->f_flags & O_NONBLOCK, &cnt);
 234         if (res < 0)
 235                 return res;
 236
 237         return put_user(cnt, (__u64 __user *) buf) ? -EFAULT : sizeof(cnt);
 238 }
 239
 240 static ssize_t eventfd_write(struct file *file, const char __user *buf, size_t count,
 241                              loff_t *ppos)
 242 {
 243         struct eventfd_ctx *ctx = file->private_data;
 244         ssize_t res;
 245         __u64 ucnt;
 246         DECLARE_WAITQUEUE(wait, current);
 247
 248         if (count < sizeof(ucnt))
 249                 return -EINVAL;
 250         if (copy_from_user(&ucnt, buf, sizeof(ucnt)))
 251                 return -EFAULT;
 252         if (ucnt == ULLONG_MAX)
 253                 return -EINVAL;
 254         spin_lock_irq(&ctx->wqh.lock);
 255         res = -EAGAIN;
 256         if (ULLONG_MAX - ctx->count > ucnt)
 257                 res = sizeof(ucnt);
 258         else if (!(file->f_flags & O_NONBLOCK)) {
 259                 __add_wait_queue(&ctx->wqh, &wait);
 260                 for (res = 0;;) {
 261                         set_current_state(TASK_INTERRUPTIBLE);
 262                         if (ULLONG_MAX - ctx->count > ucnt) {
 263                                 res = sizeof(ucnt);
 264                                 break;
 265                         }
 266                         if (signal_pending(current)) {
 267                                 res = -ERESTARTSYS;
 268                                 break;
 269                         }
 270                         spin_unlock_irq(&ctx->wqh.lock);
 271                         schedule();
 272                         spin_lock_irq(&ctx->wqh.lock);
 273                 }
 274                 __remove_wait_queue(&ctx->wqh, &wait);
 275                 __set_current_state(TASK_RUNNING);
 276         }
 277         if (likely(res > 0)) {
 278                 ctx->count += ucnt;
 279                 if (waitqueue_active(&ctx->wqh))
 280                         wake_up_locked_poll(&ctx->wqh, POLLIN);
 281         }
 282         spin_unlock_irq(&ctx->wqh.lock);
 283
 284         return res;
 285 }
 286
 287 static const struct file_operations eventfd_fops = {
 288         .release        = eventfd_release,
 289         .poll           = eventfd_poll,
 290         .read           = eventfd_read,
 291         .write          = eventfd_write,
 292         .llseek         = noop_llseek,
 293 };
 294
 295 /**
 296  * eventfd_fget - Acquire a reference of an eventfd file descriptor.
 297  * @fd: [in] Eventfd file descriptor.
 298  *
 299  * Returns a pointer to the eventfd file structure in case of success, or the
 300  * following error pointer:
 301  *
 302  * -EBADF    : Invalid @fd file descriptor.
 303  * -EINVAL   : The @fd file descriptor is not an eventfd file.
 304  */
 305 struct file *eventfd_fget(int fd)
 306 {
 307         struct file *file;
 308
 309         file = fget(fd);
 310         if (!file)
 311                 return ERR_PTR(-EBADF);
 312         if (file->f_op != &eventfd_fops) {
 313                 fput(file);
 314                 return ERR_PTR(-EINVAL);
 315         }
 316
 317         return file;
 318 }
 319 EXPORT_SYMBOL_GPL(eventfd_fget);
 320
 321 /**
 322  * eventfd_ctx_fdget - Acquires a reference to the internal eventfd context.
 323  * @fd: [in] Eventfd file descriptor.
 324  *
 325  * Returns a pointer to the internal eventfd context, otherwise the error
 326  * pointers returned by the following functions:
 327  *
 328  * eventfd_fget
 329  */
 330 struct eventfd_ctx *eventfd_ctx_fdget(int fd)
 331 {
 332         struct file *file;
 333         struct eventfd_ctx *ctx;
 334
 335         file = eventfd_fget(fd);
 336         if (IS_ERR(file))
 337                 return (struct eventfd_ctx *) file;
 338         ctx = eventfd_ctx_get(file->private_data);
 339         fput(file);
 340
 341         return ctx;
 342 }
 343 EXPORT_SYMBOL_GPL(eventfd_ctx_fdget);
 344
 345 /**
 346  * eventfd_ctx_fileget - Acquires a reference to the internal eventfd context.
 347  * @file: [in] Eventfd file pointer.
 348  *
 349  * Returns a pointer to the internal eventfd context, otherwise the error
 350  * pointer:
 351  *
 352  * -EINVAL   : The @fd file descriptor is not an eventfd file.
 353  */
 354 struct eventfd_ctx *eventfd_ctx_fileget(struct file *file)
 355 {
 356         if (file->f_op != &eventfd_fops)
 357                 return ERR_PTR(-EINVAL);
 358
 359         return eventfd_ctx_get(file->private_data);
 360 }
 361 EXPORT_SYMBOL_GPL(eventfd_ctx_fileget);
 362
 363 /**
 364  * eventfd_file_create - Creates an eventfd file pointer.
 365  * @count: Initial eventfd counter value.
 366  * @flags: Flags for the eventfd file.
 367  *
 368  * This function creates an eventfd file pointer, w/out installing it into
 369  * the fd table. This is useful when the eventfd file is used during the
 370  * initialization of data structures that require extra setup after the eventfd
 371  * creation. So the eventfd creation is split into the file pointer creation
 372  * phase, and the file descriptor installation phase.
 373  * In this way races with userspace closing the newly installed file descriptor
 374  * can be avoided.
 375  * Returns an eventfd file pointer, or a proper error pointer.
 376  */
 377 struct file *eventfd_file_create(unsigned int count, int flags)
 378 {
 379         struct file *file;
 380         struct eventfd_ctx *ctx;
 381
 382         /* Check the EFD_* constants for consistency.  */
 383         BUILD_BUG_ON(EFD_CLOEXEC != O_CLOEXEC);
 384         BUILD_BUG_ON(EFD_NONBLOCK != O_NONBLOCK);
 385
 386         if (flags & ~EFD_FLAGS_SET)
 387                 return ERR_PTR(-EINVAL);
 388
 389         ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
 390         if (!ctx)
 391                 return ERR_PTR(-ENOMEM);
 392
 393         kref_init(&ctx->kref);
 394         init_waitqueue_head(&ctx->wqh);
 395         ctx->count = count;
 396         ctx->flags = flags;
 397
 398         file = anon_inode_getfile("[eventfd]", &eventfd_fops, ctx,
 399                                   O_RDWR | (flags & EFD_SHARED_FCNTL_FLAGS));
 400         if (IS_ERR(file))
 401                 eventfd_free_ctx(ctx);
 402
 403         return file;
 404 }
 405
 406 SYSCALL_DEFINE2(eventfd2, unsigned int, count, int, flags)
 407 {
 408         int fd, error;
 409         struct file *file;
 410
 411         error = get_unused_fd_flags(flags & EFD_SHARED_FCNTL_FLAGS);
 412         if (error < 0)
 413                 return error;
 414         fd = error;
 415
 416         file = eventfd_file_create(count, flags);
 417         if (IS_ERR(file)) {
 418                 error = PTR_ERR(file);
 419                 goto err_put_unused_fd;
 420         }
 421         fd_install(fd, file);
 422
 423         return fd;
 424
 425 err_put_unused_fd:
 426         put_unused_fd(fd);
 427
 428         return error;
 429 }
 430
 431 SYSCALL_DEFINE1(eventfd, unsigned int, count)
 432 {
 433         return sys_eventfd2(count, 0);
 434 }
 435