linux/fs/userfaultfd.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 *  fs/userfaultfd.c
   4 *
   5 *  Copyright (C) 2007  Davide Libenzi <davidel@xmailserver.org>
   6 *  Copyright (C) 2008-2009 Red Hat, Inc.
   7 *  Copyright (C) 2015  Red Hat, Inc.
   8 *
   9 *  Some part derived from fs/eventfd.c (anon inode setup) and
  10 *  mm/ksm.c (mm hashing).
  11 */
  12
  13#include <linux/list.h>
  14#include <linux/hashtable.h>
  15#include <linux/sched/signal.h>
  16#include <linux/sched/mm.h>
  17#include <linux/mm.h>
  18#include <linux/poll.h>
  19#include <linux/slab.h>
  20#include <linux/seq_file.h>
  21#include <linux/file.h>
  22#include <linux/bug.h>
  23#include <linux/anon_inodes.h>
  24#include <linux/syscalls.h>
  25#include <linux/userfaultfd_k.h>
  26#include <linux/mempolicy.h>
  27#include <linux/ioctl.h>
  28#include <linux/security.h>
  29#include <linux/hugetlb.h>
  30
  31int sysctl_unprivileged_userfaultfd __read_mostly = 1;
  32
  33static struct kmem_cache *userfaultfd_ctx_cachep __read_mostly;
  34
  35enum userfaultfd_state {
  36        UFFD_STATE_WAIT_API,
  37        UFFD_STATE_RUNNING,
  38};
  39
  40/*
  41 * Start with fault_pending_wqh and fault_wqh so they're more likely
  42 * to be in the same cacheline.
  43 *
  44 * Locking order:
  45 *      fd_wqh.lock
  46 *              fault_pending_wqh.lock
  47 *                      fault_wqh.lock
  48 *              event_wqh.lock
  49 *
  50 * To avoid deadlocks, IRQs must be disabled when taking any of the above locks,
  51 * since fd_wqh.lock is taken by aio_poll() while it's holding a lock that's
  52 * also taken in IRQ context.
  53 */
  54struct userfaultfd_ctx {
  55        /* waitqueue head for the pending (i.e. not read) userfaults */
  56        wait_queue_head_t fault_pending_wqh;
  57        /* waitqueue head for the userfaults */
  58        wait_queue_head_t fault_wqh;
  59        /* waitqueue head for the pseudo fd to wakeup poll/read */
  60        wait_queue_head_t fd_wqh;
  61        /* waitqueue head for events */
  62        wait_queue_head_t event_wqh;
  63        /* a refile sequence protected by fault_pending_wqh lock */
  64        struct seqcount refile_seq;
  65        /* pseudo fd refcounting */
  66        refcount_t refcount;
  67        /* userfaultfd syscall flags */
  68        unsigned int flags;
  69        /* features requested from the userspace */
  70        unsigned int features;
  71        /* state machine */
  72        enum userfaultfd_state state;
  73        /* released */
  74        bool released;
  75        /* memory mappings are changing because of non-cooperative event */
  76        bool mmap_changing;
  77        /* mm with one ore more vmas attached to this userfaultfd_ctx */
  78        struct mm_struct *mm;
  79};
  80
  81struct userfaultfd_fork_ctx {
  82        struct userfaultfd_ctx *orig;
  83        struct userfaultfd_ctx *new;
  84        struct list_head list;
  85};
  86
  87struct userfaultfd_unmap_ctx {
  88        struct userfaultfd_ctx *ctx;
  89        unsigned long start;
  90        unsigned long end;
  91        struct list_head list;
  92};
  93
  94struct userfaultfd_wait_queue {
  95        struct uffd_msg msg;
  96        wait_queue_entry_t wq;
  97        struct userfaultfd_ctx *ctx;
  98        bool waken;
  99};
 100
 101struct userfaultfd_wake_range {
 102        unsigned long start;
 103        unsigned long len;
 104};
 105
 106static int userfaultfd_wake_function(wait_queue_entry_t *wq, unsigned mode,
 107                                     int wake_flags, void *key)
 108{
 109        struct userfaultfd_wake_range *range = key;
 110        int ret;
 111        struct userfaultfd_wait_queue *uwq;
 112        unsigned long start, len;
 113
 114        uwq = container_of(wq, struct userfaultfd_wait_queue, wq);
 115        ret = 0;
 116        /* len == 0 means wake all */
 117        start = range->start;
 118        len = range->len;
 119        if (len && (start > uwq->msg.arg.pagefault.address ||
 120                    start + len <= uwq->msg.arg.pagefault.address))
 121                goto out;
 122        WRITE_ONCE(uwq->waken, true);
 123        /*
 124         * The Program-Order guarantees provided by the scheduler
 125         * ensure uwq->waken is visible before the task is woken.
 126         */
 127        ret = wake_up_state(wq->private, mode);
 128        if (ret) {
 129                /*
 130                 * Wake only once, autoremove behavior.
 131                 *
 132                 * After the effect of list_del_init is visible to the other
 133                 * CPUs, the waitqueue may disappear from under us, see the
 134                 * !list_empty_careful() in handle_userfault().
 135                 *
 136                 * try_to_wake_up() has an implicit smp_mb(), and the
 137                 * wq->private is read before calling the extern function
 138                 * "wake_up_state" (which in turns calls try_to_wake_up).
 139                 */
 140                list_del_init(&wq->entry);
 141        }
 142out:
 143        return ret;
 144}
 145
 146/**
 147 * userfaultfd_ctx_get - Acquires a reference to the internal userfaultfd
 148 * context.
 149 * @ctx: [in] Pointer to the userfaultfd context.
 150 */
 151static void userfaultfd_ctx_get(struct userfaultfd_ctx *ctx)
 152{
 153        refcount_inc(&ctx->refcount);
 154}
 155
 156/**
 157 * userfaultfd_ctx_put - Releases a reference to the internal userfaultfd
 158 * context.
 159 * @ctx: [in] Pointer to userfaultfd context.
 160 *
 161 * The userfaultfd context reference must have been previously acquired either
 162 * with userfaultfd_ctx_get() or userfaultfd_ctx_fdget().
 163 */
 164static void userfaultfd_ctx_put(struct userfaultfd_ctx *ctx)
 165{
 166        if (refcount_dec_and_test(&ctx->refcount)) {
 167                VM_BUG_ON(spin_is_locked(&ctx->fault_pending_wqh.lock));
 168                VM_BUG_ON(waitqueue_active(&ctx->fault_pending_wqh));
 169                VM_BUG_ON(spin_is_locked(&ctx->fault_wqh.lock));
 170                VM_BUG_ON(waitqueue_active(&ctx->fault_wqh));
 171                VM_BUG_ON(spin_is_locked(&ctx->event_wqh.lock));
 172                VM_BUG_ON(waitqueue_active(&ctx->event_wqh));
 173                VM_BUG_ON(spin_is_locked(&ctx->fd_wqh.lock));
 174                VM_BUG_ON(waitqueue_active(&ctx->fd_wqh));
 175                mmdrop(ctx->mm);
 176                kmem_cache_free(userfaultfd_ctx_cachep, ctx);
 177        }
 178}
 179
 180static inline void msg_init(struct uffd_msg *msg)
 181{
 182        BUILD_BUG_ON(sizeof(struct uffd_msg) != 32);
 183        /*
 184         * Must use memset to zero out the paddings or kernel data is
 185         * leaked to userland.
 186         */
 187        memset(msg, 0, sizeof(struct uffd_msg));
 188}
 189
 190static inline struct uffd_msg userfault_msg(unsigned long address,
 191                                            unsigned int flags,
 192                                            unsigned long reason,
 193                                            unsigned int features)
 194{
 195        struct uffd_msg msg;
 196        msg_init(&msg);
 197        msg.event = UFFD_EVENT_PAGEFAULT;
 198        msg.arg.pagefault.address = address;
 199        if (flags & FAULT_FLAG_WRITE)
 200                /*
 201                 * If UFFD_FEATURE_PAGEFAULT_FLAG_WP was set in the
 202                 * uffdio_api.features and UFFD_PAGEFAULT_FLAG_WRITE
 203                 * was not set in a UFFD_EVENT_PAGEFAULT, it means it
 204                 * was a read fault, otherwise if set it means it's
 205                 * a write fault.
 206                 */
 207                msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WRITE;
 208        if (reason & VM_UFFD_WP)
 209                /*
 210                 * If UFFD_FEATURE_PAGEFAULT_FLAG_WP was set in the
 211                 * uffdio_api.features and UFFD_PAGEFAULT_FLAG_WP was
 212                 * not set in a UFFD_EVENT_PAGEFAULT, it means it was
 213                 * a missing fault, otherwise if set it means it's a
 214                 * write protect fault.
 215                 */
 216                msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WP;
 217        if (features & UFFD_FEATURE_THREAD_ID)
 218                msg.arg.pagefault.feat.ptid = task_pid_vnr(current);
 219        return msg;
 220}
 221
 222#ifdef CONFIG_HUGETLB_PAGE
 223/*
 224 * Same functionality as userfaultfd_must_wait below with modifications for
 225 * hugepmd ranges.
 226 */
 227static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx,
 228                                         struct vm_area_struct *vma,
 229                                         unsigned long address,
 230                                         unsigned long flags,
 231                                         unsigned long reason)
 232{
 233        struct mm_struct *mm = ctx->mm;
 234        pte_t *ptep, pte;
 235        bool ret = true;
 236
 237        VM_BUG_ON(!rwsem_is_locked(&mm->mmap_sem));
 238
 239        ptep = huge_pte_offset(mm, address, vma_mmu_pagesize(vma));
 240
 241        if (!ptep)
 242                goto out;
 243
 244        ret = false;
 245        pte = huge_ptep_get(ptep);
 246
 247        /*
 248         * Lockless access: we're in a wait_event so it's ok if it
 249         * changes under us.
 250         */
 251        if (huge_pte_none(pte))
 252                ret = true;
 253        if (!huge_pte_write(pte) && (reason & VM_UFFD_WP))
 254                ret = true;
 255out:
 256        return ret;
 257}
 258#else
 259static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx,
 260                                         struct vm_area_struct *vma,
 261                                         unsigned long address,
 262                                         unsigned long flags,
 263                                         unsigned long reason)
 264{
 265        return false;   /* should never get here */
 266}
 267#endif /* CONFIG_HUGETLB_PAGE */
 268
 269/*
 270 * Verify the pagetables are still not ok after having reigstered into
 271 * the fault_pending_wqh to avoid userland having to UFFDIO_WAKE any
 272 * userfault that has already been resolved, if userfaultfd_read and
 273 * UFFDIO_COPY|ZEROPAGE are being run simultaneously on two different
 274 * threads.
 275 */
 276static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx,
 277                                         unsigned long address,
 278                                         unsigned long flags,
 279                                         unsigned long reason)
 280{
 281        struct mm_struct *mm = ctx->mm;
 282        pgd_t *pgd;
 283        p4d_t *p4d;
 284        pud_t *pud;
 285        pmd_t *pmd, _pmd;
 286        pte_t *pte;
 287        bool ret = true;
 288
 289        VM_BUG_ON(!rwsem_is_locked(&mm->mmap_sem));
 290
 291        pgd = pgd_offset(mm, address);
 292        if (!pgd_present(*pgd))
 293                goto out;
 294        p4d = p4d_offset(pgd, address);
 295        if (!p4d_present(*p4d))
 296                goto out;
 297        pud = pud_offset(p4d, address);
 298        if (!pud_present(*pud))
 299                goto out;
 300        pmd = pmd_offset(pud, address);
 301        /*
 302         * READ_ONCE must function as a barrier with narrower scope
 303         * and it must be equivalent to:
 304         *      _pmd = *pmd; barrier();
 305         *
 306         * This is to deal with the instability (as in
 307         * pmd_trans_unstable) of the pmd.
 308         */
 309        _pmd = READ_ONCE(*pmd);
 310        if (pmd_none(_pmd))
 311                goto out;
 312
 313        ret = false;
 314        if (!pmd_present(_pmd))
 315                goto out;
 316
 317        if (pmd_trans_huge(_pmd))
 318                goto out;
 319
 320        /*
 321         * the pmd is stable (as in !pmd_trans_unstable) so we can re-read it
 322         * and use the standard pte_offset_map() instead of parsing _pmd.
 323         */
 324        pte = pte_offset_map(pmd, address);
 325        /*
 326         * Lockless access: we're in a wait_event so it's ok if it
 327         * changes under us.
 328         */
 329        if (pte_none(*pte))
 330                ret = true;
 331        pte_unmap(pte);
 332
 333out:
 334        return ret;
 335}
 336
 337/*
 338 * The locking rules involved in returning VM_FAULT_RETRY depending on
 339 * FAULT_FLAG_ALLOW_RETRY, FAULT_FLAG_RETRY_NOWAIT and
 340 * FAULT_FLAG_KILLABLE are not straightforward. The "Caution"
 341 * recommendation in __lock_page_or_retry is not an understatement.
 342 *
 343 * If FAULT_FLAG_ALLOW_RETRY is set, the mmap_sem must be released
 344 * before returning VM_FAULT_RETRY only if FAULT_FLAG_RETRY_NOWAIT is
 345 * not set.
 346 *
 347 * If FAULT_FLAG_ALLOW_RETRY is set but FAULT_FLAG_KILLABLE is not
 348 * set, VM_FAULT_RETRY can still be returned if and only if there are
 349 * fatal_signal_pending()s, and the mmap_sem must be released before
 350 * returning it.
 351 */
 352vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason)
 353{
 354        struct mm_struct *mm = vmf->vma->vm_mm;
 355        struct userfaultfd_ctx *ctx;
 356        struct userfaultfd_wait_queue uwq;
 357        vm_fault_t ret = VM_FAULT_SIGBUS;
 358        bool must_wait, return_to_userland;
 359        long blocking_state;
 360
 361        /*
 362         * We don't do userfault handling for the final child pid update.
 363         *
 364         * We also don't do userfault handling during
 365         * coredumping. hugetlbfs has the special
 366         * follow_hugetlb_page() to skip missing pages in the
 367         * FOLL_DUMP case, anon memory also checks for FOLL_DUMP with
 368         * the no_page_table() helper in follow_page_mask(), but the
 369         * shmem_vm_ops->fault method is invoked even during
 370         * coredumping without mmap_sem and it ends up here.
 371         */
 372        if (current->flags & (PF_EXITING|PF_DUMPCORE))
 373                goto out;
 374
 375        /*
 376         * Coredumping runs without mmap_sem so we can only check that
 377         * the mmap_sem is held, if PF_DUMPCORE was not set.
 378         */
 379        WARN_ON_ONCE(!rwsem_is_locked(&mm->mmap_sem));
 380
 381        ctx = vmf->vma->vm_userfaultfd_ctx.ctx;
 382        if (!ctx)
 383                goto out;
 384
 385        BUG_ON(ctx->mm != mm);
 386
 387        VM_BUG_ON(reason & ~(VM_UFFD_MISSING|VM_UFFD_WP));
 388        VM_BUG_ON(!(reason & VM_UFFD_MISSING) ^ !!(reason & VM_UFFD_WP));
 389
 390        if (ctx->features & UFFD_FEATURE_SIGBUS)
 391                goto out;
 392
 393        /*
 394         * If it's already released don't get it. This avoids to loop
 395         * in __get_user_pages if userfaultfd_release waits on the
 396         * caller of handle_userfault to release the mmap_sem.
 397         */
 398        if (unlikely(READ_ONCE(ctx->released))) {
 399                /*
 400                 * Don't return VM_FAULT_SIGBUS in this case, so a non
 401                 * cooperative manager can close the uffd after the
 402                 * last UFFDIO_COPY, without risking to trigger an
 403                 * involuntary SIGBUS if the process was starting the
 404                 * userfaultfd while the userfaultfd was still armed
 405                 * (but after the last UFFDIO_COPY). If the uffd
 406                 * wasn't already closed when the userfault reached
 407                 * this point, that would normally be solved by
 408                 * userfaultfd_must_wait returning 'false'.
 409                 *
 410                 * If we were to return VM_FAULT_SIGBUS here, the non
 411                 * cooperative manager would be instead forced to
 412                 * always call UFFDIO_UNREGISTER before it can safely
 413                 * close the uffd.
 414                 */
 415                ret = VM_FAULT_NOPAGE;
 416                goto out;
 417        }
 418
 419        /*
 420         * Check that we can return VM_FAULT_RETRY.
 421         *
 422         * NOTE: it should become possible to return VM_FAULT_RETRY
 423         * even if FAULT_FLAG_TRIED is set without leading to gup()
 424         * -EBUSY failures, if the userfaultfd is to be extended for
 425         * VM_UFFD_WP tracking and we intend to arm the userfault
 426         * without first stopping userland access to the memory. For
 427         * VM_UFFD_MISSING userfaults this is enough for now.
 428         */
 429        if (unlikely(!(vmf->flags & FAULT_FLAG_ALLOW_RETRY))) {
 430                /*
 431                 * Validate the invariant that nowait must allow retry
 432                 * to be sure not to return SIGBUS erroneously on
 433                 * nowait invocations.
 434                 */
 435                BUG_ON(vmf->flags & FAULT_FLAG_RETRY_NOWAIT);
 436#ifdef CONFIG_DEBUG_VM
 437                if (printk_ratelimit()) {
 438                        printk(KERN_WARNING
 439                               "FAULT_FLAG_ALLOW_RETRY missing %x\n",
 440                               vmf->flags);
 441                        dump_stack();
 442                }
 443#endif
 444                goto out;
 445        }
 446
 447        /*
 448         * Handle nowait, not much to do other than tell it to retry
 449         * and wait.
 450         */
 451        ret = VM_FAULT_RETRY;
 452        if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT)
 453                goto out;
 454
 455        /* take the reference before dropping the mmap_sem */
 456        userfaultfd_ctx_get(ctx);
 457
 458        init_waitqueue_func_entry(&uwq.wq, userfaultfd_wake_function);
 459        uwq.wq.private = current;
 460        uwq.msg = userfault_msg(vmf->address, vmf->flags, reason,
 461                        ctx->features);
 462        uwq.ctx = ctx;
 463        uwq.waken = false;
 464
 465        return_to_userland =
 466                (vmf->flags & (FAULT_FLAG_USER|FAULT_FLAG_KILLABLE)) ==
 467                (FAULT_FLAG_USER|FAULT_FLAG_KILLABLE);
 468        blocking_state = return_to_userland ? TASK_INTERRUPTIBLE :
 469                         TASK_KILLABLE;
 470
 471        spin_lock_irq(&ctx->fault_pending_wqh.lock);
 472        /*
 473         * After the __add_wait_queue the uwq is visible to userland
 474         * through poll/read().
 475         */
 476        __add_wait_queue(&ctx->fault_pending_wqh, &uwq.wq);
 477        /*
 478         * The smp_mb() after __set_current_state prevents the reads
 479         * following the spin_unlock to happen before the list_add in
 480         * __add_wait_queue.
 481         */
 482        set_current_state(blocking_state);
 483        spin_unlock_irq(&ctx->fault_pending_wqh.lock);
 484
 485        if (!is_vm_hugetlb_page(vmf->vma))
 486                must_wait = userfaultfd_must_wait(ctx, vmf->address, vmf->flags,
 487                                                  reason);
 488        else
 489                must_wait = userfaultfd_huge_must_wait(ctx, vmf->vma,
 490                                                       vmf->address,
 491                                                       vmf->flags, reason);
 492        up_read(&mm->mmap_sem);
 493
 494        if (likely(must_wait && !READ_ONCE(ctx->released) &&
 495                   (return_to_userland ? !signal_pending(current) :
 496                    !fatal_signal_pending(current)))) {
 497                wake_up_poll(&ctx->fd_wqh, EPOLLIN);
 498                schedule();
 499                ret |= VM_FAULT_MAJOR;
 500
 501                /*
 502                 * False wakeups can orginate even from rwsem before
 503                 * up_read() however userfaults will wait either for a
 504                 * targeted wakeup on the specific uwq waitqueue from
 505                 * wake_userfault() or for signals or for uffd
 506                 * release.
 507                 */
 508                while (!READ_ONCE(uwq.waken)) {
 509                        /*
 510                         * This needs the full smp_store_mb()
 511                         * guarantee as the state write must be
 512                         * visible to other CPUs before reading
 513                         * uwq.waken from other CPUs.
 514                         */
 515                        set_current_state(blocking_state);
 516                        if (READ_ONCE(uwq.waken) ||
 517                            READ_ONCE(ctx->released) ||
 518                            (return_to_userland ? signal_pending(current) :
 519                             fatal_signal_pending(current)))
 520                                break;
 521                        schedule();
 522                }
 523        }
 524
 525        __set_current_state(TASK_RUNNING);
 526
 527        if (return_to_userland) {
 528                if (signal_pending(current) &&
 529                    !fatal_signal_pending(current)) {
 530                        /*
 531                         * If we got a SIGSTOP or SIGCONT and this is
 532                         * a normal userland page fault, just let
 533                         * userland return so the signal will be
 534                         * handled and gdb debugging works.  The page
 535                         * fault code immediately after we return from
 536                         * this function is going to release the
 537                         * mmap_sem and it's not depending on it
 538                         * (unlike gup would if we were not to return
 539                         * VM_FAULT_RETRY).
 540                         *
 541                         * If a fatal signal is pending we still take
 542                         * the streamlined VM_FAULT_RETRY failure path
 543                         * and there's no need to retake the mmap_sem
 544                         * in such case.
 545                         */
 546                        down_read(&mm->mmap_sem);
 547                        ret = VM_FAULT_NOPAGE;
 548                }
 549        }
 550
 551        /*
 552         * Here we race with the list_del; list_add in
 553         * userfaultfd_ctx_read(), however because we don't ever run
 554         * list_del_init() to refile across the two lists, the prev
 555         * and next pointers will never point to self. list_add also
 556         * would never let any of the two pointers to point to
 557         * self. So list_empty_careful won't risk to see both pointers
 558         * pointing to self at any time during the list refile. The
 559         * only case where list_del_init() is called is the full
 560         * removal in the wake function and there we don't re-list_add
 561         * and it's fine not to block on the spinlock. The uwq on this
 562         * kernel stack can be released after the list_del_init.
 563         */
 564        if (!list_empty_careful(&uwq.wq.entry)) {
 565                spin_lock_irq(&ctx->fault_pending_wqh.lock);
 566                /*
 567                 * No need of list_del_init(), the uwq on the stack
 568                 * will be freed shortly anyway.
 569                 */
 570                list_del(&uwq.wq.entry);
 571                spin_unlock_irq(&ctx->fault_pending_wqh.lock);
 572        }
 573
 574        /*
 575         * ctx may go away after this if the userfault pseudo fd is
 576         * already released.
 577         */
 578        userfaultfd_ctx_put(ctx);
 579
 580out:
 581        return ret;
 582}
 583
 584static void userfaultfd_event_wait_completion(struct userfaultfd_ctx *ctx,
 585                                              struct userfaultfd_wait_queue *ewq)
 586{
 587        struct userfaultfd_ctx *release_new_ctx;
 588
 589        if (WARN_ON_ONCE(current->flags & PF_EXITING))
 590                goto out;
 591
 592        ewq->ctx = ctx;
 593        init_waitqueue_entry(&ewq->wq, current);
 594        release_new_ctx = NULL;
 595
 596        spin_lock_irq(&ctx->event_wqh.lock);
 597        /*
 598         * After the __add_wait_queue the uwq is visible to userland
 599         * through poll/read().
 600         */
 601        __add_wait_queue(&ctx->event_wqh, &ewq->wq);
 602        for (;;) {
 603                set_current_state(TASK_KILLABLE);
 604                if (ewq->msg.event == 0)
 605                        break;
 606                if (READ_ONCE(ctx->released) ||
 607                    fatal_signal_pending(current)) {
 608                        /*
 609                         * &ewq->wq may be queued in fork_event, but
 610                         * __remove_wait_queue ignores the head
 611                         * parameter. It would be a problem if it
 612                         * didn't.
 613                         */
 614                        __remove_wait_queue(&ctx->event_wqh, &ewq->wq);
 615                        if (ewq->msg.event == UFFD_EVENT_FORK) {
 616                                struct userfaultfd_ctx *new;
 617
 618                                new = (struct userfaultfd_ctx *)
 619                                        (unsigned long)
 620                                        ewq->msg.arg.reserved.reserved1;
 621                                release_new_ctx = new;
 622                        }
 623                        break;
 624                }
 625
 626                spin_unlock_irq(&ctx->event_wqh.lock);
 627
 628                wake_up_poll(&ctx->fd_wqh, EPOLLIN);
 629                schedule();
 630
 631                spin_lock_irq(&ctx->event_wqh.lock);
 632        }
 633        __set_current_state(TASK_RUNNING);
 634        spin_unlock_irq(&ctx->event_wqh.lock);
 635
 636        if (release_new_ctx) {
 637                struct vm_area_struct *vma;
 638                struct mm_struct *mm = release_new_ctx->mm;
 639
 640                /* the various vma->vm_userfaultfd_ctx still points to it */
 641                down_write(&mm->mmap_sem);
 642                /* no task can run (and in turn coredump) yet */
 643                VM_WARN_ON(!mmget_still_valid(mm));
 644                for (vma = mm->mmap; vma; vma = vma->vm_next)
 645                        if (vma->vm_userfaultfd_ctx.ctx == release_new_ctx) {
 646                                vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
 647                                vma->vm_flags &= ~(VM_UFFD_WP | VM_UFFD_MISSING);
 648                        }
 649                up_write(&mm->mmap_sem);
 650
 651                userfaultfd_ctx_put(release_new_ctx);
 652        }
 653
 654        /*
 655         * ctx may go away after this if the userfault pseudo fd is
 656         * already released.
 657         */
 658out:
 659        WRITE_ONCE(ctx->mmap_changing, false);
 660        userfaultfd_ctx_put(ctx);
 661}
 662
 663static void userfaultfd_event_complete(struct userfaultfd_ctx *ctx,
 664                                       struct userfaultfd_wait_queue *ewq)
 665{
 666        ewq->msg.event = 0;
 667        wake_up_locked(&ctx->event_wqh);
 668        __remove_wait_queue(&ctx->event_wqh, &ewq->wq);
 669}
 670
 671int dup_userfaultfd(struct vm_area_struct *vma, struct list_head *fcs)
 672{
 673        struct userfaultfd_ctx *ctx = NULL, *octx;
 674        struct userfaultfd_fork_ctx *fctx;
 675
 676        octx = vma->vm_userfaultfd_ctx.ctx;
 677        if (!octx || !(octx->features & UFFD_FEATURE_EVENT_FORK)) {
 678                vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
 679                vma->vm_flags &= ~(VM_UFFD_WP | VM_UFFD_MISSING);
 680                return 0;
 681        }
 682
 683        list_for_each_entry(fctx, fcs, list)
 684                if (fctx->orig == octx) {
 685                        ctx = fctx->new;
 686                        break;
 687                }
 688
 689        if (!ctx) {
 690                fctx = kmalloc(sizeof(*fctx), GFP_KERNEL);
 691                if (!fctx)
 692                        return -ENOMEM;
 693
 694                ctx = kmem_cache_alloc(userfaultfd_ctx_cachep, GFP_KERNEL);
 695                if (!ctx) {
 696                        kfree(fctx);
 697                        return -ENOMEM;
 698                }
 699
 700                refcount_set(&ctx->refcount, 1);
 701                ctx->flags = octx->flags;
 702                ctx->state = UFFD_STATE_RUNNING;
 703                ctx->features = octx->features;
 704                ctx->released = false;
 705                ctx->mmap_changing = false;
 706                ctx->mm = vma->vm_mm;
 707                mmgrab(ctx->mm);
 708
 709                userfaultfd_ctx_get(octx);
 710                WRITE_ONCE(octx->mmap_changing, true);
 711                fctx->orig = octx;
 712                fctx->new = ctx;
 713                list_add_tail(&fctx->list, fcs);
 714        }
 715
 716        vma->vm_userfaultfd_ctx.ctx = ctx;
 717        return 0;
 718}
 719
 720static void dup_fctx(struct userfaultfd_fork_ctx *fctx)
 721{
 722        struct userfaultfd_ctx *ctx = fctx->orig;
 723        struct userfaultfd_wait_queue ewq;
 724
 725        msg_init(&ewq.msg);
 726
 727        ewq.msg.event = UFFD_EVENT_FORK;
 728        ewq.msg.arg.reserved.reserved1 = (unsigned long)fctx->new;
 729
 730        userfaultfd_event_wait_completion(ctx, &ewq);
 731}
 732
 733void dup_userfaultfd_complete(struct list_head *fcs)
 734{
 735        struct userfaultfd_fork_ctx *fctx, *n;
 736
 737        list_for_each_entry_safe(fctx, n, fcs, list) {
 738                dup_fctx(fctx);
 739                list_del(&fctx->list);
 740                kfree(fctx);
 741        }
 742}
 743
 744void mremap_userfaultfd_prep(struct vm_area_struct *vma,
 745                             struct vm_userfaultfd_ctx *vm_ctx)
 746{
 747        struct userfaultfd_ctx *ctx;
 748
 749        ctx = vma->vm_userfaultfd_ctx.ctx;
 750
 751        if (!ctx)
 752                return;
 753
 754        if (ctx->features & UFFD_FEATURE_EVENT_REMAP) {
 755                vm_ctx->ctx = ctx;
 756                userfaultfd_ctx_get(ctx);
 757                WRITE_ONCE(ctx->mmap_changing, true);
 758        } else {
 759                /* Drop uffd context if remap feature not enabled */
 760                vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
 761                vma->vm_flags &= ~(VM_UFFD_WP | VM_UFFD_MISSING);
 762        }
 763}
 764
 765void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *vm_ctx,
 766                                 unsigned long from, unsigned long to,
 767                                 unsigned long len)
 768{
 769        struct userfaultfd_ctx *ctx = vm_ctx->ctx;
 770        struct userfaultfd_wait_queue ewq;
 771
 772        if (!ctx)
 773                return;
 774
 775        if (to & ~PAGE_MASK) {
 776                userfaultfd_ctx_put(ctx);
 777                return;
 778        }
 779
 780        msg_init(&ewq.msg);
 781
 782        ewq.msg.event = UFFD_EVENT_REMAP;
 783        ewq.msg.arg.remap.from = from;
 784        ewq.msg.arg.remap.to = to;
 785        ewq.msg.arg.remap.len = len;
 786
 787        userfaultfd_event_wait_completion(ctx, &ewq);
 788}
 789
 790bool userfaultfd_remove(struct vm_area_struct *vma,
 791                        unsigned long start, unsigned long end)
 792{
 793        struct mm_struct *mm = vma->vm_mm;
 794        struct userfaultfd_ctx *ctx;
 795        struct userfaultfd_wait_queue ewq;
 796
 797        ctx = vma->vm_userfaultfd_ctx.ctx;
 798        if (!ctx || !(ctx->features & UFFD_FEATURE_EVENT_REMOVE))
 799                return true;
 800
 801        userfaultfd_ctx_get(ctx);
 802        WRITE_ONCE(ctx->mmap_changing, true);
 803        up_read(&mm->mmap_sem);
 804
 805        msg_init(&ewq.msg);
 806
 807        ewq.msg.event = UFFD_EVENT_REMOVE;
 808        ewq.msg.arg.remove.start = start;
 809        ewq.msg.arg.remove.end = end;
 810
 811        userfaultfd_event_wait_completion(ctx, &ewq);
 812
 813        return false;
 814}
 815
 816static bool has_unmap_ctx(struct userfaultfd_ctx *ctx, struct list_head *unmaps,
 817                          unsigned long start, unsigned long end)
 818{
 819        struct userfaultfd_unmap_ctx *unmap_ctx;
 820
 821        list_for_each_entry(unmap_ctx, unmaps, list)
 822                if (unmap_ctx->ctx == ctx && unmap_ctx->start == start &&
 823                    unmap_ctx->end == end)
 824                        return true;
 825
 826        return false;
 827}
 828
 829int userfaultfd_unmap_prep(struct vm_area_struct *vma,
 830                           unsigned long start, unsigned long end,
 831                           struct list_head *unmaps)
 832{
 833        for ( ; vma && vma->vm_start < end; vma = vma->vm_next) {
 834                struct userfaultfd_unmap_ctx *unmap_ctx;
 835                struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx;
 836
 837                if (!ctx || !(ctx->features & UFFD_FEATURE_EVENT_UNMAP) ||
 838                    has_unmap_ctx(ctx, unmaps, start, end))
 839                        continue;
 840
 841                unmap_ctx = kzalloc(sizeof(*unmap_ctx), GFP_KERNEL);
 842                if (!unmap_ctx)
 843                        return -ENOMEM;
 844
 845                userfaultfd_ctx_get(ctx);
 846                WRITE_ONCE(ctx->mmap_changing, true);
 847                unmap_ctx->ctx = ctx;
 848                unmap_ctx->start = start;
 849                unmap_ctx->end = end;
 850                list_add_tail(&unmap_ctx->list, unmaps);
 851        }
 852
 853        return 0;
 854}
 855
 856void userfaultfd_unmap_complete(struct mm_struct *mm, struct list_head *uf)
 857{
 858        struct userfaultfd_unmap_ctx *ctx, *n;
 859        struct userfaultfd_wait_queue ewq;
 860
 861        list_for_each_entry_safe(ctx, n, uf, list) {
 862                msg_init(&ewq.msg);
 863
 864                ewq.msg.event = UFFD_EVENT_UNMAP;
 865                ewq.msg.arg.remove.start = ctx->start;
 866                ewq.msg.arg.remove.end = ctx->end;
 867
 868                userfaultfd_event_wait_completion(ctx->ctx, &ewq);
 869
 870                list_del(&ctx->list);
 871                kfree(ctx);
 872        }
 873}
 874
 875static int userfaultfd_release(struct inode *inode, struct file *file)
 876{
 877        struct userfaultfd_ctx *ctx = file->private_data;
 878        struct mm_struct *mm = ctx->mm;
 879        struct vm_area_struct *vma, *prev;
 880        /* len == 0 means wake all */
 881        struct userfaultfd_wake_range range = { .len = 0, };
 882        unsigned long new_flags;
 883        bool still_valid;
 884
 885        WRITE_ONCE(ctx->released, true);
 886
 887        if (!mmget_not_zero(mm))
 888                goto wakeup;
 889
 890        /*
 891         * Flush page faults out of all CPUs. NOTE: all page faults
 892         * must be retried without returning VM_FAULT_SIGBUS if
 893         * userfaultfd_ctx_get() succeeds but vma->vma_userfault_ctx
 894         * changes while handle_userfault released the mmap_sem. So
 895         * it's critical that released is set to true (above), before
 896         * taking the mmap_sem for writing.
 897         */
 898        down_write(&mm->mmap_sem);
 899        still_valid = mmget_still_valid(mm);
 900        prev = NULL;
 901        for (vma = mm->mmap; vma; vma = vma->vm_next) {
 902                cond_resched();
 903                BUG_ON(!!vma->vm_userfaultfd_ctx.ctx ^
 904                       !!(vma->vm_flags & (VM_UFFD_MISSING | VM_UFFD_WP)));
 905                if (vma->vm_userfaultfd_ctx.ctx != ctx) {
 906                        prev = vma;
 907                        continue;
 908                }
 909                new_flags = vma->vm_flags & ~(VM_UFFD_MISSING | VM_UFFD_WP);
 910                if (still_valid) {
 911                        prev = vma_merge(mm, prev, vma->vm_start, vma->vm_end,
 912                                         new_flags, vma->anon_vma,
 913                                         vma->vm_file, vma->vm_pgoff,
 914                                         vma_policy(vma),
 915                                         NULL_VM_UFFD_CTX);
 916                        if (prev)
 917                                vma = prev;
 918                        else
 919                                prev = vma;
 920                }
 921                vma->vm_flags = new_flags;
 922                vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
 923        }
 924        up_write(&mm->mmap_sem);
 925        mmput(mm);
 926wakeup:
 927        /*
 928         * After no new page faults can wait on this fault_*wqh, flush
 929         * the last page faults that may have been already waiting on
 930         * the fault_*wqh.
 931         */
 932        spin_lock_irq(&ctx->fault_pending_wqh.lock);
 933        __wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL, &range);
 934        __wake_up(&ctx->fault_wqh, TASK_NORMAL, 1, &range);
 935        spin_unlock_irq(&ctx->fault_pending_wqh.lock);
 936
 937        /* Flush pending events that may still wait on event_wqh */
 938        wake_up_all(&ctx->event_wqh);
 939
 940        wake_up_poll(&ctx->fd_wqh, EPOLLHUP);
 941        userfaultfd_ctx_put(ctx);
 942        return 0;
 943}
 944
 945/* fault_pending_wqh.lock must be hold by the caller */
 946static inline struct userfaultfd_wait_queue *find_userfault_in(
 947                wait_queue_head_t *wqh)
 948{
 949        wait_queue_entry_t *wq;
 950        struct userfaultfd_wait_queue *uwq;
 951
 952        lockdep_assert_held(&wqh->lock);
 953
 954        uwq = NULL;
 955        if (!waitqueue_active(wqh))
 956                goto out;
 957        /* walk in reverse to provide FIFO behavior to read userfaults */
 958        wq = list_last_entry(&wqh->head, typeof(*wq), entry);
 959        uwq = container_of(wq, struct userfaultfd_wait_queue, wq);
 960out:
 961        return uwq;
 962}
 963
 964static inline struct userfaultfd_wait_queue *find_userfault(
 965                struct userfaultfd_ctx *ctx)
 966{
 967        return find_userfault_in(&ctx->fault_pending_wqh);
 968}
 969
 970static inline struct userfaultfd_wait_queue *find_userfault_evt(
 971                struct userfaultfd_ctx *ctx)
 972{
 973        return find_userfault_in(&ctx->event_wqh);
 974}
 975
 976static __poll_t userfaultfd_poll(struct file *file, poll_table *wait)
 977{
 978        struct userfaultfd_ctx *ctx = file->private_data;
 979        __poll_t ret;
 980
 981        poll_wait(file, &ctx->fd_wqh, wait);
 982
 983        switch (ctx->state) {
 984        case UFFD_STATE_WAIT_API:
 985                return EPOLLERR;
 986        case UFFD_STATE_RUNNING:
 987                /*
 988                 * poll() never guarantees that read won't block.
 989                 * userfaults can be waken before they're read().
 990                 */
 991                if (unlikely(!(file->f_flags & O_NONBLOCK)))
 992                        return EPOLLERR;
 993                /*
 994                 * lockless access to see if there are pending faults
 995                 * __pollwait last action is the add_wait_queue but
 996                 * the spin_unlock would allow the waitqueue_active to
 997                 * pass above the actual list_add inside
 998                 * add_wait_queue critical section. So use a full
 999                 * memory barrier to serialize the list_add write of
1000                 * add_wait_queue() with the waitqueue_active read
1001                 * below.
1002                 */
1003                ret = 0;
1004                smp_mb();
1005                if (waitqueue_active(&ctx->fault_pending_wqh))
1006                        ret = EPOLLIN;
1007                else if (waitqueue_active(&ctx->event_wqh))
1008                        ret = EPOLLIN;
1009
1010                return ret;
1011        default:
1012                WARN_ON_ONCE(1);
1013                return EPOLLERR;
1014        }
1015}
1016
1017static const struct file_operations userfaultfd_fops;
1018
1019static int resolve_userfault_fork(struct userfaultfd_ctx *ctx,
1020                                  struct userfaultfd_ctx *new,
1021                                  struct uffd_msg *msg)
1022{
1023        int fd;
1024
1025        fd = anon_inode_getfd("[userfaultfd]", &userfaultfd_fops, new,
1026                              O_RDWR | (new->flags & UFFD_SHARED_FCNTL_FLAGS));
1027        if (fd < 0)
1028                return fd;
1029
1030        msg->arg.reserved.reserved1 = 0;
1031        msg->arg.fork.ufd = fd;
1032        return 0;
1033}
1034
1035static ssize_t userfaultfd_ctx_read(struct userfaultfd_ctx *ctx, int no_wait,
1036                                    struct uffd_msg *msg)
1037{
1038        ssize_t ret;
1039        DECLARE_WAITQUEUE(wait, current);
1040        struct userfaultfd_wait_queue *uwq;
1041        /*
1042         * Handling fork event requires sleeping operations, so
1043         * we drop the event_wqh lock, then do these ops, then
1044         * lock it back and wake up the waiter. While the lock is
1045         * dropped the ewq may go away so we keep track of it
1046         * carefully.
1047         */
1048        LIST_HEAD(fork_event);
1049        struct userfaultfd_ctx *fork_nctx = NULL;
1050
1051        /* always take the fd_wqh lock before the fault_pending_wqh lock */
1052        spin_lock_irq(&ctx->fd_wqh.lock);
1053        __add_wait_queue(&ctx->fd_wqh, &wait);
1054        for (;;) {
1055                set_current_state(TASK_INTERRUPTIBLE);
1056                spin_lock(&ctx->fault_pending_wqh.lock);
1057                uwq = find_userfault(ctx);
1058                if (uwq) {
1059                        /*
1060                         * Use a seqcount to repeat the lockless check
1061                         * in wake_userfault() to avoid missing
1062                         * wakeups because during the refile both
1063                         * waitqueue could become empty if this is the
1064                         * only userfault.
1065                         */
1066                        write_seqcount_begin(&ctx->refile_seq);
1067
1068                        /*
1069                         * The fault_pending_wqh.lock prevents the uwq
1070                         * to disappear from under us.
1071                         *
1072                         * Refile this userfault from
1073                         * fault_pending_wqh to fault_wqh, it's not
1074                         * pending anymore after we read it.
1075                         *
1076                         * Use list_del() by hand (as
1077                         * userfaultfd_wake_function also uses
1078                         * list_del_init() by hand) to be sure nobody
1079                         * changes __remove_wait_queue() to use
1080                         * list_del_init() in turn breaking the
1081                         * !list_empty_careful() check in
1082                         * handle_userfault(). The uwq->wq.head list
1083                         * must never be empty at any time during the
1084                         * refile, or the waitqueue could disappear
1085                         * from under us. The "wait_queue_head_t"
1086                         * parameter of __remove_wait_queue() is unused
1087                         * anyway.
1088                         */
1089                        list_del(&uwq->wq.entry);
1090                        add_wait_queue(&ctx->fault_wqh, &uwq->wq);
1091
1092                        write_seqcount_end(&ctx->refile_seq);
1093
1094                        /* careful to always initialize msg if ret == 0 */
1095                        *msg = uwq->msg;
1096                        spin_unlock(&ctx->fault_pending_wqh.lock);
1097                        ret = 0;
1098                        break;
1099                }
1100                spin_unlock(&ctx->fault_pending_wqh.lock);
1101
1102                spin_lock(&ctx->event_wqh.lock);
1103                uwq = find_userfault_evt(ctx);
1104                if (uwq) {
1105                        *msg = uwq->msg;
1106
1107                        if (uwq->msg.event == UFFD_EVENT_FORK) {
1108                                fork_nctx = (struct userfaultfd_ctx *)
1109                                        (unsigned long)
1110                                        uwq->msg.arg.reserved.reserved1;
1111                                list_move(&uwq->wq.entry, &fork_event);
1112                                /*
1113                                 * fork_nctx can be freed as soon as
1114                                 * we drop the lock, unless we take a
1115                                 * reference on it.
1116                                 */
1117                                userfaultfd_ctx_get(fork_nctx);
1118                                spin_unlock(&ctx->event_wqh.lock);
1119                                ret = 0;
1120                                break;
1121                        }
1122
1123                        userfaultfd_event_complete(ctx, uwq);
1124                        spin_unlock(&ctx->event_wqh.lock);
1125                        ret = 0;
1126                        break;
1127                }
1128                spin_unlock(&ctx->event_wqh.lock);
1129
1130                if (signal_pending(current)) {
1131                        ret = -ERESTARTSYS;
1132                        break;
1133                }
1134                if (no_wait) {
1135                        ret = -EAGAIN;
1136                        break;
1137                }
1138                spin_unlock_irq(&ctx->fd_wqh.lock);
1139                schedule();
1140                spin_lock_irq(&ctx->fd_wqh.lock);
1141        }
1142        __remove_wait_queue(&ctx->fd_wqh, &wait);
1143        __set_current_state(TASK_RUNNING);
1144        spin_unlock_irq(&ctx->fd_wqh.lock);
1145
1146        if (!ret && msg->event == UFFD_EVENT_FORK) {
1147                ret = resolve_userfault_fork(ctx, fork_nctx, msg);
1148                spin_lock_irq(&ctx->event_wqh.lock);
1149                if (!list_empty(&fork_event)) {
1150                        /*
1151                         * The fork thread didn't abort, so we can
1152                         * drop the temporary refcount.
1153                         */
1154                        userfaultfd_ctx_put(fork_nctx);
1155
1156                        uwq = list_first_entry(&fork_event,
1157                                               typeof(*uwq),
1158                                               wq.entry);
1159                        /*
1160                         * If fork_event list wasn't empty and in turn
1161                         * the event wasn't already released by fork
1162                         * (the event is allocated on fork kernel
1163                         * stack), put the event back to its place in
1164                         * the event_wq. fork_event head will be freed
1165                         * as soon as we return so the event cannot
1166                         * stay queued there no matter the current
1167                         * "ret" value.
1168                         */
1169                        list_del(&uwq->wq.entry);
1170                        __add_wait_queue(&ctx->event_wqh, &uwq->wq);
1171
1172                        /*
1173                         * Leave the event in the waitqueue and report
1174                         * error to userland if we failed to resolve
1175                         * the userfault fork.
1176                         */
1177                        if (likely(!ret))
1178                                userfaultfd_event_complete(ctx, uwq);
1179                } else {
1180                        /*
1181                         * Here the fork thread aborted and the
1182                         * refcount from the fork thread on fork_nctx
1183                         * has already been released. We still hold
1184                         * the reference we took before releasing the
1185                         * lock above. If resolve_userfault_fork
1186                         * failed we've to drop it because the
1187                         * fork_nctx has to be freed in such case. If
1188                         * it succeeded we'll hold it because the new
1189                         * uffd references it.
1190                         */
1191                        if (ret)
1192                                userfaultfd_ctx_put(fork_nctx);
1193                }
1194                spin_unlock_irq(&ctx->event_wqh.lock);
1195        }
1196
1197        return ret;
1198}
1199
1200static ssize_t userfaultfd_read(struct file *file, char __user *buf,
1201                                size_t count, loff_t *ppos)
1202{
1203        struct userfaultfd_ctx *ctx = file->private_data;
1204        ssize_t _ret, ret = 0;
1205        struct uffd_msg msg;
1206        int no_wait = file->f_flags & O_NONBLOCK;
1207
1208        if (ctx->state == UFFD_STATE_WAIT_API)
1209                return -EINVAL;
1210
1211        for (;;) {
1212                if (count < sizeof(msg))
1213                        return ret ? ret : -EINVAL;
1214                _ret = userfaultfd_ctx_read(ctx, no_wait, &msg);
1215                if (_ret < 0)
1216                        return ret ? ret : _ret;
1217                if (copy_to_user((__u64 __user *) buf, &msg, sizeof(msg)))
1218                        return ret ? ret : -EFAULT;
1219                ret += sizeof(msg);
1220                buf += sizeof(msg);
1221                count -= sizeof(msg);
1222                /*
1223                 * Allow to read more than one fault at time but only
1224                 * block if waiting for the very first one.
1225                 */
1226                no_wait = O_NONBLOCK;
1227        }
1228}
1229
1230static void __wake_userfault(struct userfaultfd_ctx *ctx,
1231                             struct userfaultfd_wake_range *range)
1232{
1233        spin_lock_irq(&ctx->fault_pending_wqh.lock);
1234        /* wake all in the range and autoremove */
1235        if (waitqueue_active(&ctx->fault_pending_wqh))
1236                __wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL,
1237                                     range);
1238        if (waitqueue_active(&ctx->fault_wqh))
1239                __wake_up(&ctx->fault_wqh, TASK_NORMAL, 1, range);
1240        spin_unlock_irq(&ctx->fault_pending_wqh.lock);
1241}
1242
1243static __always_inline void wake_userfault(struct userfaultfd_ctx *ctx,
1244                                           struct userfaultfd_wake_range *range)
1245{
1246        unsigned seq;
1247        bool need_wakeup;
1248
1249        /*
1250         * To be sure waitqueue_active() is not reordered by the CPU
1251         * before the pagetable update, use an explicit SMP memory
1252         * barrier here. PT lock release or up_read(mmap_sem) still
1253         * have release semantics that can allow the
1254         * waitqueue_active() to be reordered before the pte update.
1255         */
1256        smp_mb();
1257
1258        /*
1259         * Use waitqueue_active because it's very frequent to
1260         * change the address space atomically even if there are no
1261         * userfaults yet. So we take the spinlock only when we're
1262         * sure we've userfaults to wake.
1263         */
1264        do {
1265                seq = read_seqcount_begin(&ctx->refile_seq);
1266                need_wakeup = waitqueue_active(&ctx->fault_pending_wqh) ||
1267                        waitqueue_active(&ctx->fault_wqh);
1268                cond_resched();
1269        } while (read_seqcount_retry(&ctx->refile_seq, seq));
1270        if (need_wakeup)
1271                __wake_userfault(ctx, range);
1272}
1273
1274static __always_inline int validate_range(struct mm_struct *mm,
1275                                          __u64 *start, __u64 len)
1276{
1277        __u64 task_size = mm->task_size;
1278
1279        *start = untagged_addr(*start);
1280
1281        if (*start & ~PAGE_MASK)
1282                return -EINVAL;
1283        if (len & ~PAGE_MASK)
1284                return -EINVAL;
1285        if (!len)
1286                return -EINVAL;
1287        if (*start < mmap_min_addr)
1288                return -EINVAL;
1289        if (*start >= task_size)
1290                return -EINVAL;
1291        if (len > task_size - *start)
1292                return -EINVAL;
1293        return 0;
1294}
1295
1296static inline bool vma_can_userfault(struct vm_area_struct *vma)
1297{
1298        return vma_is_anonymous(vma) || is_vm_hugetlb_page(vma) ||
1299                vma_is_shmem(vma);
1300}
1301
1302static int userfaultfd_register(struct userfaultfd_ctx *ctx,
1303                                unsigned long arg)
1304{
1305        struct mm_struct *mm = ctx->mm;
1306        struct vm_area_struct *vma, *prev, *cur;
1307        int ret;
1308        struct uffdio_register uffdio_register;
1309        struct uffdio_register __user *user_uffdio_register;
1310        unsigned long vm_flags, new_flags;
1311        bool found;
1312        bool basic_ioctls;
1313        unsigned long start, end, vma_end;
1314
1315        user_uffdio_register = (struct uffdio_register __user *) arg;
1316
1317        ret = -EFAULT;
1318        if (copy_from_user(&uffdio_register, user_uffdio_register,
1319                           sizeof(uffdio_register)-sizeof(__u64)))
1320                goto out;
1321
1322        ret = -EINVAL;
1323        if (!uffdio_register.mode)
1324                goto out;
1325        if (uffdio_register.mode & ~(UFFDIO_REGISTER_MODE_MISSING|
1326                                     UFFDIO_REGISTER_MODE_WP))
1327                goto out;
1328        vm_flags = 0;
1329        if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MISSING)
1330                vm_flags |= VM_UFFD_MISSING;
1331        if (uffdio_register.mode & UFFDIO_REGISTER_MODE_WP) {
1332                vm_flags |= VM_UFFD_WP;
1333                /*
1334                 * FIXME: remove the below error constraint by
1335                 * implementing the wprotect tracking mode.
1336                 */
1337                ret = -EINVAL;
1338                goto out;
1339        }
1340
1341        ret = validate_range(mm, &uffdio_register.range.start,
1342                             uffdio_register.range.len);
1343        if (ret)
1344                goto out;
1345
1346        start = uffdio_register.range.start;
1347        end = start + uffdio_register.range.len;
1348
1349        ret = -ENOMEM;
1350        if (!mmget_not_zero(mm))
1351                goto out;
1352
1353        down_write(&mm->mmap_sem);
1354        if (!mmget_still_valid(mm))
1355                goto out_unlock;
1356        vma = find_vma_prev(mm, start, &prev);
1357        if (!vma)
1358                goto out_unlock;
1359
1360        /* check that there's at least one vma in the range */
1361        ret = -EINVAL;
1362        if (vma->vm_start >= end)
1363                goto out_unlock;
1364
1365        /*
1366         * If the first vma contains huge pages, make sure start address
1367         * is aligned to huge page size.
1368         */
1369        if (is_vm_hugetlb_page(vma)) {
1370                unsigned long vma_hpagesize = vma_kernel_pagesize(vma);
1371
1372                if (start & (vma_hpagesize - 1))
1373                        goto out_unlock;
1374        }
1375
1376        /*
1377         * Search for not compatible vmas.
1378         */
1379        found = false;
1380        basic_ioctls = false;
1381        for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) {
1382                cond_resched();
1383
1384                BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^
1385                       !!(cur->vm_flags & (VM_UFFD_MISSING | VM_UFFD_WP)));
1386
1387                /* check not compatible vmas */
1388                ret = -EINVAL;
1389                if (!vma_can_userfault(cur))
1390                        goto out_unlock;
1391
1392                /*
1393                 * UFFDIO_COPY will fill file holes even without
1394                 * PROT_WRITE. This check enforces that if this is a
1395                 * MAP_SHARED, the process has write permission to the backing
1396                 * file. If VM_MAYWRITE is set it also enforces that on a
1397                 * MAP_SHARED vma: there is no F_WRITE_SEAL and no further
1398                 * F_WRITE_SEAL can be taken until the vma is destroyed.
1399                 */
1400                ret = -EPERM;
1401                if (unlikely(!(cur->vm_flags & VM_MAYWRITE)))
1402                        goto out_unlock;
1403
1404                /*
1405                 * If this vma contains ending address, and huge pages
1406                 * check alignment.
1407                 */
1408                if (is_vm_hugetlb_page(cur) && end <= cur->vm_end &&
1409                    end > cur->vm_start) {
1410                        unsigned long vma_hpagesize = vma_kernel_pagesize(cur);
1411
1412                        ret = -EINVAL;
1413
1414                        if (end & (vma_hpagesize - 1))
1415                                goto out_unlock;
1416                }
1417
1418                /*
1419                 * Check that this vma isn't already owned by a
1420                 * different userfaultfd. We can't allow more than one
1421                 * userfaultfd to own a single vma simultaneously or we
1422                 * wouldn't know which one to deliver the userfaults to.
1423                 */
1424                ret = -EBUSY;
1425                if (cur->vm_userfaultfd_ctx.ctx &&
1426                    cur->vm_userfaultfd_ctx.ctx != ctx)
1427                        goto out_unlock;
1428
1429                /*
1430                 * Note vmas containing huge pages
1431                 */
1432                if (is_vm_hugetlb_page(cur))
1433                        basic_ioctls = true;
1434
1435                found = true;
1436        }
1437        BUG_ON(!found);
1438
1439        if (vma->vm_start < start)
1440                prev = vma;
1441
1442        ret = 0;
1443        do {
1444                cond_resched();
1445
1446                BUG_ON(!vma_can_userfault(vma));
1447                BUG_ON(vma->vm_userfaultfd_ctx.ctx &&
1448                       vma->vm_userfaultfd_ctx.ctx != ctx);
1449                WARN_ON(!(vma->vm_flags & VM_MAYWRITE));
1450
1451                /*
1452                 * Nothing to do: this vma is already registered into this
1453                 * userfaultfd and with the right tracking mode too.
1454                 */
1455                if (vma->vm_userfaultfd_ctx.ctx == ctx &&
1456                    (vma->vm_flags & vm_flags) == vm_flags)
1457                        goto skip;
1458
1459                if (vma->vm_start > start)
1460                        start = vma->vm_start;
1461                vma_end = min(end, vma->vm_end);
1462
1463                new_flags = (vma->vm_flags & ~vm_flags) | vm_flags;
1464                prev = vma_merge(mm, prev, start, vma_end, new_flags,
1465                                 vma->anon_vma, vma->vm_file, vma->vm_pgoff,
1466                                 vma_policy(vma),
1467                                 ((struct vm_userfaultfd_ctx){ ctx }));
1468                if (prev) {
1469                        vma = prev;
1470                        goto next;
1471                }
1472                if (vma->vm_start < start) {
1473                        ret = split_vma(mm, vma, start, 1);
1474                        if (ret)
1475                                break;
1476                }
1477                if (vma->vm_end > end) {
1478                        ret = split_vma(mm, vma, end, 0);
1479                        if (ret)
1480                                break;
1481                }
1482        next:
1483                /*
1484                 * In the vma_merge() successful mprotect-like case 8:
1485                 * the next vma was merged into the current one and
1486                 * the current one has not been updated yet.
1487                 */
1488                vma->vm_flags = new_flags;
1489                vma->vm_userfaultfd_ctx.ctx = ctx;
1490
1491        skip:
1492                prev = vma;
1493                start = vma->vm_end;
1494                vma = vma->vm_next;
1495        } while (vma && vma->vm_start < end);
1496out_unlock:
1497        up_write(&mm->mmap_sem);
1498        mmput(mm);
1499        if (!ret) {
1500                /*
1501                 * Now that we scanned all vmas we can already tell
1502                 * userland which ioctls methods are guaranteed to
1503                 * succeed on this range.
1504                 */
1505                if (put_user(basic_ioctls ? UFFD_API_RANGE_IOCTLS_BASIC :
1506                             UFFD_API_RANGE_IOCTLS,
1507                             &user_uffdio_register->ioctls))
1508                        ret = -EFAULT;
1509        }
1510out:
1511        return ret;
1512}
1513
1514static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
1515                                  unsigned long arg)
1516{
1517        struct mm_struct *mm = ctx->mm;
1518        struct vm_area_struct *vma, *prev, *cur;
1519        int ret;
1520        struct uffdio_range uffdio_unregister;
1521        unsigned long new_flags;
1522        bool found;
1523        unsigned long start, end, vma_end;
1524        const void __user *buf = (void __user *)arg;
1525
1526        ret = -EFAULT;
1527        if (copy_from_user(&uffdio_unregister, buf, sizeof(uffdio_unregister)))
1528                goto out;
1529
1530        ret = validate_range(mm, &uffdio_unregister.start,
1531                             uffdio_unregister.len);
1532        if (ret)
1533                goto out;
1534
1535        start = uffdio_unregister.start;
1536        end = start + uffdio_unregister.len;
1537
1538        ret = -ENOMEM;
1539        if (!mmget_not_zero(mm))
1540                goto out;
1541
1542        down_write(&mm->mmap_sem);
1543        if (!mmget_still_valid(mm))
1544                goto out_unlock;
1545        vma = find_vma_prev(mm, start, &prev);
1546        if (!vma)
1547                goto out_unlock;
1548
1549        /* check that there's at least one vma in the range */
1550        ret = -EINVAL;
1551        if (vma->vm_start >= end)
1552                goto out_unlock;
1553
1554        /*
1555         * If the first vma contains huge pages, make sure start address
1556         * is aligned to huge page size.
1557         */
1558        if (is_vm_hugetlb_page(vma)) {
1559                unsigned long vma_hpagesize = vma_kernel_pagesize(vma);
1560
1561                if (start & (vma_hpagesize - 1))
1562                        goto out_unlock;
1563        }
1564
1565        /*
1566         * Search for not compatible vmas.
1567         */
1568        found = false;
1569        ret = -EINVAL;
1570        for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) {
1571                cond_resched();
1572
1573                BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^
1574                       !!(cur->vm_flags & (VM_UFFD_MISSING | VM_UFFD_WP)));
1575
1576                /*
1577                 * Check not compatible vmas, not strictly required
1578                 * here as not compatible vmas cannot have an
1579                 * userfaultfd_ctx registered on them, but this
1580                 * provides for more strict behavior to notice
1581                 * unregistration errors.
1582                 */
1583                if (!vma_can_userfault(cur))
1584                        goto out_unlock;
1585
1586                found = true;
1587        }
1588        BUG_ON(!found);
1589
1590        if (vma->vm_start < start)
1591                prev = vma;
1592
1593        ret = 0;
1594        do {
1595                cond_resched();
1596
1597                BUG_ON(!vma_can_userfault(vma));
1598
1599                /*
1600                 * Nothing to do: this vma is already registered into this
1601                 * userfaultfd and with the right tracking mode too.
1602                 */
1603                if (!vma->vm_userfaultfd_ctx.ctx)
1604                        goto skip;
1605
1606                WARN_ON(!(vma->vm_flags & VM_MAYWRITE));
1607
1608                if (vma->vm_start > start)
1609                        start = vma->vm_start;
1610                vma_end = min(end, vma->vm_end);
1611
1612                if (userfaultfd_missing(vma)) {
1613                        /*
1614                         * Wake any concurrent pending userfault while
1615                         * we unregister, so they will not hang
1616                         * permanently and it avoids userland to call
1617                         * UFFDIO_WAKE explicitly.
1618                         */
1619                        struct userfaultfd_wake_range range;
1620                        range.start = start;
1621                        range.len = vma_end - start;
1622                        wake_userfault(vma->vm_userfaultfd_ctx.ctx, &range);
1623                }
1624
1625                new_flags = vma->vm_flags & ~(VM_UFFD_MISSING | VM_UFFD_WP);
1626                prev = vma_merge(mm, prev, start, vma_end, new_flags,
1627                                 vma->anon_vma, vma->vm_file, vma->vm_pgoff,
1628                                 vma_policy(vma),
1629                                 NULL_VM_UFFD_CTX);
1630                if (prev) {
1631                        vma = prev;
1632                        goto next;
1633                }
1634                if (vma->vm_start < start) {
1635                        ret = split_vma(mm, vma, start, 1);
1636                        if (ret)
1637                                break;
1638                }
1639                if (vma->vm_end > end) {
1640                        ret = split_vma(mm, vma, end, 0);
1641                        if (ret)
1642                                break;
1643                }
1644        next:
1645                /*
1646                 * In the vma_merge() successful mprotect-like case 8:
1647                 * the next vma was merged into the current one and
1648                 * the current one has not been updated yet.
1649                 */
1650                vma->vm_flags = new_flags;
1651                vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
1652
1653        skip:
1654                prev = vma;
1655                start = vma->vm_end;
1656                vma = vma->vm_next;
1657        } while (vma && vma->vm_start < end);
1658out_unlock:
1659        up_write(&mm->mmap_sem);
1660        mmput(mm);
1661out:
1662        return ret;
1663}
1664
1665/*
1666 * userfaultfd_wake may be used in combination with the
1667 * UFFDIO_*_MODE_DONTWAKE to wakeup userfaults in batches.
1668 */
1669static int userfaultfd_wake(struct userfaultfd_ctx *ctx,
1670                            unsigned long arg)
1671{
1672        int ret;
1673        struct uffdio_range uffdio_wake;
1674        struct userfaultfd_wake_range range;
1675        const void __user *buf = (void __user *)arg;
1676
1677        ret = -EFAULT;
1678        if (copy_from_user(&uffdio_wake, buf, sizeof(uffdio_wake)))
1679                goto out;
1680
1681        ret = validate_range(ctx->mm, &uffdio_wake.start, uffdio_wake.len);
1682        if (ret)
1683                goto out;
1684
1685        range.start = uffdio_wake.start;
1686        range.len = uffdio_wake.len;
1687
1688        /*
1689         * len == 0 means wake all and we don't want to wake all here,
1690         * so check it again to be sure.
1691         */
1692        VM_BUG_ON(!range.len);
1693
1694        wake_userfault(ctx, &range);
1695        ret = 0;
1696
1697out:
1698        return ret;
1699}
1700
1701static int userfaultfd_copy(struct userfaultfd_ctx *ctx,
1702                            unsigned long arg)
1703{
1704        __s64 ret;
1705        struct uffdio_copy uffdio_copy;
1706        struct uffdio_copy __user *user_uffdio_copy;
1707        struct userfaultfd_wake_range range;
1708
1709        user_uffdio_copy = (struct uffdio_copy __user *) arg;
1710
1711        ret = -EAGAIN;
1712        if (READ_ONCE(ctx->mmap_changing))
1713                goto out;
1714
1715        ret = -EFAULT;
1716        if (copy_from_user(&uffdio_copy, user_uffdio_copy,
1717                           /* don't copy "copy" last field */
1718                           sizeof(uffdio_copy)-sizeof(__s64)))
1719                goto out;
1720
1721        ret = validate_range(ctx->mm, &uffdio_copy.dst, uffdio_copy.len);
1722        if (ret)
1723                goto out;
1724        /*
1725         * double check for wraparound just in case. copy_from_user()
1726         * will later check uffdio_copy.src + uffdio_copy.len to fit
1727         * in the userland range.
1728         */
1729        ret = -EINVAL;
1730        if (uffdio_copy.src + uffdio_copy.len <= uffdio_copy.src)
1731                goto out;
1732        if (uffdio_copy.mode & ~UFFDIO_COPY_MODE_DONTWAKE)
1733                goto out;
1734        if (mmget_not_zero(ctx->mm)) {
1735                ret = mcopy_atomic(ctx->mm, uffdio_copy.dst, uffdio_copy.src,
1736                                   uffdio_copy.len, &ctx->mmap_changing);
1737                mmput(ctx->mm);
1738        } else {
1739                return -ESRCH;
1740        }
1741        if (unlikely(put_user(ret, &user_uffdio_copy->copy)))
1742                return -EFAULT;
1743        if (ret < 0)
1744                goto out;
1745        BUG_ON(!ret);
1746        /* len == 0 would wake all */
1747        range.len = ret;
1748        if (!(uffdio_copy.mode & UFFDIO_COPY_MODE_DONTWAKE)) {
1749                range.start = uffdio_copy.dst;
1750                wake_userfault(ctx, &range);
1751        }
1752        ret = range.len == uffdio_copy.len ? 0 : -EAGAIN;
1753out:
1754        return ret;
1755}
1756
1757static int userfaultfd_zeropage(struct userfaultfd_ctx *ctx,
1758                                unsigned long arg)
1759{
1760        __s64 ret;
1761        struct uffdio_zeropage uffdio_zeropage;
1762        struct uffdio_zeropage __user *user_uffdio_zeropage;
1763        struct userfaultfd_wake_range range;
1764
1765        user_uffdio_zeropage = (struct uffdio_zeropage __user *) arg;
1766
1767        ret = -EAGAIN;
1768        if (READ_ONCE(ctx->mmap_changing))
1769                goto out;
1770
1771        ret = -EFAULT;
1772        if (copy_from_user(&uffdio_zeropage, user_uffdio_zeropage,
1773                           /* don't copy "zeropage" last field */
1774                           sizeof(uffdio_zeropage)-sizeof(__s64)))
1775                goto out;
1776
1777        ret = validate_range(ctx->mm, &uffdio_zeropage.range.start,
1778                             uffdio_zeropage.range.len);
1779        if (ret)
1780                goto out;
1781        ret = -EINVAL;
1782        if (uffdio_zeropage.mode & ~UFFDIO_ZEROPAGE_MODE_DONTWAKE)
1783                goto out;
1784
1785        if (mmget_not_zero(ctx->mm)) {
1786                ret = mfill_zeropage(ctx->mm, uffdio_zeropage.range.start,
1787                                     uffdio_zeropage.range.len,
1788                                     &ctx->mmap_changing);
1789                mmput(ctx->mm);
1790        } else {
1791                return -ESRCH;
1792        }
1793        if (unlikely(put_user(ret, &user_uffdio_zeropage->zeropage)))
1794                return -EFAULT;
1795        if (ret < 0)
1796                goto out;
1797        /* len == 0 would wake all */
1798        BUG_ON(!ret);
1799        range.len = ret;
1800        if (!(uffdio_zeropage.mode & UFFDIO_ZEROPAGE_MODE_DONTWAKE)) {
1801                range.start = uffdio_zeropage.range.start;
1802                wake_userfault(ctx, &range);
1803        }
1804        ret = range.len == uffdio_zeropage.range.len ? 0 : -EAGAIN;
1805out:
1806        return ret;
1807}
1808
1809static inline unsigned int uffd_ctx_features(__u64 user_features)
1810{
1811        /*
1812         * For the current set of features the bits just coincide
1813         */
1814        return (unsigned int)user_features;
1815}
1816
1817/*
1818 * userland asks for a certain API version and we return which bits
1819 * and ioctl commands are implemented in this kernel for such API
1820 * version or -EINVAL if unknown.
1821 */
1822static int userfaultfd_api(struct userfaultfd_ctx *ctx,
1823                           unsigned long arg)
1824{
1825        struct uffdio_api uffdio_api;
1826        void __user *buf = (void __user *)arg;
1827        int ret;
1828        __u64 features;
1829
1830        ret = -EINVAL;
1831        if (ctx->state != UFFD_STATE_WAIT_API)
1832                goto out;
1833        ret = -EFAULT;
1834        if (copy_from_user(&uffdio_api, buf, sizeof(uffdio_api)))
1835                goto out;
1836        features = uffdio_api.features;
1837        if (uffdio_api.api != UFFD_API || (features & ~UFFD_API_FEATURES)) {
1838                memset(&uffdio_api, 0, sizeof(uffdio_api));
1839                if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api)))
1840                        goto out;
1841                ret = -EINVAL;
1842                goto out;
1843        }
1844        /* report all available features and ioctls to userland */
1845        uffdio_api.features = UFFD_API_FEATURES;
1846        uffdio_api.ioctls = UFFD_API_IOCTLS;
1847        ret = -EFAULT;
1848        if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api)))
1849                goto out;
1850        ctx->state = UFFD_STATE_RUNNING;
1851        /* only enable the requested features for this uffd context */
1852        ctx->features = uffd_ctx_features(features);
1853        ret = 0;
1854out:
1855        return ret;
1856}
1857
1858static long userfaultfd_ioctl(struct file *file, unsigned cmd,
1859                              unsigned long arg)
1860{
1861        int ret = -EINVAL;
1862        struct userfaultfd_ctx *ctx = file->private_data;
1863
1864        if (cmd != UFFDIO_API && ctx->state == UFFD_STATE_WAIT_API)
1865                return -EINVAL;
1866
1867        switch(cmd) {
1868        case UFFDIO_API:
1869                ret = userfaultfd_api(ctx, arg);
1870                break;
1871        case UFFDIO_REGISTER:
1872                ret = userfaultfd_register(ctx, arg);
1873                break;
1874        case UFFDIO_UNREGISTER:
1875                ret = userfaultfd_unregister(ctx, arg);
1876                break;
1877        case UFFDIO_WAKE:
1878                ret = userfaultfd_wake(ctx, arg);
1879                break;
1880        case UFFDIO_COPY:
1881                ret = userfaultfd_copy(ctx, arg);
1882                break;
1883        case UFFDIO_ZEROPAGE:
1884                ret = userfaultfd_zeropage(ctx, arg);
1885                break;
1886        }
1887        return ret;
1888}
1889
1890#ifdef CONFIG_PROC_FS
1891static void userfaultfd_show_fdinfo(struct seq_file *m, struct file *f)
1892{
1893        struct userfaultfd_ctx *ctx = f->private_data;
1894        wait_queue_entry_t *wq;
1895        unsigned long pending = 0, total = 0;
1896
1897        spin_lock_irq(&ctx->fault_pending_wqh.lock);
1898        list_for_each_entry(wq, &ctx->fault_pending_wqh.head, entry) {
1899                pending++;
1900                total++;
1901        }
1902        list_for_each_entry(wq, &ctx->fault_wqh.head, entry) {
1903                total++;
1904        }
1905        spin_unlock_irq(&ctx->fault_pending_wqh.lock);
1906
1907        /*
1908         * If more protocols will be added, there will be all shown
1909         * separated by a space. Like this:
1910         *      protocols: aa:... bb:...
1911         */
1912        seq_printf(m, "pending:\t%lu\ntotal:\t%lu\nAPI:\t%Lx:%x:%Lx\n",
1913                   pending, total, UFFD_API, ctx->features,
1914                   UFFD_API_IOCTLS|UFFD_API_RANGE_IOCTLS);
1915}
1916#endif
1917
1918static const struct file_operations userfaultfd_fops = {
1919#ifdef CONFIG_PROC_FS
1920        .show_fdinfo    = userfaultfd_show_fdinfo,
1921#endif
1922        .release        = userfaultfd_release,
1923        .poll           = userfaultfd_poll,
1924        .read           = userfaultfd_read,
1925        .unlocked_ioctl = userfaultfd_ioctl,
1926        .compat_ioctl   = userfaultfd_ioctl,
1927        .llseek         = noop_llseek,
1928};
1929
1930static void init_once_userfaultfd_ctx(void *mem)
1931{
1932        struct userfaultfd_ctx *ctx = (struct userfaultfd_ctx *) mem;
1933
1934        init_waitqueue_head(&ctx->fault_pending_wqh);
1935        init_waitqueue_head(&ctx->fault_wqh);
1936        init_waitqueue_head(&ctx->event_wqh);
1937        init_waitqueue_head(&ctx->fd_wqh);
1938        seqcount_init(&ctx->refile_seq);
1939}
1940
1941SYSCALL_DEFINE1(userfaultfd, int, flags)
1942{
1943        struct userfaultfd_ctx *ctx;
1944        int fd;
1945
1946        if (!sysctl_unprivileged_userfaultfd && !capable(CAP_SYS_PTRACE))
1947                return -EPERM;
1948
1949        BUG_ON(!current->mm);
1950
1951        /* Check the UFFD_* constants for consistency.  */
1952        BUILD_BUG_ON(UFFD_CLOEXEC != O_CLOEXEC);
1953        BUILD_BUG_ON(UFFD_NONBLOCK != O_NONBLOCK);
1954
1955        if (flags & ~UFFD_SHARED_FCNTL_FLAGS)
1956                return -EINVAL;
1957
1958        ctx = kmem_cache_alloc(userfaultfd_ctx_cachep, GFP_KERNEL);
1959        if (!ctx)
1960                return -ENOMEM;
1961
1962        refcount_set(&ctx->refcount, 1);
1963        ctx->flags = flags;
1964        ctx->features = 0;
1965        ctx->state = UFFD_STATE_WAIT_API;
1966        ctx->released = false;
1967        ctx->mmap_changing = false;
1968        ctx->mm = current->mm;
1969        /* prevent the mm struct to be freed */
1970        mmgrab(ctx->mm);
1971
1972        fd = anon_inode_getfd("[userfaultfd]", &userfaultfd_fops, ctx,
1973                              O_RDWR | (flags & UFFD_SHARED_FCNTL_FLAGS));
1974        if (fd < 0) {
1975                mmdrop(ctx->mm);
1976                kmem_cache_free(userfaultfd_ctx_cachep, ctx);
1977        }
1978        return fd;
1979}
1980
1981static int __init userfaultfd_init(void)
1982{
1983        userfaultfd_ctx_cachep = kmem_cache_create("userfaultfd_ctx_cache",
1984                                                sizeof(struct userfaultfd_ctx),
1985                                                0,
1986                                                SLAB_HWCACHE_ALIGN|SLAB_PANIC,
1987                                                init_once_userfaultfd_ctx);
1988        return 0;
1989}
1990__initcall(userfaultfd_init);
1991