LXR linux/arch/s390/mm/gup.c

   1/*
   2 *  Lockless get_user_pages_fast for s390
   3 *
   4 *  Copyright IBM Corp. 2010
   5 *  Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
   6 */
   7#include <linux/sched.h>
   8#include <linux/mm.h>
   9#include <linux/hugetlb.h>
  10#include <linux/vmstat.h>
  11#include <linux/pagemap.h>
  12#include <linux/rwsem.h>
  13#include <asm/pgtable.h>
  14
  15/*
  16 * The performance critical leaf functions are made noinline otherwise gcc
  17 * inlines everything into a single function which results in too much
  18 * register pressure.
  19 */
  20static inline int gup_pte_range(pmd_t *pmdp, pmd_t pmd, unsigned long addr,
  21                unsigned long end, int write, struct page **pages, int *nr)
  22{
  23        struct page *head, *page;
  24        unsigned long mask;
  25        pte_t *ptep, pte;
  26
  27        mask = (write ? _PAGE_PROTECT : 0) | _PAGE_INVALID | _PAGE_SPECIAL;
  28
  29        ptep = ((pte_t *) pmd_deref(pmd)) + pte_index(addr);
  30        do {
  31                pte = *ptep;
  32                barrier();
  33                /* Similar to the PMD case, NUMA hinting must take slow path */
  34                if (pte_protnone(pte))
  35                        return 0;
  36                if ((pte_val(pte) & mask) != 0)
  37                        return 0;
  38                VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
  39                page = pte_page(pte);
  40                head = compound_head(page);
  41                if (!page_cache_get_speculative(head))
  42                        return 0;
  43                if (unlikely(pte_val(pte) != pte_val(*ptep))) {
  44                        put_page(head);
  45                        return 0;
  46                }
  47                VM_BUG_ON_PAGE(compound_head(page) != head, page);
  48                pages[*nr] = page;
  49                (*nr)++;
  50
  51        } while (ptep++, addr += PAGE_SIZE, addr != end);
  52
  53        return 1;
  54}
  55
  56static inline int gup_huge_pmd(pmd_t *pmdp, pmd_t pmd, unsigned long addr,
  57                unsigned long end, int write, struct page **pages, int *nr)
  58{
  59        unsigned long mask, result;
  60        struct page *head, *page;
  61        int refs;
  62
  63        result = write ? 0 : _SEGMENT_ENTRY_PROTECT;
  64        mask = result | _SEGMENT_ENTRY_INVALID;
  65        if ((pmd_val(pmd) & mask) != result)
  66                return 0;
  67        VM_BUG_ON(!pfn_valid(pmd_val(pmd) >> PAGE_SHIFT));
  68
  69        refs = 0;
  70        head = pmd_page(pmd);
  71        page = head + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
  72        do {
  73                VM_BUG_ON(compound_head(page) != head);
  74                pages[*nr] = page;
  75                (*nr)++;
  76                page++;
  77                refs++;
  78        } while (addr += PAGE_SIZE, addr != end);
  79
  80        if (!page_cache_add_speculative(head, refs)) {
  81                *nr -= refs;
  82                return 0;
  83        }
  84
  85        if (unlikely(pmd_val(pmd) != pmd_val(*pmdp))) {
  86                *nr -= refs;
  87                while (refs--)
  88                        put_page(head);
  89                return 0;
  90        }
  91
  92        return 1;
  93}
  94
  95
  96static inline int gup_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr,
  97                unsigned long end, int write, struct page **pages, int *nr)
  98{
  99        unsigned long next;
 100        pmd_t *pmdp, pmd;
 101
 102        pmdp = (pmd_t *) pudp;
 103        if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
 104                pmdp = (pmd_t *) pud_deref(pud);
 105        pmdp += pmd_index(addr);
 106        do {
 107                pmd = *pmdp;
 108                barrier();
 109                next = pmd_addr_end(addr, end);
 110                if (pmd_none(pmd))
 111                        return 0;
 112                if (unlikely(pmd_large(pmd))) {
 113                        /*
 114                         * NUMA hinting faults need to be handled in the GUP
 115                         * slowpath for accounting purposes and so that they
 116                         * can be serialised against THP migration.
 117                         */
 118                        if (pmd_protnone(pmd))
 119                                return 0;
 120                        if (!gup_huge_pmd(pmdp, pmd, addr, next,
 121                                          write, pages, nr))
 122                                return 0;
 123                } else if (!gup_pte_range(pmdp, pmd, addr, next,
 124                                          write, pages, nr))
 125                        return 0;
 126        } while (pmdp++, addr = next, addr != end);
 127
 128        return 1;
 129}
 130
 131static int gup_huge_pud(pud_t *pudp, pud_t pud, unsigned long addr,
 132                unsigned long end, int write, struct page **pages, int *nr)
 133{
 134        struct page *head, *page;
 135        unsigned long mask;
 136        int refs;
 137
 138        mask = (write ? _REGION_ENTRY_PROTECT : 0) | _REGION_ENTRY_INVALID;
 139        if ((pud_val(pud) & mask) != 0)
 140                return 0;
 141        VM_BUG_ON(!pfn_valid(pud_pfn(pud)));
 142
 143        refs = 0;
 144        head = pud_page(pud);
 145        page = head + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
 146        do {
 147                VM_BUG_ON_PAGE(compound_head(page) != head, page);
 148                pages[*nr] = page;
 149                (*nr)++;
 150                page++;
 151                refs++;
 152        } while (addr += PAGE_SIZE, addr != end);
 153
 154        if (!page_cache_add_speculative(head, refs)) {
 155                *nr -= refs;
 156                return 0;
 157        }
 158
 159        if (unlikely(pud_val(pud) != pud_val(*pudp))) {
 160                *nr -= refs;
 161                while (refs--)
 162                        put_page(head);
 163                return 0;
 164        }
 165
 166        return 1;
 167}
 168
 169static inline int gup_pud_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr,
 170                unsigned long end, int write, struct page **pages, int *nr)
 171{
 172        unsigned long next;
 173        pud_t *pudp, pud;
 174
 175        pudp = (pud_t *) pgdp;
 176        if ((pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2)
 177                pudp = (pud_t *) pgd_deref(pgd);
 178        pudp += pud_index(addr);
 179        do {
 180                pud = *pudp;
 181                barrier();
 182                next = pud_addr_end(addr, end);
 183                if (pud_none(pud))
 184                        return 0;
 185                if (unlikely(pud_large(pud))) {
 186                        if (!gup_huge_pud(pudp, pud, addr, next, write, pages,
 187                                          nr))
 188                                return 0;
 189                } else if (!gup_pmd_range(pudp, pud, addr, next, write, pages,
 190                                          nr))
 191                        return 0;
 192        } while (pudp++, addr = next, addr != end);
 193
 194        return 1;
 195}
 196
 197/*
 198 * Like get_user_pages_fast() except its IRQ-safe in that it won't fall
 199 * back to the regular GUP.
 200 */
 201int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
 202                          struct page **pages)
 203{
 204        struct mm_struct *mm = current->mm;
 205        unsigned long addr, len, end;
 206        unsigned long next, flags;
 207        pgd_t *pgdp, pgd;
 208        int nr = 0;
 209
 210        start &= PAGE_MASK;
 211        addr = start;
 212        len = (unsigned long) nr_pages << PAGE_SHIFT;
 213        end = start + len;
 214        if ((end <= start) || (end > TASK_SIZE))
 215                return 0;
 216        /*
 217         * local_irq_save() doesn't prevent pagetable teardown, but does
 218         * prevent the pagetables from being freed on s390.
 219         *
 220         * So long as we atomically load page table pointers versus teardown,
 221         * we can follow the address down to the the page and take a ref on it.
 222         */
 223        local_irq_save(flags);
 224        pgdp = pgd_offset(mm, addr);
 225        do {
 226                pgd = *pgdp;
 227                barrier();
 228                next = pgd_addr_end(addr, end);
 229                if (pgd_none(pgd))
 230                        break;
 231                if (!gup_pud_range(pgdp, pgd, addr, next, write, pages, &nr))
 232                        break;
 233        } while (pgdp++, addr = next, addr != end);
 234        local_irq_restore(flags);
 235
 236        return nr;
 237}
 238
 239/**
 240 * get_user_pages_fast() - pin user pages in memory
 241 * @start:      starting user address
 242 * @nr_pages:   number of pages from start to pin
 243 * @write:      whether pages will be written to
 244 * @pages:      array that receives pointers to the pages pinned.
 245 *              Should be at least nr_pages long.
 246 *
 247 * Attempt to pin user pages in memory without taking mm->mmap_sem.
 248 * If not successful, it will fall back to taking the lock and
 249 * calling get_user_pages().
 250 *
 251 * Returns number of pages pinned. This may be fewer than the number
 252 * requested. If nr_pages is 0 or negative, returns 0. If no pages
 253 * were pinned, returns -errno.
 254 */
 255int get_user_pages_fast(unsigned long start, int nr_pages, int write,
 256                        struct page **pages)
 257{
 258        int nr, ret;
 259
 260        might_sleep();
 261        start &= PAGE_MASK;
 262        nr = __get_user_pages_fast(start, nr_pages, write, pages);
 263        if (nr == nr_pages)
 264                return nr;
 265
 266        /* Try to get the remaining pages with get_user_pages */
 267        start += nr << PAGE_SHIFT;
 268        pages += nr;
 269        ret = get_user_pages_unlocked(start, nr_pages - nr, pages,
 270                                      write ? FOLL_WRITE : 0);
 271        /* Have to be a bit careful with return values */
 272        if (nr > 0)
 273                ret = (ret < 0) ? nr : ret + nr;
 274        return ret;
 275}
 276