LXR linux/arch/powerpc/mm/mmu_context

   1/*
   2 *  IOMMU helpers in MMU context.
   3 *
   4 *  Copyright (C) 2015 IBM Corp. <aik@ozlabs.ru>
   5 *
   6 *  This program is free software; you can redistribute it and/or
   7 *  modify it under the terms of the GNU General Public License
   8 *  as published by the Free Software Foundation; either version
   9 *  2 of the License, or (at your option) any later version.
  10 *
  11 */
  12
  13#include <linux/sched/signal.h>
  14#include <linux/slab.h>
  15#include <linux/rculist.h>
  16#include <linux/vmalloc.h>
  17#include <linux/mutex.h>
  18#include <linux/migrate.h>
  19#include <linux/hugetlb.h>
  20#include <linux/swap.h>
  21#include <asm/mmu_context.h>
  22#include <asm/pte-walk.h>
  23
  24static DEFINE_MUTEX(mem_list_mutex);
  25
  26struct mm_iommu_table_group_mem_t {
  27        struct list_head next;
  28        struct rcu_head rcu;
  29        unsigned long used;
  30        atomic64_t mapped;
  31        unsigned int pageshift;
  32        u64 ua;                 /* userspace address */
  33        u64 entries;            /* number of entries in hpas[] */
  34        u64 *hpas;              /* vmalloc'ed */
  35};
  36
  37static long mm_iommu_adjust_locked_vm(struct mm_struct *mm,
  38                unsigned long npages, bool incr)
  39{
  40        long ret = 0, locked, lock_limit;
  41
  42        if (!npages)
  43                return 0;
  44
  45        down_write(&mm->mmap_sem);
  46
  47        if (incr) {
  48                locked = mm->locked_vm + npages;
  49                lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
  50                if (locked > lock_limit && !capable(CAP_IPC_LOCK))
  51                        ret = -ENOMEM;
  52                else
  53                        mm->locked_vm += npages;
  54        } else {
  55                if (WARN_ON_ONCE(npages > mm->locked_vm))
  56                        npages = mm->locked_vm;
  57                mm->locked_vm -= npages;
  58        }
  59
  60        pr_debug("[%d] RLIMIT_MEMLOCK HASH64 %c%ld %ld/%ld\n",
  61                        current ? current->pid : 0,
  62                        incr ? '+' : '-',
  63                        npages << PAGE_SHIFT,
  64                        mm->locked_vm << PAGE_SHIFT,
  65                        rlimit(RLIMIT_MEMLOCK));
  66        up_write(&mm->mmap_sem);
  67
  68        return ret;
  69}
  70
  71bool mm_iommu_preregistered(struct mm_struct *mm)
  72{
  73        return !list_empty(&mm->context.iommu_group_mem_list);
  74}
  75EXPORT_SYMBOL_GPL(mm_iommu_preregistered);
  76
  77/*
  78 * Taken from alloc_migrate_target with changes to remove CMA allocations
  79 */
  80struct page *new_iommu_non_cma_page(struct page *page, unsigned long private)
  81{
  82        gfp_t gfp_mask = GFP_USER;
  83        struct page *new_page;
  84
  85        if (PageCompound(page))
  86                return NULL;
  87
  88        if (PageHighMem(page))
  89                gfp_mask |= __GFP_HIGHMEM;
  90
  91        /*
  92         * We don't want the allocation to force an OOM if possibe
  93         */
  94        new_page = alloc_page(gfp_mask | __GFP_NORETRY | __GFP_NOWARN);
  95        return new_page;
  96}
  97
  98static int mm_iommu_move_page_from_cma(struct page *page)
  99{
 100        int ret = 0;
 101        LIST_HEAD(cma_migrate_pages);
 102
 103        /* Ignore huge pages for now */
 104        if (PageCompound(page))
 105                return -EBUSY;
 106
 107        lru_add_drain();
 108        ret = isolate_lru_page(page);
 109        if (ret)
 110                return ret;
 111
 112        list_add(&page->lru, &cma_migrate_pages);
 113        put_page(page); /* Drop the gup reference */
 114
 115        ret = migrate_pages(&cma_migrate_pages, new_iommu_non_cma_page,
 116                                NULL, 0, MIGRATE_SYNC, MR_CONTIG_RANGE);
 117        if (ret) {
 118                if (!list_empty(&cma_migrate_pages))
 119                        putback_movable_pages(&cma_migrate_pages);
 120        }
 121
 122        return 0;
 123}
 124
 125long mm_iommu_get(struct mm_struct *mm, unsigned long ua, unsigned long entries,
 126                struct mm_iommu_table_group_mem_t **pmem)
 127{
 128        struct mm_iommu_table_group_mem_t *mem;
 129        long i, j, ret = 0, locked_entries = 0;
 130        unsigned int pageshift;
 131        unsigned long flags;
 132        struct page *page = NULL;
 133
 134        mutex_lock(&mem_list_mutex);
 135
 136        list_for_each_entry_rcu(mem, &mm->context.iommu_group_mem_list,
 137                        next) {
 138                if ((mem->ua == ua) && (mem->entries == entries)) {
 139                        ++mem->used;
 140                        *pmem = mem;
 141                        goto unlock_exit;
 142                }
 143
 144                /* Overlap? */
 145                if ((mem->ua < (ua + (entries << PAGE_SHIFT))) &&
 146                                (ua < (mem->ua +
 147                                       (mem->entries << PAGE_SHIFT)))) {
 148                        ret = -EINVAL;
 149                        goto unlock_exit;
 150                }
 151
 152        }
 153
 154        ret = mm_iommu_adjust_locked_vm(mm, entries, true);
 155        if (ret)
 156                goto unlock_exit;
 157
 158        locked_entries = entries;
 159
 160        mem = kzalloc(sizeof(*mem), GFP_KERNEL);
 161        if (!mem) {
 162                ret = -ENOMEM;
 163                goto unlock_exit;
 164        }
 165
 166        /*
 167         * For a starting point for a maximum page size calculation
 168         * we use @ua and @entries natural alignment to allow IOMMU pages
 169         * smaller than huge pages but still bigger than PAGE_SIZE.
 170         */
 171        mem->pageshift = __ffs(ua | (entries << PAGE_SHIFT));
 172        mem->hpas = vzalloc(array_size(entries, sizeof(mem->hpas[0])));
 173        if (!mem->hpas) {
 174                kfree(mem);
 175                ret = -ENOMEM;
 176                goto unlock_exit;
 177        }
 178
 179        for (i = 0; i < entries; ++i) {
 180                if (1 != get_user_pages_fast(ua + (i << PAGE_SHIFT),
 181                                        1/* pages */, 1/* iswrite */, &page)) {
 182                        ret = -EFAULT;
 183                        for (j = 0; j < i; ++j)
 184                                put_page(pfn_to_page(mem->hpas[j] >>
 185                                                PAGE_SHIFT));
 186                        vfree(mem->hpas);
 187                        kfree(mem);
 188                        goto unlock_exit;
 189                }
 190                /*
 191                 * If we get a page from the CMA zone, since we are going to
 192                 * be pinning these entries, we might as well move them out
 193                 * of the CMA zone if possible. NOTE: faulting in + migration
 194                 * can be expensive. Batching can be considered later
 195                 */
 196                if (is_migrate_cma_page(page)) {
 197                        if (mm_iommu_move_page_from_cma(page))
 198                                goto populate;
 199                        if (1 != get_user_pages_fast(ua + (i << PAGE_SHIFT),
 200                                                1/* pages */, 1/* iswrite */,
 201                                                &page)) {
 202                                ret = -EFAULT;
 203                                for (j = 0; j < i; ++j)
 204                                        put_page(pfn_to_page(mem->hpas[j] >>
 205                                                                PAGE_SHIFT));
 206                                vfree(mem->hpas);
 207                                kfree(mem);
 208                                goto unlock_exit;
 209                        }
 210                }
 211populate:
 212                pageshift = PAGE_SHIFT;
 213                if (PageCompound(page)) {
 214                        pte_t *pte;
 215                        struct page *head = compound_head(page);
 216                        unsigned int compshift = compound_order(head);
 217
 218                        local_irq_save(flags); /* disables as well */
 219                        pte = find_linux_pte(mm->pgd, ua, NULL, &pageshift);
 220                        local_irq_restore(flags);
 221
 222                        /* Double check it is still the same pinned page */
 223                        if (pte && pte_page(*pte) == head &&
 224                                        pageshift == compshift)
 225                                pageshift = max_t(unsigned int, pageshift,
 226                                                PAGE_SHIFT);
 227                }
 228                mem->pageshift = min(mem->pageshift, pageshift);
 229                mem->hpas[i] = page_to_pfn(page) << PAGE_SHIFT;
 230        }
 231
 232        atomic64_set(&mem->mapped, 1);
 233        mem->used = 1;
 234        mem->ua = ua;
 235        mem->entries = entries;
 236        *pmem = mem;
 237
 238        list_add_rcu(&mem->next, &mm->context.iommu_group_mem_list);
 239
 240unlock_exit:
 241        if (locked_entries && ret)
 242                mm_iommu_adjust_locked_vm(mm, locked_entries, false);
 243
 244        mutex_unlock(&mem_list_mutex);
 245
 246        return ret;
 247}
 248EXPORT_SYMBOL_GPL(mm_iommu_get);
 249
 250static void mm_iommu_unpin(struct mm_iommu_table_group_mem_t *mem)
 251{
 252        long i;
 253        struct page *page = NULL;
 254
 255        for (i = 0; i < mem->entries; ++i) {
 256                if (!mem->hpas[i])
 257                        continue;
 258
 259                page = pfn_to_page(mem->hpas[i] >> PAGE_SHIFT);
 260                if (!page)
 261                        continue;
 262
 263                put_page(page);
 264                mem->hpas[i] = 0;
 265        }
 266}
 267
 268static void mm_iommu_do_free(struct mm_iommu_table_group_mem_t *mem)
 269{
 270
 271        mm_iommu_unpin(mem);
 272        vfree(mem->hpas);
 273        kfree(mem);
 274}
 275
 276static void mm_iommu_free(struct rcu_head *head)
 277{
 278        struct mm_iommu_table_group_mem_t *mem = container_of(head,
 279                        struct mm_iommu_table_group_mem_t, rcu);
 280
 281        mm_iommu_do_free(mem);
 282}
 283
 284static void mm_iommu_release(struct mm_iommu_table_group_mem_t *mem)
 285{
 286        list_del_rcu(&mem->next);
 287        call_rcu(&mem->rcu, mm_iommu_free);
 288}
 289
 290long mm_iommu_put(struct mm_struct *mm, struct mm_iommu_table_group_mem_t *mem)
 291{
 292        long ret = 0;
 293
 294        mutex_lock(&mem_list_mutex);
 295
 296        if (mem->used == 0) {
 297                ret = -ENOENT;
 298                goto unlock_exit;
 299        }
 300
 301        --mem->used;
 302        /* There are still users, exit */
 303        if (mem->used)
 304                goto unlock_exit;
 305
 306        /* Are there still mappings? */
 307        if (atomic_cmpxchg(&mem->mapped, 1, 0) != 1) {
 308                ++mem->used;
 309                ret = -EBUSY;
 310                goto unlock_exit;
 311        }
 312
 313        /* @mapped became 0 so now mappings are disabled, release the region */
 314        mm_iommu_release(mem);
 315
 316        mm_iommu_adjust_locked_vm(mm, mem->entries, false);
 317
 318unlock_exit:
 319        mutex_unlock(&mem_list_mutex);
 320
 321        return ret;
 322}
 323EXPORT_SYMBOL_GPL(mm_iommu_put);
 324
 325struct mm_iommu_table_group_mem_t *mm_iommu_lookup(struct mm_struct *mm,
 326                unsigned long ua, unsigned long size)
 327{
 328        struct mm_iommu_table_group_mem_t *mem, *ret = NULL;
 329
 330        list_for_each_entry_rcu(mem, &mm->context.iommu_group_mem_list, next) {
 331                if ((mem->ua <= ua) &&
 332                                (ua + size <= mem->ua +
 333                                 (mem->entries << PAGE_SHIFT))) {
 334                        ret = mem;
 335                        break;
 336                }
 337        }
 338
 339        return ret;
 340}
 341EXPORT_SYMBOL_GPL(mm_iommu_lookup);
 342
 343struct mm_iommu_table_group_mem_t *mm_iommu_lookup_rm(struct mm_struct *mm,
 344                unsigned long ua, unsigned long size)
 345{
 346        struct mm_iommu_table_group_mem_t *mem, *ret = NULL;
 347
 348        list_for_each_entry_lockless(mem, &mm->context.iommu_group_mem_list,
 349                        next) {
 350                if ((mem->ua <= ua) &&
 351                                (ua + size <= mem->ua +
 352                                 (mem->entries << PAGE_SHIFT))) {
 353                        ret = mem;
 354                        break;
 355                }
 356        }
 357
 358        return ret;
 359}
 360EXPORT_SYMBOL_GPL(mm_iommu_lookup_rm);
 361
 362struct mm_iommu_table_group_mem_t *mm_iommu_find(struct mm_struct *mm,
 363                unsigned long ua, unsigned long entries)
 364{
 365        struct mm_iommu_table_group_mem_t *mem, *ret = NULL;
 366
 367        list_for_each_entry_rcu(mem, &mm->context.iommu_group_mem_list, next) {
 368                if ((mem->ua == ua) && (mem->entries == entries)) {
 369                        ret = mem;
 370                        break;
 371                }
 372        }
 373
 374        return ret;
 375}
 376EXPORT_SYMBOL_GPL(mm_iommu_find);
 377
 378long mm_iommu_ua_to_hpa(struct mm_iommu_table_group_mem_t *mem,
 379                unsigned long ua, unsigned int pageshift, unsigned long *hpa)
 380{
 381        const long entry = (ua - mem->ua) >> PAGE_SHIFT;
 382        u64 *va = &mem->hpas[entry];
 383
 384        if (entry >= mem->entries)
 385                return -EFAULT;
 386
 387        if (pageshift > mem->pageshift)
 388                return -EFAULT;
 389
 390        *hpa = *va | (ua & ~PAGE_MASK);
 391
 392        return 0;
 393}
 394EXPORT_SYMBOL_GPL(mm_iommu_ua_to_hpa);
 395
 396long mm_iommu_ua_to_hpa_rm(struct mm_iommu_table_group_mem_t *mem,
 397                unsigned long ua, unsigned int pageshift, unsigned long *hpa)
 398{
 399        const long entry = (ua - mem->ua) >> PAGE_SHIFT;
 400        void *va = &mem->hpas[entry];
 401        unsigned long *pa;
 402
 403        if (entry >= mem->entries)
 404                return -EFAULT;
 405
 406        if (pageshift > mem->pageshift)
 407                return -EFAULT;
 408
 409        pa = (void *) vmalloc_to_phys(va);
 410        if (!pa)
 411                return -EFAULT;
 412
 413        *hpa = *pa | (ua & ~PAGE_MASK);
 414
 415        return 0;
 416}
 417EXPORT_SYMBOL_GPL(mm_iommu_ua_to_hpa_rm);
 418
 419long mm_iommu_mapped_inc(struct mm_iommu_table_group_mem_t *mem)
 420{
 421        if (atomic64_inc_not_zero(&mem->mapped))
 422                return 0;
 423
 424        /* Last mm_iommu_put() has been called, no more mappings allowed() */
 425        return -ENXIO;
 426}
 427EXPORT_SYMBOL_GPL(mm_iommu_mapped_inc);
 428
 429void mm_iommu_mapped_dec(struct mm_iommu_table_group_mem_t *mem)
 430{
 431        atomic64_add_unless(&mem->mapped, -1, 1);
 432}
 433EXPORT_SYMBOL_GPL(mm_iommu_mapped_dec);
 434
 435void mm_iommu_init(struct mm_struct *mm)
 436{
 437        INIT_LIST_HEAD_RCU(&mm->context.iommu_group_mem_list);
 438}
 439