LXR linux/mm/vmacache.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright (C) 2014 Davidlohr Bueso.
   4 */
   5#include <linux/sched/signal.h>
   6#include <linux/sched/task.h>
   7#include <linux/mm.h>
   8#include <linux/vmacache.h>
   9
  10/*
  11 * Hash based on the pmd of addr if configured with MMU, which provides a good
  12 * hit rate for workloads with spatial locality.  Otherwise, use pages.
  13 */
  14#ifdef CONFIG_MMU
  15#define VMACACHE_SHIFT  PMD_SHIFT
  16#else
  17#define VMACACHE_SHIFT  PAGE_SHIFT
  18#endif
  19#define VMACACHE_HASH(addr) ((addr >> VMACACHE_SHIFT) & VMACACHE_MASK)
  20
  21/*
  22 * This task may be accessing a foreign mm via (for example)
  23 * get_user_pages()->find_vma().  The vmacache is task-local and this
  24 * task's vmacache pertains to a different mm (ie, its own).  There is
  25 * nothing we can do here.
  26 *
  27 * Also handle the case where a kernel thread has adopted this mm via
  28 * kthread_use_mm(). That kernel thread's vmacache is not applicable to this mm.
  29 */
  30static inline bool vmacache_valid_mm(struct mm_struct *mm)
  31{
  32        return current->mm == mm && !(current->flags & PF_KTHREAD);
  33}
  34
  35void vmacache_update(unsigned long addr, struct vm_area_struct *newvma)
  36{
  37        if (vmacache_valid_mm(newvma->vm_mm))
  38                current->vmacache.vmas[VMACACHE_HASH(addr)] = newvma;
  39}
  40
  41static bool vmacache_valid(struct mm_struct *mm)
  42{
  43        struct task_struct *curr;
  44
  45        if (!vmacache_valid_mm(mm))
  46                return false;
  47
  48        curr = current;
  49        if (mm->vmacache_seqnum != curr->vmacache.seqnum) {
  50                /*
  51                 * First attempt will always be invalid, initialize
  52                 * the new cache for this task here.
  53                 */
  54                curr->vmacache.seqnum = mm->vmacache_seqnum;
  55                vmacache_flush(curr);
  56                return false;
  57        }
  58        return true;
  59}
  60
  61struct vm_area_struct *vmacache_find(struct mm_struct *mm, unsigned long addr)
  62{
  63        int idx = VMACACHE_HASH(addr);
  64        int i;
  65
  66        count_vm_vmacache_event(VMACACHE_FIND_CALLS);
  67
  68        if (!vmacache_valid(mm))
  69                return NULL;
  70
  71        for (i = 0; i < VMACACHE_SIZE; i++) {
  72                struct vm_area_struct *vma = current->vmacache.vmas[idx];
  73
  74                if (vma) {
  75#ifdef CONFIG_DEBUG_VM_VMACACHE
  76                        if (WARN_ON_ONCE(vma->vm_mm != mm))
  77                                break;
  78#endif
  79                        if (vma->vm_start <= addr && vma->vm_end > addr) {
  80                                count_vm_vmacache_event(VMACACHE_FIND_HITS);
  81                                return vma;
  82                        }
  83                }
  84                if (++idx == VMACACHE_SIZE)
  85                        idx = 0;
  86        }
  87
  88        return NULL;
  89}
  90
  91#ifndef CONFIG_MMU
  92struct vm_area_struct *vmacache_find_exact(struct mm_struct *mm,
  93                                           unsigned long start,
  94                                           unsigned long end)
  95{
  96        int idx = VMACACHE_HASH(start);
  97        int i;
  98
  99        count_vm_vmacache_event(VMACACHE_FIND_CALLS);
 100
 101        if (!vmacache_valid(mm))
 102                return NULL;
 103
 104        for (i = 0; i < VMACACHE_SIZE; i++) {
 105                struct vm_area_struct *vma = current->vmacache.vmas[idx];
 106
 107                if (vma && vma->vm_start == start && vma->vm_end == end) {
 108                        count_vm_vmacache_event(VMACACHE_FIND_HITS);
 109                        return vma;
 110                }
 111                if (++idx == VMACACHE_SIZE)
 112                        idx = 0;
 113        }
 114
 115        return NULL;
 116}
 117#endif
 118