LXR linux/arch/arm/mm/context.c

   1/*
   2 *  linux/arch/arm/mm/context.c
   3 *
   4 *  Copyright (C) 2002-2003 Deep Blue Solutions Ltd, all rights reserved.
   5 *  Copyright (C) 2012 ARM Limited
   6 *
   7 *  Author: Will Deacon <will.deacon@arm.com>
   8 *
   9 * This program is free software; you can redistribute it and/or modify
  10 * it under the terms of the GNU General Public License version 2 as
  11 * published by the Free Software Foundation.
  12 */
  13#include <linux/init.h>
  14#include <linux/sched.h>
  15#include <linux/mm.h>
  16#include <linux/smp.h>
  17#include <linux/percpu.h>
  18
  19#include <asm/mmu_context.h>
  20#include <asm/smp_plat.h>
  21#include <asm/thread_notify.h>
  22#include <asm/tlbflush.h>
  23#include <asm/proc-fns.h>
  24
  25/*
  26 * On ARMv6, we have the following structure in the Context ID:
  27 *
  28 * 31                         7          0
  29 * +-------------------------+-----------+
  30 * |      process ID         |   ASID    |
  31 * +-------------------------+-----------+
  32 * |              context ID             |
  33 * +-------------------------------------+
  34 *
  35 * The ASID is used to tag entries in the CPU caches and TLBs.
  36 * The context ID is used by debuggers and trace logic, and
  37 * should be unique within all running processes.
  38 *
  39 * In big endian operation, the two 32 bit words are swapped if accessed
  40 * by non-64-bit operations.
  41 */
  42#define ASID_FIRST_VERSION      (1ULL << ASID_BITS)
  43#define NUM_USER_ASIDS          ASID_FIRST_VERSION
  44
  45static DEFINE_RAW_SPINLOCK(cpu_asid_lock);
  46static atomic64_t asid_generation = ATOMIC64_INIT(ASID_FIRST_VERSION);
  47static DECLARE_BITMAP(asid_map, NUM_USER_ASIDS);
  48
  49static DEFINE_PER_CPU(atomic64_t, active_asids);
  50static DEFINE_PER_CPU(u64, reserved_asids);
  51static cpumask_t tlb_flush_pending;
  52
  53#ifdef CONFIG_ARM_ERRATA_798181
  54void a15_erratum_get_cpumask(int this_cpu, struct mm_struct *mm,
  55                             cpumask_t *mask)
  56{
  57        int cpu;
  58        unsigned long flags;
  59        u64 context_id, asid;
  60
  61        raw_spin_lock_irqsave(&cpu_asid_lock, flags);
  62        context_id = mm->context.id.counter;
  63        for_each_online_cpu(cpu) {
  64                if (cpu == this_cpu)
  65                        continue;
  66                /*
  67                 * We only need to send an IPI if the other CPUs are
  68                 * running the same ASID as the one being invalidated.
  69                 */
  70                asid = per_cpu(active_asids, cpu).counter;
  71                if (asid == 0)
  72                        asid = per_cpu(reserved_asids, cpu);
  73                if (context_id == asid)
  74                        cpumask_set_cpu(cpu, mask);
  75        }
  76        raw_spin_unlock_irqrestore(&cpu_asid_lock, flags);
  77}
  78#endif
  79
  80#ifdef CONFIG_ARM_LPAE
  81/*
  82 * With LPAE, the ASID and page tables are updated atomicly, so there is
  83 * no need for a reserved set of tables (the active ASID tracking prevents
  84 * any issues across a rollover).
  85 */
  86#define cpu_set_reserved_ttbr0()
  87#else
  88static void cpu_set_reserved_ttbr0(void)
  89{
  90        u32 ttb;
  91        /*
  92         * Copy TTBR1 into TTBR0.
  93         * This points at swapper_pg_dir, which contains only global
  94         * entries so any speculative walks are perfectly safe.
  95         */
  96        asm volatile(
  97        "       mrc     p15, 0, %0, c2, c0, 1           @ read TTBR1\n"
  98        "       mcr     p15, 0, %0, c2, c0, 0           @ set TTBR0\n"
  99        : "=r" (ttb));
 100        isb();
 101}
 102#endif
 103
 104#ifdef CONFIG_PID_IN_CONTEXTIDR
 105static int contextidr_notifier(struct notifier_block *unused, unsigned long cmd,
 106                               void *t)
 107{
 108        u32 contextidr;
 109        pid_t pid;
 110        struct thread_info *thread = t;
 111
 112        if (cmd != THREAD_NOTIFY_SWITCH)
 113                return NOTIFY_DONE;
 114
 115        pid = task_pid_nr(thread->task) << ASID_BITS;
 116        asm volatile(
 117        "       mrc     p15, 0, %0, c13, c0, 1\n"
 118        "       and     %0, %0, %2\n"
 119        "       orr     %0, %0, %1\n"
 120        "       mcr     p15, 0, %0, c13, c0, 1\n"
 121        : "=r" (contextidr), "+r" (pid)
 122        : "I" (~ASID_MASK));
 123        isb();
 124
 125        return NOTIFY_OK;
 126}
 127
 128static struct notifier_block contextidr_notifier_block = {
 129        .notifier_call = contextidr_notifier,
 130};
 131
 132static int __init contextidr_notifier_init(void)
 133{
 134        return thread_register_notifier(&contextidr_notifier_block);
 135}
 136arch_initcall(contextidr_notifier_init);
 137#endif
 138
 139static void flush_context(unsigned int cpu)
 140{
 141        int i;
 142        u64 asid;
 143
 144        /* Update the list of reserved ASIDs and the ASID bitmap. */
 145        bitmap_clear(asid_map, 0, NUM_USER_ASIDS);
 146        for_each_possible_cpu(i) {
 147                asid = atomic64_xchg(&per_cpu(active_asids, i), 0);
 148                /*
 149                 * If this CPU has already been through a
 150                 * rollover, but hasn't run another task in
 151                 * the meantime, we must preserve its reserved
 152                 * ASID, as this is the only trace we have of
 153                 * the process it is still running.
 154                 */
 155                if (asid == 0)
 156                        asid = per_cpu(reserved_asids, i);
 157                __set_bit(asid & ~ASID_MASK, asid_map);
 158                per_cpu(reserved_asids, i) = asid;
 159        }
 160
 161        /* Queue a TLB invalidate and flush the I-cache if necessary. */
 162        cpumask_setall(&tlb_flush_pending);
 163
 164        if (icache_is_vivt_asid_tagged())
 165                __flush_icache_all();
 166}
 167
 168static bool check_update_reserved_asid(u64 asid, u64 newasid)
 169{
 170        int cpu;
 171        bool hit = false;
 172
 173        /*
 174         * Iterate over the set of reserved ASIDs looking for a match.
 175         * If we find one, then we can update our mm to use newasid
 176         * (i.e. the same ASID in the current generation) but we can't
 177         * exit the loop early, since we need to ensure that all copies
 178         * of the old ASID are updated to reflect the mm. Failure to do
 179         * so could result in us missing the reserved ASID in a future
 180         * generation.
 181         */
 182        for_each_possible_cpu(cpu) {
 183                if (per_cpu(reserved_asids, cpu) == asid) {
 184                        hit = true;
 185                        per_cpu(reserved_asids, cpu) = newasid;
 186                }
 187        }
 188
 189        return hit;
 190}
 191
 192static u64 new_context(struct mm_struct *mm, unsigned int cpu)
 193{
 194        static u32 cur_idx = 1;
 195        u64 asid = atomic64_read(&mm->context.id);
 196        u64 generation = atomic64_read(&asid_generation);
 197
 198        if (asid != 0) {
 199                u64 newasid = generation | (asid & ~ASID_MASK);
 200
 201                /*
 202                 * If our current ASID was active during a rollover, we
 203                 * can continue to use it and this was just a false alarm.
 204                 */
 205                if (check_update_reserved_asid(asid, newasid))
 206                        return newasid;
 207
 208                /*
 209                 * We had a valid ASID in a previous life, so try to re-use
 210                 * it if possible.,
 211                 */
 212                asid &= ~ASID_MASK;
 213                if (!__test_and_set_bit(asid, asid_map))
 214                        return newasid;
 215        }
 216
 217        /*
 218         * Allocate a free ASID. If we can't find one, take a note of the
 219         * currently active ASIDs and mark the TLBs as requiring flushes.
 220         * We always count from ASID #1, as we reserve ASID #0 to switch
 221         * via TTBR0 and to avoid speculative page table walks from hitting
 222         * in any partial walk caches, which could be populated from
 223         * overlapping level-1 descriptors used to map both the module
 224         * area and the userspace stack.
 225         */
 226        asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, cur_idx);
 227        if (asid == NUM_USER_ASIDS) {
 228                generation = atomic64_add_return(ASID_FIRST_VERSION,
 229                                                 &asid_generation);
 230                flush_context(cpu);
 231                asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, 1);
 232        }
 233
 234        __set_bit(asid, asid_map);
 235        cur_idx = asid;
 236        cpumask_clear(mm_cpumask(mm));
 237        return asid | generation;
 238}
 239
 240void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk)
 241{
 242        unsigned long flags;
 243        unsigned int cpu = smp_processor_id();
 244        u64 asid;
 245
 246        if (unlikely(mm->context.vmalloc_seq != init_mm.context.vmalloc_seq))
 247                __check_vmalloc_seq(mm);
 248
 249        /*
 250         * We cannot update the pgd and the ASID atomicly with classic
 251         * MMU, so switch exclusively to global mappings to avoid
 252         * speculative page table walking with the wrong TTBR.
 253         */
 254        cpu_set_reserved_ttbr0();
 255
 256        asid = atomic64_read(&mm->context.id);
 257        if (!((asid ^ atomic64_read(&asid_generation)) >> ASID_BITS)
 258            && atomic64_xchg(&per_cpu(active_asids, cpu), asid))
 259                goto switch_mm_fastpath;
 260
 261        raw_spin_lock_irqsave(&cpu_asid_lock, flags);
 262        /* Check that our ASID belongs to the current generation. */
 263        asid = atomic64_read(&mm->context.id);
 264        if ((asid ^ atomic64_read(&asid_generation)) >> ASID_BITS) {
 265                asid = new_context(mm, cpu);
 266                atomic64_set(&mm->context.id, asid);
 267        }
 268
 269        if (cpumask_test_and_clear_cpu(cpu, &tlb_flush_pending)) {
 270                local_flush_bp_all();
 271                local_flush_tlb_all();
 272        }
 273
 274        atomic64_set(&per_cpu(active_asids, cpu), asid);
 275        cpumask_set_cpu(cpu, mm_cpumask(mm));
 276        raw_spin_unlock_irqrestore(&cpu_asid_lock, flags);
 277
 278switch_mm_fastpath:
 279        cpu_switch_mm(mm->pgd, mm);
 280}
 281