LXR linux/arch/arm/kernel/smp

   1/*
   2 *  linux/arch/arm/kernel/smp_tlb.c
   3 *
   4 *  Copyright (C) 2002 ARM Limited, All Rights Reserved.
   5 *
   6 * This program is free software; you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License version 2 as
   8 * published by the Free Software Foundation.
   9 */
  10#include <linux/preempt.h>
  11#include <linux/smp.h>
  12
  13#include <asm/smp_plat.h>
  14#include <asm/tlbflush.h>
  15#include <asm/mmu_context.h>
  16
  17/**********************************************************************/
  18
  19/*
  20 * TLB operations
  21 */
  22struct tlb_args {
  23        struct vm_area_struct *ta_vma;
  24        unsigned long ta_start;
  25        unsigned long ta_end;
  26};
  27
  28static inline void ipi_flush_tlb_all(void *ignored)
  29{
  30        local_flush_tlb_all();
  31}
  32
  33static inline void ipi_flush_tlb_mm(void *arg)
  34{
  35        struct mm_struct *mm = (struct mm_struct *)arg;
  36
  37        local_flush_tlb_mm(mm);
  38}
  39
  40static inline void ipi_flush_tlb_page(void *arg)
  41{
  42        struct tlb_args *ta = (struct tlb_args *)arg;
  43
  44        local_flush_tlb_page(ta->ta_vma, ta->ta_start);
  45}
  46
  47static inline void ipi_flush_tlb_kernel_page(void *arg)
  48{
  49        struct tlb_args *ta = (struct tlb_args *)arg;
  50
  51        local_flush_tlb_kernel_page(ta->ta_start);
  52}
  53
  54static inline void ipi_flush_tlb_range(void *arg)
  55{
  56        struct tlb_args *ta = (struct tlb_args *)arg;
  57
  58        local_flush_tlb_range(ta->ta_vma, ta->ta_start, ta->ta_end);
  59}
  60
  61static inline void ipi_flush_tlb_kernel_range(void *arg)
  62{
  63        struct tlb_args *ta = (struct tlb_args *)arg;
  64
  65        local_flush_tlb_kernel_range(ta->ta_start, ta->ta_end);
  66}
  67
  68static inline void ipi_flush_bp_all(void *ignored)
  69{
  70        local_flush_bp_all();
  71}
  72
  73static void ipi_flush_tlb_a15_erratum(void *arg)
  74{
  75        dmb();
  76}
  77
  78static void broadcast_tlb_a15_erratum(void)
  79{
  80        if (!erratum_a15_798181())
  81                return;
  82
  83        dummy_flush_tlb_a15_erratum();
  84        smp_call_function(ipi_flush_tlb_a15_erratum, NULL, 1);
  85}
  86
  87static void broadcast_tlb_mm_a15_erratum(struct mm_struct *mm)
  88{
  89        int this_cpu;
  90        cpumask_t mask = { CPU_BITS_NONE };
  91
  92        if (!erratum_a15_798181())
  93                return;
  94
  95        dummy_flush_tlb_a15_erratum();
  96        this_cpu = get_cpu();
  97        a15_erratum_get_cpumask(this_cpu, mm, &mask);
  98        smp_call_function_many(&mask, ipi_flush_tlb_a15_erratum, NULL, 1);
  99        put_cpu();
 100}
 101
 102void flush_tlb_all(void)
 103{
 104        if (tlb_ops_need_broadcast())
 105                on_each_cpu(ipi_flush_tlb_all, NULL, 1);
 106        else
 107                __flush_tlb_all();
 108        broadcast_tlb_a15_erratum();
 109}
 110
 111void flush_tlb_mm(struct mm_struct *mm)
 112{
 113        if (tlb_ops_need_broadcast())
 114                on_each_cpu_mask(mm_cpumask(mm), ipi_flush_tlb_mm, mm, 1);
 115        else
 116                __flush_tlb_mm(mm);
 117        broadcast_tlb_mm_a15_erratum(mm);
 118}
 119
 120void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
 121{
 122        if (tlb_ops_need_broadcast()) {
 123                struct tlb_args ta;
 124                ta.ta_vma = vma;
 125                ta.ta_start = uaddr;
 126                on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_tlb_page,
 127                                        &ta, 1);
 128        } else
 129                __flush_tlb_page(vma, uaddr);
 130        broadcast_tlb_mm_a15_erratum(vma->vm_mm);
 131}
 132
 133void flush_tlb_kernel_page(unsigned long kaddr)
 134{
 135        if (tlb_ops_need_broadcast()) {
 136                struct tlb_args ta;
 137                ta.ta_start = kaddr;
 138                on_each_cpu(ipi_flush_tlb_kernel_page, &ta, 1);
 139        } else
 140                __flush_tlb_kernel_page(kaddr);
 141        broadcast_tlb_a15_erratum();
 142}
 143
 144void flush_tlb_range(struct vm_area_struct *vma,
 145                     unsigned long start, unsigned long end)
 146{
 147        if (tlb_ops_need_broadcast()) {
 148                struct tlb_args ta;
 149                ta.ta_vma = vma;
 150                ta.ta_start = start;
 151                ta.ta_end = end;
 152                on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_tlb_range,
 153                                        &ta, 1);
 154        } else
 155                local_flush_tlb_range(vma, start, end);
 156        broadcast_tlb_mm_a15_erratum(vma->vm_mm);
 157}
 158
 159void flush_tlb_kernel_range(unsigned long start, unsigned long end)
 160{
 161        if (tlb_ops_need_broadcast()) {
 162                struct tlb_args ta;
 163                ta.ta_start = start;
 164                ta.ta_end = end;
 165                on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1);
 166        } else
 167                local_flush_tlb_kernel_range(start, end);
 168        broadcast_tlb_a15_erratum();
 169}
 170
 171void flush_bp_all(void)
 172{
 173        if (tlb_ops_need_broadcast())
 174                on_each_cpu(ipi_flush_bp_all, NULL, 1);
 175        else
 176                __flush_bp_all();
 177}
 178