LXR linux/arch/arm/include/asm/tlbflush.h

   1/*
   2 *  arch/arm/include/asm/tlbflush.h
   3 *
   4 *  Copyright (C) 1999-2003 Russell King
   5 *
   6 * This program is free software; you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License version 2 as
   8 * published by the Free Software Foundation.
   9 */
  10#ifndef _ASMARM_TLBFLUSH_H
  11#define _ASMARM_TLBFLUSH_H
  12
  13#ifdef CONFIG_MMU
  14
  15#include <asm/glue.h>
  16
  17#define TLB_V4_U_PAGE   (1 << 1)
  18#define TLB_V4_D_PAGE   (1 << 2)
  19#define TLB_V4_I_PAGE   (1 << 3)
  20#define TLB_V6_U_PAGE   (1 << 4)
  21#define TLB_V6_D_PAGE   (1 << 5)
  22#define TLB_V6_I_PAGE   (1 << 6)
  23
  24#define TLB_V4_U_FULL   (1 << 9)
  25#define TLB_V4_D_FULL   (1 << 10)
  26#define TLB_V4_I_FULL   (1 << 11)
  27#define TLB_V6_U_FULL   (1 << 12)
  28#define TLB_V6_D_FULL   (1 << 13)
  29#define TLB_V6_I_FULL   (1 << 14)
  30
  31#define TLB_V6_U_ASID   (1 << 16)
  32#define TLB_V6_D_ASID   (1 << 17)
  33#define TLB_V6_I_ASID   (1 << 18)
  34
  35#define TLB_V6_BP       (1 << 19)
  36
  37/* Unified Inner Shareable TLB operations (ARMv7 MP extensions) */
  38#define TLB_V7_UIS_PAGE (1 << 20)
  39#define TLB_V7_UIS_FULL (1 << 21)
  40#define TLB_V7_UIS_ASID (1 << 22)
  41#define TLB_V7_UIS_BP   (1 << 23)
  42
  43#define TLB_BARRIER     (1 << 28)
  44#define TLB_L2CLEAN_FR  (1 << 29)               /* Feroceon */
  45#define TLB_DCLEAN      (1 << 30)
  46#define TLB_WB          (1 << 31)
  47
  48/*
  49 *      MMU TLB Model
  50 *      =============
  51 *
  52 *      We have the following to choose from:
  53 *        v4    - ARMv4 without write buffer
  54 *        v4wb  - ARMv4 with write buffer without I TLB flush entry instruction
  55 *        v4wbi - ARMv4 with write buffer with I TLB flush entry instruction
  56 *        fr    - Feroceon (v4wbi with non-outer-cacheable page table walks)
  57 *        fa    - Faraday (v4 with write buffer with UTLB)
  58 *        v6wbi - ARMv6 with write buffer with I TLB flush entry instruction
  59 *        v7wbi - identical to v6wbi
  60 */
  61#undef _TLB
  62#undef MULTI_TLB
  63
  64#ifdef CONFIG_SMP_ON_UP
  65#define MULTI_TLB 1
  66#endif
  67
  68#define v4_tlb_flags    (TLB_V4_U_FULL | TLB_V4_U_PAGE)
  69
  70#ifdef CONFIG_CPU_TLB_V4WT
  71# define v4_possible_flags      v4_tlb_flags
  72# define v4_always_flags        v4_tlb_flags
  73# ifdef _TLB
  74#  define MULTI_TLB 1
  75# else
  76#  define _TLB v4
  77# endif
  78#else
  79# define v4_possible_flags      0
  80# define v4_always_flags        (-1UL)
  81#endif
  82
  83#define fa_tlb_flags    (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \
  84                         TLB_V4_U_FULL | TLB_V4_U_PAGE)
  85
  86#ifdef CONFIG_CPU_TLB_FA
  87# define fa_possible_flags      fa_tlb_flags
  88# define fa_always_flags        fa_tlb_flags
  89# ifdef _TLB
  90#  define MULTI_TLB 1
  91# else
  92#  define _TLB fa
  93# endif
  94#else
  95# define fa_possible_flags      0
  96# define fa_always_flags        (-1UL)
  97#endif
  98
  99#define v4wbi_tlb_flags (TLB_WB | TLB_DCLEAN | \
 100                         TLB_V4_I_FULL | TLB_V4_D_FULL | \
 101                         TLB_V4_I_PAGE | TLB_V4_D_PAGE)
 102
 103#ifdef CONFIG_CPU_TLB_V4WBI
 104# define v4wbi_possible_flags   v4wbi_tlb_flags
 105# define v4wbi_always_flags     v4wbi_tlb_flags
 106# ifdef _TLB
 107#  define MULTI_TLB 1
 108# else
 109#  define _TLB v4wbi
 110# endif
 111#else
 112# define v4wbi_possible_flags   0
 113# define v4wbi_always_flags     (-1UL)
 114#endif
 115
 116#define fr_tlb_flags    (TLB_WB | TLB_DCLEAN | TLB_L2CLEAN_FR | \
 117                         TLB_V4_I_FULL | TLB_V4_D_FULL | \
 118                         TLB_V4_I_PAGE | TLB_V4_D_PAGE)
 119
 120#ifdef CONFIG_CPU_TLB_FEROCEON
 121# define fr_possible_flags      fr_tlb_flags
 122# define fr_always_flags        fr_tlb_flags
 123# ifdef _TLB
 124#  define MULTI_TLB 1
 125# else
 126#  define _TLB v4wbi
 127# endif
 128#else
 129# define fr_possible_flags      0
 130# define fr_always_flags        (-1UL)
 131#endif
 132
 133#define v4wb_tlb_flags  (TLB_WB | TLB_DCLEAN | \
 134                         TLB_V4_I_FULL | TLB_V4_D_FULL | \
 135                         TLB_V4_D_PAGE)
 136
 137#ifdef CONFIG_CPU_TLB_V4WB
 138# define v4wb_possible_flags    v4wb_tlb_flags
 139# define v4wb_always_flags      v4wb_tlb_flags
 140# ifdef _TLB
 141#  define MULTI_TLB 1
 142# else
 143#  define _TLB v4wb
 144# endif
 145#else
 146# define v4wb_possible_flags    0
 147# define v4wb_always_flags      (-1UL)
 148#endif
 149
 150#define v6wbi_tlb_flags (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \
 151                         TLB_V6_I_FULL | TLB_V6_D_FULL | \
 152                         TLB_V6_I_PAGE | TLB_V6_D_PAGE | \
 153                         TLB_V6_I_ASID | TLB_V6_D_ASID | \
 154                         TLB_V6_BP)
 155
 156#ifdef CONFIG_CPU_TLB_V6
 157# define v6wbi_possible_flags   v6wbi_tlb_flags
 158# define v6wbi_always_flags     v6wbi_tlb_flags
 159# ifdef _TLB
 160#  define MULTI_TLB 1
 161# else
 162#  define _TLB v6wbi
 163# endif
 164#else
 165# define v6wbi_possible_flags   0
 166# define v6wbi_always_flags     (-1UL)
 167#endif
 168
 169#define v7wbi_tlb_flags_smp     (TLB_WB | TLB_BARRIER | \
 170                                 TLB_V7_UIS_FULL | TLB_V7_UIS_PAGE | \
 171                                 TLB_V7_UIS_ASID | TLB_V7_UIS_BP)
 172#define v7wbi_tlb_flags_up      (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \
 173                                 TLB_V6_U_FULL | TLB_V6_U_PAGE | \
 174                                 TLB_V6_U_ASID | TLB_V6_BP)
 175
 176#ifdef CONFIG_CPU_TLB_V7
 177
 178# ifdef CONFIG_SMP_ON_UP
 179#  define v7wbi_possible_flags  (v7wbi_tlb_flags_smp | v7wbi_tlb_flags_up)
 180#  define v7wbi_always_flags    (v7wbi_tlb_flags_smp & v7wbi_tlb_flags_up)
 181# elif defined(CONFIG_SMP)
 182#  define v7wbi_possible_flags  v7wbi_tlb_flags_smp
 183#  define v7wbi_always_flags    v7wbi_tlb_flags_smp
 184# else
 185#  define v7wbi_possible_flags  v7wbi_tlb_flags_up
 186#  define v7wbi_always_flags    v7wbi_tlb_flags_up
 187# endif
 188# ifdef _TLB
 189#  define MULTI_TLB 1
 190# else
 191#  define _TLB v7wbi
 192# endif
 193#else
 194# define v7wbi_possible_flags   0
 195# define v7wbi_always_flags     (-1UL)
 196#endif
 197
 198#ifndef _TLB
 199#error Unknown TLB model
 200#endif
 201
 202#ifndef __ASSEMBLY__
 203
 204#include <linux/sched.h>
 205
 206struct cpu_tlb_fns {
 207        void (*flush_user_range)(unsigned long, unsigned long, struct vm_area_struct *);
 208        void (*flush_kern_range)(unsigned long, unsigned long);
 209        unsigned long tlb_flags;
 210};
 211
 212/*
 213 * Select the calling method
 214 */
 215#ifdef MULTI_TLB
 216
 217#define __cpu_flush_user_tlb_range      cpu_tlb.flush_user_range
 218#define __cpu_flush_kern_tlb_range      cpu_tlb.flush_kern_range
 219
 220#else
 221
 222#define __cpu_flush_user_tlb_range      __glue(_TLB,_flush_user_tlb_range)
 223#define __cpu_flush_kern_tlb_range      __glue(_TLB,_flush_kern_tlb_range)
 224
 225extern void __cpu_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *);
 226extern void __cpu_flush_kern_tlb_range(unsigned long, unsigned long);
 227
 228#endif
 229
 230extern struct cpu_tlb_fns cpu_tlb;
 231
 232#define __cpu_tlb_flags                 cpu_tlb.tlb_flags
 233
 234/*
 235 *      TLB Management
 236 *      ==============
 237 *
 238 *      The arch/arm/mm/tlb-*.S files implement these methods.
 239 *
 240 *      The TLB specific code is expected to perform whatever tests it
 241 *      needs to determine if it should invalidate the TLB for each
 242 *      call.  Start addresses are inclusive and end addresses are
 243 *      exclusive; it is safe to round these addresses down.
 244 *
 245 *      flush_tlb_all()
 246 *
 247 *              Invalidate the entire TLB.
 248 *
 249 *      flush_tlb_mm(mm)
 250 *
 251 *              Invalidate all TLB entries in a particular address
 252 *              space.
 253 *              - mm    - mm_struct describing address space
 254 *
 255 *      flush_tlb_range(mm,start,end)
 256 *
 257 *              Invalidate a range of TLB entries in the specified
 258 *              address space.
 259 *              - mm    - mm_struct describing address space
 260 *              - start - start address (may not be aligned)
 261 *              - end   - end address (exclusive, may not be aligned)
 262 *
 263 *      flush_tlb_page(vaddr,vma)
 264 *
 265 *              Invalidate the specified page in the specified address range.
 266 *              - vaddr - virtual address (may not be aligned)
 267 *              - vma   - vma_struct describing address range
 268 *
 269 *      flush_kern_tlb_page(kaddr)
 270 *
 271 *              Invalidate the TLB entry for the specified page.  The address
 272 *              will be in the kernels virtual memory space.  Current uses
 273 *              only require the D-TLB to be invalidated.
 274 *              - kaddr - Kernel virtual memory address
 275 */
 276
 277/*
 278 * We optimise the code below by:
 279 *  - building a set of TLB flags that might be set in __cpu_tlb_flags
 280 *  - building a set of TLB flags that will always be set in __cpu_tlb_flags
 281 *  - if we're going to need __cpu_tlb_flags, access it once and only once
 282 *
 283 * This allows us to build optimal assembly for the single-CPU type case,
 284 * and as close to optimal given the compiler constrants for multi-CPU
 285 * case.  We could do better for the multi-CPU case if the compiler
 286 * implemented the "%?" method, but this has been discontinued due to too
 287 * many people getting it wrong.
 288 */
 289#define possible_tlb_flags      (v4_possible_flags | \
 290                                 v4wbi_possible_flags | \
 291                                 fr_possible_flags | \
 292                                 v4wb_possible_flags | \
 293                                 fa_possible_flags | \
 294                                 v6wbi_possible_flags | \
 295                                 v7wbi_possible_flags)
 296
 297#define always_tlb_flags        (v4_always_flags & \
 298                                 v4wbi_always_flags & \
 299                                 fr_always_flags & \
 300                                 v4wb_always_flags & \
 301                                 fa_always_flags & \
 302                                 v6wbi_always_flags & \
 303                                 v7wbi_always_flags)
 304
 305#define tlb_flag(f)     ((always_tlb_flags & (f)) || (__tlb_flag & possible_tlb_flags & (f)))
 306
 307#define __tlb_op(f, insnarg, arg)                                       \
 308        do {                                                            \
 309                if (always_tlb_flags & (f))                             \
 310                        asm("mcr " insnarg                              \
 311                            : : "r" (arg) : "cc");                      \
 312                else if (possible_tlb_flags & (f))                      \
 313                        asm("tst %1, %2\n\t"                            \
 314                            "mcrne " insnarg                            \
 315                            : : "r" (arg), "r" (__tlb_flag), "Ir" (f)   \
 316                            : "cc");                                    \
 317        } while (0)
 318
 319#define tlb_op(f, regs, arg)    __tlb_op(f, "p15, 0, %0, " regs, arg)
 320#define tlb_l2_op(f, regs, arg) __tlb_op(f, "p15, 1, %0, " regs, arg)
 321
 322static inline void __local_flush_tlb_all(void)
 323{
 324        const int zero = 0;
 325        const unsigned int __tlb_flag = __cpu_tlb_flags;
 326
 327        tlb_op(TLB_V4_U_FULL | TLB_V6_U_FULL, "c8, c7, 0", zero);
 328        tlb_op(TLB_V4_D_FULL | TLB_V6_D_FULL, "c8, c6, 0", zero);
 329        tlb_op(TLB_V4_I_FULL | TLB_V6_I_FULL, "c8, c5, 0", zero);
 330}
 331
 332static inline void local_flush_tlb_all(void)
 333{
 334        const int zero = 0;
 335        const unsigned int __tlb_flag = __cpu_tlb_flags;
 336
 337        if (tlb_flag(TLB_WB))
 338                dsb(nshst);
 339
 340        __local_flush_tlb_all();
 341        tlb_op(TLB_V7_UIS_FULL, "c8, c7, 0", zero);
 342
 343        if (tlb_flag(TLB_BARRIER)) {
 344                dsb(nsh);
 345                isb();
 346        }
 347}
 348
 349static inline void __flush_tlb_all(void)
 350{
 351        const int zero = 0;
 352        const unsigned int __tlb_flag = __cpu_tlb_flags;
 353
 354        if (tlb_flag(TLB_WB))
 355                dsb(ishst);
 356
 357        __local_flush_tlb_all();
 358        tlb_op(TLB_V7_UIS_FULL, "c8, c3, 0", zero);
 359
 360        if (tlb_flag(TLB_BARRIER)) {
 361                dsb(ish);
 362                isb();
 363        }
 364}
 365
 366static inline void __local_flush_tlb_mm(struct mm_struct *mm)
 367{
 368        const int zero = 0;
 369        const int asid = ASID(mm);
 370        const unsigned int __tlb_flag = __cpu_tlb_flags;
 371
 372        if (possible_tlb_flags & (TLB_V4_U_FULL|TLB_V4_D_FULL|TLB_V4_I_FULL)) {
 373                if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(mm))) {
 374                        tlb_op(TLB_V4_U_FULL, "c8, c7, 0", zero);
 375                        tlb_op(TLB_V4_D_FULL, "c8, c6, 0", zero);
 376                        tlb_op(TLB_V4_I_FULL, "c8, c5, 0", zero);
 377                }
 378        }
 379
 380        tlb_op(TLB_V6_U_ASID, "c8, c7, 2", asid);
 381        tlb_op(TLB_V6_D_ASID, "c8, c6, 2", asid);
 382        tlb_op(TLB_V6_I_ASID, "c8, c5, 2", asid);
 383}
 384
 385static inline void local_flush_tlb_mm(struct mm_struct *mm)
 386{
 387        const int asid = ASID(mm);
 388        const unsigned int __tlb_flag = __cpu_tlb_flags;
 389
 390        if (tlb_flag(TLB_WB))
 391                dsb(nshst);
 392
 393        __local_flush_tlb_mm(mm);
 394        tlb_op(TLB_V7_UIS_ASID, "c8, c7, 2", asid);
 395
 396        if (tlb_flag(TLB_BARRIER))
 397                dsb(nsh);
 398}
 399
 400static inline void __flush_tlb_mm(struct mm_struct *mm)
 401{
 402        const unsigned int __tlb_flag = __cpu_tlb_flags;
 403
 404        if (tlb_flag(TLB_WB))
 405                dsb(ishst);
 406
 407        __local_flush_tlb_mm(mm);
 408#ifdef CONFIG_ARM_ERRATA_720789
 409        tlb_op(TLB_V7_UIS_ASID, "c8, c3, 0", 0);
 410#else
 411        tlb_op(TLB_V7_UIS_ASID, "c8, c3, 2", ASID(mm));
 412#endif
 413
 414        if (tlb_flag(TLB_BARRIER))
 415                dsb(ish);
 416}
 417
 418static inline void
 419__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
 420{
 421        const int zero = 0;
 422        const unsigned int __tlb_flag = __cpu_tlb_flags;
 423
 424        uaddr = (uaddr & PAGE_MASK) | ASID(vma->vm_mm);
 425
 426        if (possible_tlb_flags & (TLB_V4_U_PAGE|TLB_V4_D_PAGE|TLB_V4_I_PAGE|TLB_V4_I_FULL) &&
 427            cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm))) {
 428                tlb_op(TLB_V4_U_PAGE, "c8, c7, 1", uaddr);
 429                tlb_op(TLB_V4_D_PAGE, "c8, c6, 1", uaddr);
 430                tlb_op(TLB_V4_I_PAGE, "c8, c5, 1", uaddr);
 431                if (!tlb_flag(TLB_V4_I_PAGE) && tlb_flag(TLB_V4_I_FULL))
 432                        asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero) : "cc");
 433        }
 434
 435        tlb_op(TLB_V6_U_PAGE, "c8, c7, 1", uaddr);
 436        tlb_op(TLB_V6_D_PAGE, "c8, c6, 1", uaddr);
 437        tlb_op(TLB_V6_I_PAGE, "c8, c5, 1", uaddr);
 438}
 439
 440static inline void
 441local_flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
 442{
 443        const unsigned int __tlb_flag = __cpu_tlb_flags;
 444
 445        uaddr = (uaddr & PAGE_MASK) | ASID(vma->vm_mm);
 446
 447        if (tlb_flag(TLB_WB))
 448                dsb(nshst);
 449
 450        __local_flush_tlb_page(vma, uaddr);
 451        tlb_op(TLB_V7_UIS_PAGE, "c8, c7, 1", uaddr);
 452
 453        if (tlb_flag(TLB_BARRIER))
 454                dsb(nsh);
 455}
 456
 457static inline void
 458__flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
 459{
 460        const unsigned int __tlb_flag = __cpu_tlb_flags;
 461
 462        uaddr = (uaddr & PAGE_MASK) | ASID(vma->vm_mm);
 463
 464        if (tlb_flag(TLB_WB))
 465                dsb(ishst);
 466
 467        __local_flush_tlb_page(vma, uaddr);
 468#ifdef CONFIG_ARM_ERRATA_720789
 469        tlb_op(TLB_V7_UIS_PAGE, "c8, c3, 3", uaddr & PAGE_MASK);
 470#else
 471        tlb_op(TLB_V7_UIS_PAGE, "c8, c3, 1", uaddr);
 472#endif
 473
 474        if (tlb_flag(TLB_BARRIER))
 475                dsb(ish);
 476}
 477
 478static inline void __local_flush_tlb_kernel_page(unsigned long kaddr)
 479{
 480        const int zero = 0;
 481        const unsigned int __tlb_flag = __cpu_tlb_flags;
 482
 483        tlb_op(TLB_V4_U_PAGE, "c8, c7, 1", kaddr);
 484        tlb_op(TLB_V4_D_PAGE, "c8, c6, 1", kaddr);
 485        tlb_op(TLB_V4_I_PAGE, "c8, c5, 1", kaddr);
 486        if (!tlb_flag(TLB_V4_I_PAGE) && tlb_flag(TLB_V4_I_FULL))
 487                asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero) : "cc");
 488
 489        tlb_op(TLB_V6_U_PAGE, "c8, c7, 1", kaddr);
 490        tlb_op(TLB_V6_D_PAGE, "c8, c6, 1", kaddr);
 491        tlb_op(TLB_V6_I_PAGE, "c8, c5, 1", kaddr);
 492}
 493
 494static inline void local_flush_tlb_kernel_page(unsigned long kaddr)
 495{
 496        const unsigned int __tlb_flag = __cpu_tlb_flags;
 497
 498        kaddr &= PAGE_MASK;
 499
 500        if (tlb_flag(TLB_WB))
 501                dsb(nshst);
 502
 503        __local_flush_tlb_kernel_page(kaddr);
 504        tlb_op(TLB_V7_UIS_PAGE, "c8, c7, 1", kaddr);
 505
 506        if (tlb_flag(TLB_BARRIER)) {
 507                dsb(nsh);
 508                isb();
 509        }
 510}
 511
 512static inline void __flush_tlb_kernel_page(unsigned long kaddr)
 513{
 514        const unsigned int __tlb_flag = __cpu_tlb_flags;
 515
 516        kaddr &= PAGE_MASK;
 517
 518        if (tlb_flag(TLB_WB))
 519                dsb(ishst);
 520
 521        __local_flush_tlb_kernel_page(kaddr);
 522        tlb_op(TLB_V7_UIS_PAGE, "c8, c3, 1", kaddr);
 523
 524        if (tlb_flag(TLB_BARRIER)) {
 525                dsb(ish);
 526                isb();
 527        }
 528}
 529
 530/*
 531 * Branch predictor maintenance is paired with full TLB invalidation, so
 532 * there is no need for any barriers here.
 533 */
 534static inline void __local_flush_bp_all(void)
 535{
 536        const int zero = 0;
 537        const unsigned int __tlb_flag = __cpu_tlb_flags;
 538
 539        if (tlb_flag(TLB_V6_BP))
 540                asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero));
 541}
 542
 543static inline void local_flush_bp_all(void)
 544{
 545        const int zero = 0;
 546        const unsigned int __tlb_flag = __cpu_tlb_flags;
 547
 548        __local_flush_bp_all();
 549        if (tlb_flag(TLB_V7_UIS_BP))
 550                asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero));
 551}
 552
 553static inline void __flush_bp_all(void)
 554{
 555        const int zero = 0;
 556        const unsigned int __tlb_flag = __cpu_tlb_flags;
 557
 558        __local_flush_bp_all();
 559        if (tlb_flag(TLB_V7_UIS_BP))
 560                asm("mcr p15, 0, %0, c7, c1, 6" : : "r" (zero));
 561}
 562
 563#include <asm/cputype.h>
 564#ifdef CONFIG_ARM_ERRATA_798181
 565static inline int erratum_a15_798181(void)
 566{
 567        unsigned int midr = read_cpuid_id();
 568
 569        /* Cortex-A15 r0p0..r3p2 affected */
 570        if ((midr & 0xff0ffff0) != 0x410fc0f0 || midr > 0x413fc0f2)
 571                return 0;
 572        return 1;
 573}
 574
 575static inline void dummy_flush_tlb_a15_erratum(void)
 576{
 577        /*
 578         * Dummy TLBIMVAIS. Using the unmapped address 0 and ASID 0.
 579         */
 580        asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (0));
 581        dsb(ish);
 582}
 583#else
 584static inline int erratum_a15_798181(void)
 585{
 586        return 0;
 587}
 588
 589static inline void dummy_flush_tlb_a15_erratum(void)
 590{
 591}
 592#endif
 593
 594/*
 595 *      flush_pmd_entry
 596 *
 597 *      Flush a PMD entry (word aligned, or double-word aligned) to
 598 *      RAM if the TLB for the CPU we are running on requires this.
 599 *      This is typically used when we are creating PMD entries.
 600 *
 601 *      clean_pmd_entry
 602 *
 603 *      Clean (but don't drain the write buffer) if the CPU requires
 604 *      these operations.  This is typically used when we are removing
 605 *      PMD entries.
 606 */
 607static inline void flush_pmd_entry(void *pmd)
 608{
 609        const unsigned int __tlb_flag = __cpu_tlb_flags;
 610
 611        tlb_op(TLB_DCLEAN, "c7, c10, 1  @ flush_pmd", pmd);
 612        tlb_l2_op(TLB_L2CLEAN_FR, "c15, c9, 1  @ L2 flush_pmd", pmd);
 613
 614        if (tlb_flag(TLB_WB))
 615                dsb(ishst);
 616}
 617
 618static inline void clean_pmd_entry(void *pmd)
 619{
 620        const unsigned int __tlb_flag = __cpu_tlb_flags;
 621
 622        tlb_op(TLB_DCLEAN, "c7, c10, 1  @ flush_pmd", pmd);
 623        tlb_l2_op(TLB_L2CLEAN_FR, "c15, c9, 1  @ L2 flush_pmd", pmd);
 624}
 625
 626#undef tlb_op
 627#undef tlb_flag
 628#undef always_tlb_flags
 629#undef possible_tlb_flags
 630
 631/*
 632 * Convert calls to our calling convention.
 633 */
 634#define local_flush_tlb_range(vma,start,end)    __cpu_flush_user_tlb_range(start,end,vma)
 635#define local_flush_tlb_kernel_range(s,e)       __cpu_flush_kern_tlb_range(s,e)
 636
 637#ifndef CONFIG_SMP
 638#define flush_tlb_all           local_flush_tlb_all
 639#define flush_tlb_mm            local_flush_tlb_mm
 640#define flush_tlb_page          local_flush_tlb_page
 641#define flush_tlb_kernel_page   local_flush_tlb_kernel_page
 642#define flush_tlb_range         local_flush_tlb_range
 643#define flush_tlb_kernel_range  local_flush_tlb_kernel_range
 644#define flush_bp_all            local_flush_bp_all
 645#else
 646extern void flush_tlb_all(void);
 647extern void flush_tlb_mm(struct mm_struct *mm);
 648extern void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr);
 649extern void flush_tlb_kernel_page(unsigned long kaddr);
 650extern void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
 651extern void flush_tlb_kernel_range(unsigned long start, unsigned long end);
 652extern void flush_bp_all(void);
 653#endif
 654
 655/*
 656 * If PG_dcache_clean is not set for the page, we need to ensure that any
 657 * cache entries for the kernels virtual memory range are written
 658 * back to the page. On ARMv6 and later, the cache coherency is handled via
 659 * the set_pte_at() function.
 660 */
 661#if __LINUX_ARM_ARCH__ < 6
 662extern void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr,
 663        pte_t *ptep);
 664#else
 665static inline void update_mmu_cache(struct vm_area_struct *vma,
 666                                    unsigned long addr, pte_t *ptep)
 667{
 668}
 669#endif
 670
 671#define update_mmu_cache_pmd(vma, address, pmd) do { } while (0)
 672
 673#endif
 674
 675#elif defined(CONFIG_SMP)       /* !CONFIG_MMU */
 676
 677#ifndef __ASSEMBLY__
 678
 679#include <linux/mm_types.h>
 680
 681static inline void local_flush_tlb_all(void)                                                                    { }
 682static inline void local_flush_tlb_mm(struct mm_struct *mm)                                                     { }
 683static inline void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)                        { }
 684static inline void local_flush_tlb_kernel_page(unsigned long kaddr)                                             { }
 685static inline void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)    { }
 686static inline void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)                         { }
 687static inline void local_flush_bp_all(void)                                                                     { }
 688
 689extern void flush_tlb_all(void);
 690extern void flush_tlb_mm(struct mm_struct *mm);
 691extern void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr);
 692extern void flush_tlb_kernel_page(unsigned long kaddr);
 693extern void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
 694extern void flush_tlb_kernel_range(unsigned long start, unsigned long end);
 695extern void flush_bp_all(void);
 696#endif  /* __ASSEMBLY__ */
 697
 698#endif
 699
 700#endif
 701