linux/arch/arc/mm/tlb.c
<<
>>
Prefs
   1/*
   2 * TLB Management (flush/create/diagnostics) for ARC700
   3 *
   4 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
   5 *
   6 * This program is free software; you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License version 2 as
   8 * published by the Free Software Foundation.
   9 *
  10 * vineetg: Aug 2011
  11 *  -Reintroduce duplicate PD fixup - some customer chips still have the issue
  12 *
  13 * vineetg: May 2011
  14 *  -No need to flush_cache_page( ) for each call to update_mmu_cache()
  15 *   some of the LMBench tests improved amazingly
  16 *      = page-fault thrice as fast (75 usec to 28 usec)
  17 *      = mmap twice as fast (9.6 msec to 4.6 msec),
  18 *      = fork (5.3 msec to 3.7 msec)
  19 *
  20 * vineetg: April 2011 :
  21 *  -MMU v3: PD{0,1} bits layout changed: They don't overlap anymore,
  22 *      helps avoid a shift when preparing PD0 from PTE
  23 *
  24 * vineetg: April 2011 : Preparing for MMU V3
  25 *  -MMU v2/v3 BCRs decoded differently
  26 *  -Remove TLB_SIZE hardcoding as it's variable now: 256 or 512
  27 *  -tlb_entry_erase( ) can be void
  28 *  -local_flush_tlb_range( ):
  29 *      = need not "ceil" @end
  30 *      = walks MMU only if range spans < 32 entries, as opposed to 256
  31 *
  32 * Vineetg: Sept 10th 2008
  33 *  -Changes related to MMU v2 (Rel 4.8)
  34 *
  35 * Vineetg: Aug 29th 2008
  36 *  -In TLB Flush operations (Metal Fix MMU) there is a explict command to
  37 *    flush Micro-TLBS. If TLB Index Reg is invalid prior to TLBIVUTLB cmd,
  38 *    it fails. Thus need to load it with ANY valid value before invoking
  39 *    TLBIVUTLB cmd
  40 *
  41 * Vineetg: Aug 21th 2008:
  42 *  -Reduced the duration of IRQ lockouts in TLB Flush routines
  43 *  -Multiple copies of TLB erase code seperated into a "single" function
  44 *  -In TLB Flush routines, interrupt disabling moved UP to retrieve ASID
  45 *       in interrupt-safe region.
  46 *
  47 * Vineetg: April 23rd Bug #93131
  48 *    Problem: tlb_flush_kernel_range() doesn't do anything if the range to
  49 *              flush is more than the size of TLB itself.
  50 *
  51 * Rahul Trivedi : Codito Technologies 2004
  52 */
  53
  54#include <linux/module.h>
  55#include <linux/bug.h>
  56#include <linux/mm_types.h>
  57
  58#include <asm/arcregs.h>
  59#include <asm/setup.h>
  60#include <asm/mmu_context.h>
  61#include <asm/mmu.h>
  62
  63/*                      Need for ARC MMU v2
  64 *
  65 * ARC700 MMU-v1 had a Joint-TLB for Code and Data and is 2 way set-assoc.
  66 * For a memcpy operation with 3 players (src/dst/code) such that all 3 pages
  67 * map into same set, there would be contention for the 2 ways causing severe
  68 * Thrashing.
  69 *
  70 * Although J-TLB is 2 way set assoc, ARC700 caches J-TLB into uTLBS which has
  71 * much higher associativity. u-D-TLB is 8 ways, u-I-TLB is 4 ways.
  72 * Given this, the thrasing problem should never happen because once the 3
  73 * J-TLB entries are created (even though 3rd will knock out one of the prev
  74 * two), the u-D-TLB and u-I-TLB will have what is required to accomplish memcpy
  75 *
  76 * Yet we still see the Thrashing because a J-TLB Write cause flush of u-TLBs.
  77 * This is a simple design for keeping them in sync. So what do we do?
  78 * The solution which James came up was pretty neat. It utilised the assoc
  79 * of uTLBs by not invalidating always but only when absolutely necessary.
  80 *
  81 * - Existing TLB commands work as before
  82 * - New command (TLBWriteNI) for TLB write without clearing uTLBs
  83 * - New command (TLBIVUTLB) to invalidate uTLBs.
  84 *
  85 * The uTLBs need only be invalidated when pages are being removed from the
  86 * OS page table. If a 'victim' TLB entry is being overwritten in the main TLB
  87 * as a result of a miss, the removed entry is still allowed to exist in the
  88 * uTLBs as it is still valid and present in the OS page table. This allows the
  89 * full associativity of the uTLBs to hide the limited associativity of the main
  90 * TLB.
  91 *
  92 * During a miss handler, the new "TLBWriteNI" command is used to load
  93 * entries without clearing the uTLBs.
  94 *
  95 * When the OS page table is updated, TLB entries that may be associated with a
  96 * removed page are removed (flushed) from the TLB using TLBWrite. In this
  97 * circumstance, the uTLBs must also be cleared. This is done by using the
  98 * existing TLBWrite command. An explicit IVUTLB is also required for those
  99 * corner cases when TLBWrite was not executed at all because the corresp
 100 * J-TLB entry got evicted/replaced.
 101 */
 102
 103
 104/* A copy of the ASID from the PID reg is kept in asid_cache */
 105DEFINE_PER_CPU(unsigned int, asid_cache) = MM_CTXT_FIRST_CYCLE;
 106
 107static int __read_mostly pae_exists;
 108
 109/*
 110 * Utility Routine to erase a J-TLB entry
 111 * Caller needs to setup Index Reg (manually or via getIndex)
 112 */
 113static inline void __tlb_entry_erase(void)
 114{
 115        write_aux_reg(ARC_REG_TLBPD1, 0);
 116
 117        if (is_pae40_enabled())
 118                write_aux_reg(ARC_REG_TLBPD1HI, 0);
 119
 120        write_aux_reg(ARC_REG_TLBPD0, 0);
 121        write_aux_reg(ARC_REG_TLBCOMMAND, TLBWrite);
 122}
 123
 124#if (CONFIG_ARC_MMU_VER < 4)
 125
 126static inline unsigned int tlb_entry_lkup(unsigned long vaddr_n_asid)
 127{
 128        unsigned int idx;
 129
 130        write_aux_reg(ARC_REG_TLBPD0, vaddr_n_asid);
 131
 132        write_aux_reg(ARC_REG_TLBCOMMAND, TLBProbe);
 133        idx = read_aux_reg(ARC_REG_TLBINDEX);
 134
 135        return idx;
 136}
 137
 138static void tlb_entry_erase(unsigned int vaddr_n_asid)
 139{
 140        unsigned int idx;
 141
 142        /* Locate the TLB entry for this vaddr + ASID */
 143        idx = tlb_entry_lkup(vaddr_n_asid);
 144
 145        /* No error means entry found, zero it out */
 146        if (likely(!(idx & TLB_LKUP_ERR))) {
 147                __tlb_entry_erase();
 148        } else {
 149                /* Duplicate entry error */
 150                WARN(idx == TLB_DUP_ERR, "Probe returned Dup PD for %x\n",
 151                                           vaddr_n_asid);
 152        }
 153}
 154
 155/****************************************************************************
 156 * ARC700 MMU caches recently used J-TLB entries (RAM) as uTLBs (FLOPs)
 157 *
 158 * New IVUTLB cmd in MMU v2 explictly invalidates the uTLB
 159 *
 160 * utlb_invalidate ( )
 161 *  -For v2 MMU calls Flush uTLB Cmd
 162 *  -For v1 MMU does nothing (except for Metal Fix v1 MMU)
 163 *      This is because in v1 TLBWrite itself invalidate uTLBs
 164 ***************************************************************************/
 165
 166static void utlb_invalidate(void)
 167{
 168#if (CONFIG_ARC_MMU_VER >= 2)
 169
 170#if (CONFIG_ARC_MMU_VER == 2)
 171        /* MMU v2 introduced the uTLB Flush command.
 172         * There was however an obscure hardware bug, where uTLB flush would
 173         * fail when a prior probe for J-TLB (both totally unrelated) would
 174         * return lkup err - because the entry didn't exist in MMU.
 175         * The Workround was to set Index reg with some valid value, prior to
 176         * flush. This was fixed in MMU v3 hence not needed any more
 177         */
 178        unsigned int idx;
 179
 180        /* make sure INDEX Reg is valid */
 181        idx = read_aux_reg(ARC_REG_TLBINDEX);
 182
 183        /* If not write some dummy val */
 184        if (unlikely(idx & TLB_LKUP_ERR))
 185                write_aux_reg(ARC_REG_TLBINDEX, 0xa);
 186#endif
 187
 188        write_aux_reg(ARC_REG_TLBCOMMAND, TLBIVUTLB);
 189#endif
 190
 191}
 192
 193static void tlb_entry_insert(unsigned int pd0, pte_t pd1)
 194{
 195        unsigned int idx;
 196
 197        /*
 198         * First verify if entry for this vaddr+ASID already exists
 199         * This also sets up PD0 (vaddr, ASID..) for final commit
 200         */
 201        idx = tlb_entry_lkup(pd0);
 202
 203        /*
 204         * If Not already present get a free slot from MMU.
 205         * Otherwise, Probe would have located the entry and set INDEX Reg
 206         * with existing location. This will cause Write CMD to over-write
 207         * existing entry with new PD0 and PD1
 208         */
 209        if (likely(idx & TLB_LKUP_ERR))
 210                write_aux_reg(ARC_REG_TLBCOMMAND, TLBGetIndex);
 211
 212        /* setup the other half of TLB entry (pfn, rwx..) */
 213        write_aux_reg(ARC_REG_TLBPD1, pd1);
 214
 215        /*
 216         * Commit the Entry to MMU
 217         * It doesn't sound safe to use the TLBWriteNI cmd here
 218         * which doesn't flush uTLBs. I'd rather be safe than sorry.
 219         */
 220        write_aux_reg(ARC_REG_TLBCOMMAND, TLBWrite);
 221}
 222
 223#else   /* CONFIG_ARC_MMU_VER >= 4) */
 224
 225static void utlb_invalidate(void)
 226{
 227        /* No need since uTLB is always in sync with JTLB */
 228}
 229
 230static void tlb_entry_erase(unsigned int vaddr_n_asid)
 231{
 232        write_aux_reg(ARC_REG_TLBPD0, vaddr_n_asid | _PAGE_PRESENT);
 233        write_aux_reg(ARC_REG_TLBCOMMAND, TLBDeleteEntry);
 234}
 235
 236static void tlb_entry_insert(unsigned int pd0, pte_t pd1)
 237{
 238        write_aux_reg(ARC_REG_TLBPD0, pd0);
 239        write_aux_reg(ARC_REG_TLBPD1, pd1);
 240
 241        if (is_pae40_enabled())
 242                write_aux_reg(ARC_REG_TLBPD1HI, (u64)pd1 >> 32);
 243
 244        write_aux_reg(ARC_REG_TLBCOMMAND, TLBInsertEntry);
 245}
 246
 247#endif
 248
 249/*
 250 * Un-conditionally (without lookup) erase the entire MMU contents
 251 */
 252
 253noinline void local_flush_tlb_all(void)
 254{
 255        struct cpuinfo_arc_mmu *mmu = &cpuinfo_arc700[smp_processor_id()].mmu;
 256        unsigned long flags;
 257        unsigned int entry;
 258        int num_tlb = mmu->sets * mmu->ways;
 259
 260        local_irq_save(flags);
 261
 262        /* Load PD0 and PD1 with template for a Blank Entry */
 263        write_aux_reg(ARC_REG_TLBPD1, 0);
 264
 265        if (is_pae40_enabled())
 266                write_aux_reg(ARC_REG_TLBPD1HI, 0);
 267
 268        write_aux_reg(ARC_REG_TLBPD0, 0);
 269
 270        for (entry = 0; entry < num_tlb; entry++) {
 271                /* write this entry to the TLB */
 272                write_aux_reg(ARC_REG_TLBINDEX, entry);
 273                write_aux_reg(ARC_REG_TLBCOMMAND, TLBWrite);
 274        }
 275
 276        if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
 277                const int stlb_idx = 0x800;
 278
 279                /* Blank sTLB entry */
 280                write_aux_reg(ARC_REG_TLBPD0, _PAGE_HW_SZ);
 281
 282                for (entry = stlb_idx; entry < stlb_idx + 16; entry++) {
 283                        write_aux_reg(ARC_REG_TLBINDEX, entry);
 284                        write_aux_reg(ARC_REG_TLBCOMMAND, TLBWrite);
 285                }
 286        }
 287
 288        utlb_invalidate();
 289
 290        local_irq_restore(flags);
 291}
 292
 293/*
 294 * Flush the entrie MM for userland. The fastest way is to move to Next ASID
 295 */
 296noinline void local_flush_tlb_mm(struct mm_struct *mm)
 297{
 298        /*
 299         * Small optimisation courtesy IA64
 300         * flush_mm called during fork,exit,munmap etc, multiple times as well.
 301         * Only for fork( ) do we need to move parent to a new MMU ctxt,
 302         * all other cases are NOPs, hence this check.
 303         */
 304        if (atomic_read(&mm->mm_users) == 0)
 305                return;
 306
 307        /*
 308         * - Move to a new ASID, but only if the mm is still wired in
 309         *   (Android Binder ended up calling this for vma->mm != tsk->mm,
 310         *    causing h/w - s/w ASID to get out of sync)
 311         * - Also get_new_mmu_context() new implementation allocates a new
 312         *   ASID only if it is not allocated already - so unallocate first
 313         */
 314        destroy_context(mm);
 315        if (current->mm == mm)
 316                get_new_mmu_context(mm);
 317}
 318
 319/*
 320 * Flush a Range of TLB entries for userland.
 321 * @start is inclusive, while @end is exclusive
 322 * Difference between this and Kernel Range Flush is
 323 *  -Here the fastest way (if range is too large) is to move to next ASID
 324 *      without doing any explicit Shootdown
 325 *  -In case of kernel Flush, entry has to be shot down explictly
 326 */
 327void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 328                           unsigned long end)
 329{
 330        const unsigned int cpu = smp_processor_id();
 331        unsigned long flags;
 332
 333        /* If range @start to @end is more than 32 TLB entries deep,
 334         * its better to move to a new ASID rather than searching for
 335         * individual entries and then shooting them down
 336         *
 337         * The calc above is rough, doesn't account for unaligned parts,
 338         * since this is heuristics based anyways
 339         */
 340        if (unlikely((end - start) >= PAGE_SIZE * 32)) {
 341                local_flush_tlb_mm(vma->vm_mm);
 342                return;
 343        }
 344
 345        /*
 346         * @start moved to page start: this alone suffices for checking
 347         * loop end condition below, w/o need for aligning @end to end
 348         * e.g. 2000 to 4001 will anyhow loop twice
 349         */
 350        start &= PAGE_MASK;
 351
 352        local_irq_save(flags);
 353
 354        if (asid_mm(vma->vm_mm, cpu) != MM_CTXT_NO_ASID) {
 355                while (start < end) {
 356                        tlb_entry_erase(start | hw_pid(vma->vm_mm, cpu));
 357                        start += PAGE_SIZE;
 358                }
 359        }
 360
 361        utlb_invalidate();
 362
 363        local_irq_restore(flags);
 364}
 365
 366/* Flush the kernel TLB entries - vmalloc/modules (Global from MMU perspective)
 367 *  @start, @end interpreted as kvaddr
 368 * Interestingly, shared TLB entries can also be flushed using just
 369 * @start,@end alone (interpreted as user vaddr), although technically SASID
 370 * is also needed. However our smart TLbProbe lookup takes care of that.
 371 */
 372void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
 373{
 374        unsigned long flags;
 375
 376        /* exactly same as above, except for TLB entry not taking ASID */
 377
 378        if (unlikely((end - start) >= PAGE_SIZE * 32)) {
 379                local_flush_tlb_all();
 380                return;
 381        }
 382
 383        start &= PAGE_MASK;
 384
 385        local_irq_save(flags);
 386        while (start < end) {
 387                tlb_entry_erase(start);
 388                start += PAGE_SIZE;
 389        }
 390
 391        utlb_invalidate();
 392
 393        local_irq_restore(flags);
 394}
 395
 396/*
 397 * Delete TLB entry in MMU for a given page (??? address)
 398 * NOTE One TLB entry contains translation for single PAGE
 399 */
 400
 401void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
 402{
 403        const unsigned int cpu = smp_processor_id();
 404        unsigned long flags;
 405
 406        /* Note that it is critical that interrupts are DISABLED between
 407         * checking the ASID and using it flush the TLB entry
 408         */
 409        local_irq_save(flags);
 410
 411        if (asid_mm(vma->vm_mm, cpu) != MM_CTXT_NO_ASID) {
 412                tlb_entry_erase((page & PAGE_MASK) | hw_pid(vma->vm_mm, cpu));
 413                utlb_invalidate();
 414        }
 415
 416        local_irq_restore(flags);
 417}
 418
 419#ifdef CONFIG_SMP
 420
 421struct tlb_args {
 422        struct vm_area_struct *ta_vma;
 423        unsigned long ta_start;
 424        unsigned long ta_end;
 425};
 426
 427static inline void ipi_flush_tlb_page(void *arg)
 428{
 429        struct tlb_args *ta = arg;
 430
 431        local_flush_tlb_page(ta->ta_vma, ta->ta_start);
 432}
 433
 434static inline void ipi_flush_tlb_range(void *arg)
 435{
 436        struct tlb_args *ta = arg;
 437
 438        local_flush_tlb_range(ta->ta_vma, ta->ta_start, ta->ta_end);
 439}
 440
 441#ifdef CONFIG_TRANSPARENT_HUGEPAGE
 442static inline void ipi_flush_pmd_tlb_range(void *arg)
 443{
 444        struct tlb_args *ta = arg;
 445
 446        local_flush_pmd_tlb_range(ta->ta_vma, ta->ta_start, ta->ta_end);
 447}
 448#endif
 449
 450static inline void ipi_flush_tlb_kernel_range(void *arg)
 451{
 452        struct tlb_args *ta = (struct tlb_args *)arg;
 453
 454        local_flush_tlb_kernel_range(ta->ta_start, ta->ta_end);
 455}
 456
 457void flush_tlb_all(void)
 458{
 459        on_each_cpu((smp_call_func_t)local_flush_tlb_all, NULL, 1);
 460}
 461
 462void flush_tlb_mm(struct mm_struct *mm)
 463{
 464        on_each_cpu_mask(mm_cpumask(mm), (smp_call_func_t)local_flush_tlb_mm,
 465                         mm, 1);
 466}
 467
 468void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
 469{
 470        struct tlb_args ta = {
 471                .ta_vma = vma,
 472                .ta_start = uaddr
 473        };
 474
 475        on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_tlb_page, &ta, 1);
 476}
 477
 478void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 479                     unsigned long end)
 480{
 481        struct tlb_args ta = {
 482                .ta_vma = vma,
 483                .ta_start = start,
 484                .ta_end = end
 485        };
 486
 487        on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_tlb_range, &ta, 1);
 488}
 489
 490#ifdef CONFIG_TRANSPARENT_HUGEPAGE
 491void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start,
 492                         unsigned long end)
 493{
 494        struct tlb_args ta = {
 495                .ta_vma = vma,
 496                .ta_start = start,
 497                .ta_end = end
 498        };
 499
 500        on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_pmd_tlb_range, &ta, 1);
 501}
 502#endif
 503
 504void flush_tlb_kernel_range(unsigned long start, unsigned long end)
 505{
 506        struct tlb_args ta = {
 507                .ta_start = start,
 508                .ta_end = end
 509        };
 510
 511        on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1);
 512}
 513#endif
 514
 515/*
 516 * Routine to create a TLB entry
 517 */
 518void create_tlb(struct vm_area_struct *vma, unsigned long vaddr, pte_t *ptep)
 519{
 520        unsigned long flags;
 521        unsigned int asid_or_sasid, rwx;
 522        unsigned long pd0;
 523        pte_t pd1;
 524
 525        /*
 526         * create_tlb() assumes that current->mm == vma->mm, since
 527         * -it ASID for TLB entry is fetched from MMU ASID reg (valid for curr)
 528         * -completes the lazy write to SASID reg (again valid for curr tsk)
 529         *
 530         * Removing the assumption involves
 531         * -Using vma->mm->context{ASID,SASID}, as opposed to MMU reg.
 532         * -Fix the TLB paranoid debug code to not trigger false negatives.
 533         * -More importantly it makes this handler inconsistent with fast-path
 534         *  TLB Refill handler which always deals with "current"
 535         *
 536         * Lets see the use cases when current->mm != vma->mm and we land here
 537         *  1. execve->copy_strings()->__get_user_pages->handle_mm_fault
 538         *     Here VM wants to pre-install a TLB entry for user stack while
 539         *     current->mm still points to pre-execve mm (hence the condition).
 540         *     However the stack vaddr is soon relocated (randomization) and
 541         *     move_page_tables() tries to undo that TLB entry.
 542         *     Thus not creating TLB entry is not any worse.
 543         *
 544         *  2. ptrace(POKETEXT) causes a CoW - debugger(current) inserting a
 545         *     breakpoint in debugged task. Not creating a TLB now is not
 546         *     performance critical.
 547         *
 548         * Both the cases above are not good enough for code churn.
 549         */
 550        if (current->active_mm != vma->vm_mm)
 551                return;
 552
 553        local_irq_save(flags);
 554
 555        tlb_paranoid_check(asid_mm(vma->vm_mm, smp_processor_id()), vaddr);
 556
 557        vaddr &= PAGE_MASK;
 558
 559        /* update this PTE credentials */
 560        pte_val(*ptep) |= (_PAGE_PRESENT | _PAGE_ACCESSED);
 561
 562        /* Create HW TLB(PD0,PD1) from PTE  */
 563
 564        /* ASID for this task */
 565        asid_or_sasid = read_aux_reg(ARC_REG_PID) & 0xff;
 566
 567        pd0 = vaddr | asid_or_sasid | (pte_val(*ptep) & PTE_BITS_IN_PD0);
 568
 569        /*
 570         * ARC MMU provides fully orthogonal access bits for K/U mode,
 571         * however Linux only saves 1 set to save PTE real-estate
 572         * Here we convert 3 PTE bits into 6 MMU bits:
 573         * -Kernel only entries have Kr Kw Kx 0 0 0
 574         * -User entries have mirrored K and U bits
 575         */
 576        rwx = pte_val(*ptep) & PTE_BITS_RWX;
 577
 578        if (pte_val(*ptep) & _PAGE_GLOBAL)
 579                rwx <<= 3;              /* r w x => Kr Kw Kx 0 0 0 */
 580        else
 581                rwx |= (rwx << 3);      /* r w x => Kr Kw Kx Ur Uw Ux */
 582
 583        pd1 = rwx | (pte_val(*ptep) & PTE_BITS_NON_RWX_IN_PD1);
 584
 585        tlb_entry_insert(pd0, pd1);
 586
 587        local_irq_restore(flags);
 588}
 589
 590/*
 591 * Called at the end of pagefault, for a userspace mapped page
 592 *  -pre-install the corresponding TLB entry into MMU
 593 *  -Finalize the delayed D-cache flush of kernel mapping of page due to
 594 *      flush_dcache_page(), copy_user_page()
 595 *
 596 * Note that flush (when done) involves both WBACK - so physical page is
 597 * in sync as well as INV - so any non-congruent aliases don't remain
 598 */
 599void update_mmu_cache(struct vm_area_struct *vma, unsigned long vaddr_unaligned,
 600                      pte_t *ptep)
 601{
 602        unsigned long vaddr = vaddr_unaligned & PAGE_MASK;
 603        phys_addr_t paddr = pte_val(*ptep) & PAGE_MASK;
 604        struct page *page = pfn_to_page(pte_pfn(*ptep));
 605
 606        create_tlb(vma, vaddr, ptep);
 607
 608        if (page == ZERO_PAGE(0)) {
 609                return;
 610        }
 611
 612        /*
 613         * Exec page : Independent of aliasing/page-color considerations,
 614         *             since icache doesn't snoop dcache on ARC, any dirty
 615         *             K-mapping of a code page needs to be wback+inv so that
 616         *             icache fetch by userspace sees code correctly.
 617         * !EXEC page: If K-mapping is NOT congruent to U-mapping, flush it
 618         *             so userspace sees the right data.
 619         *  (Avoids the flush for Non-exec + congruent mapping case)
 620         */
 621        if ((vma->vm_flags & VM_EXEC) ||
 622             addr_not_cache_congruent(paddr, vaddr)) {
 623
 624                int dirty = !test_and_set_bit(PG_dc_clean, &page->flags);
 625                if (dirty) {
 626                        /* wback + inv dcache lines (K-mapping) */
 627                        __flush_dcache_page(paddr, paddr);
 628
 629                        /* invalidate any existing icache lines (U-mapping) */
 630                        if (vma->vm_flags & VM_EXEC)
 631                                __inv_icache_page(paddr, vaddr);
 632                }
 633        }
 634}
 635
 636#ifdef CONFIG_TRANSPARENT_HUGEPAGE
 637
 638/*
 639 * MMUv4 in HS38x cores supports Super Pages which are basis for Linux THP
 640 * support.
 641 *
 642 * Normal and Super pages can co-exist (ofcourse not overlap) in TLB with a
 643 * new bit "SZ" in TLB page descriptor to distinguish between them.
 644 * Super Page size is configurable in hardware (4K to 16M), but fixed once
 645 * RTL builds.
 646 *
 647 * The exact THP size a Linx configuration will support is a function of:
 648 *  - MMU page size (typical 8K, RTL fixed)
 649 *  - software page walker address split between PGD:PTE:PFN (typical
 650 *    11:8:13, but can be changed with 1 line)
 651 * So for above default, THP size supported is 8K * (2^8) = 2M
 652 *
 653 * Default Page Walker is 2 levels, PGD:PTE:PFN, which in THP regime
 654 * reduces to 1 level (as PTE is folded into PGD and canonically referred
 655 * to as PMD).
 656 * Thus THP PMD accessors are implemented in terms of PTE (just like sparc)
 657 */
 658
 659void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr,
 660                                 pmd_t *pmd)
 661{
 662        pte_t pte = __pte(pmd_val(*pmd));
 663        update_mmu_cache(vma, addr, &pte);
 664}
 665
 666void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
 667                                pgtable_t pgtable)
 668{
 669        struct list_head *lh = (struct list_head *) pgtable;
 670
 671        assert_spin_locked(&mm->page_table_lock);
 672
 673        /* FIFO */
 674        if (!pmd_huge_pte(mm, pmdp))
 675                INIT_LIST_HEAD(lh);
 676        else
 677                list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));
 678        pmd_huge_pte(mm, pmdp) = pgtable;
 679}
 680
 681pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
 682{
 683        struct list_head *lh;
 684        pgtable_t pgtable;
 685
 686        assert_spin_locked(&mm->page_table_lock);
 687
 688        pgtable = pmd_huge_pte(mm, pmdp);
 689        lh = (struct list_head *) pgtable;
 690        if (list_empty(lh))
 691                pmd_huge_pte(mm, pmdp) = NULL;
 692        else {
 693                pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
 694                list_del(lh);
 695        }
 696
 697        pte_val(pgtable[0]) = 0;
 698        pte_val(pgtable[1]) = 0;
 699
 700        return pgtable;
 701}
 702
 703void local_flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start,
 704                               unsigned long end)
 705{
 706        unsigned int cpu;
 707        unsigned long flags;
 708
 709        local_irq_save(flags);
 710
 711        cpu = smp_processor_id();
 712
 713        if (likely(asid_mm(vma->vm_mm, cpu) != MM_CTXT_NO_ASID)) {
 714                unsigned int asid = hw_pid(vma->vm_mm, cpu);
 715
 716                /* No need to loop here: this will always be for 1 Huge Page */
 717                tlb_entry_erase(start | _PAGE_HW_SZ | asid);
 718        }
 719
 720        local_irq_restore(flags);
 721}
 722
 723#endif
 724
 725/* Read the Cache Build Confuration Registers, Decode them and save into
 726 * the cpuinfo structure for later use.
 727 * No Validation is done here, simply read/convert the BCRs
 728 */
 729void read_decode_mmu_bcr(void)
 730{
 731        struct cpuinfo_arc_mmu *mmu = &cpuinfo_arc700[smp_processor_id()].mmu;
 732        unsigned int tmp;
 733        struct bcr_mmu_1_2 {
 734#ifdef CONFIG_CPU_BIG_ENDIAN
 735                unsigned int ver:8, ways:4, sets:4, u_itlb:8, u_dtlb:8;
 736#else
 737                unsigned int u_dtlb:8, u_itlb:8, sets:4, ways:4, ver:8;
 738#endif
 739        } *mmu2;
 740
 741        struct bcr_mmu_3 {
 742#ifdef CONFIG_CPU_BIG_ENDIAN
 743        unsigned int ver:8, ways:4, sets:4, res:3, sasid:1, pg_sz:4,
 744                     u_itlb:4, u_dtlb:4;
 745#else
 746        unsigned int u_dtlb:4, u_itlb:4, pg_sz:4, sasid:1, res:3, sets:4,
 747                     ways:4, ver:8;
 748#endif
 749        } *mmu3;
 750
 751        struct bcr_mmu_4 {
 752#ifdef CONFIG_CPU_BIG_ENDIAN
 753        unsigned int ver:8, sasid:1, sz1:4, sz0:4, res:2, pae:1,
 754                     n_ways:2, n_entry:2, n_super:2, u_itlb:3, u_dtlb:3;
 755#else
 756        /*           DTLB      ITLB      JES        JE         JA      */
 757        unsigned int u_dtlb:3, u_itlb:3, n_super:2, n_entry:2, n_ways:2,
 758                     pae:1, res:2, sz0:4, sz1:4, sasid:1, ver:8;
 759#endif
 760        } *mmu4;
 761
 762        tmp = read_aux_reg(ARC_REG_MMU_BCR);
 763        mmu->ver = (tmp >> 24);
 764
 765        if (mmu->ver <= 2) {
 766                mmu2 = (struct bcr_mmu_1_2 *)&tmp;
 767                mmu->pg_sz_k = TO_KB(0x2000);
 768                mmu->sets = 1 << mmu2->sets;
 769                mmu->ways = 1 << mmu2->ways;
 770                mmu->u_dtlb = mmu2->u_dtlb;
 771                mmu->u_itlb = mmu2->u_itlb;
 772        } else if (mmu->ver == 3) {
 773                mmu3 = (struct bcr_mmu_3 *)&tmp;
 774                mmu->pg_sz_k = 1 << (mmu3->pg_sz - 1);
 775                mmu->sets = 1 << mmu3->sets;
 776                mmu->ways = 1 << mmu3->ways;
 777                mmu->u_dtlb = mmu3->u_dtlb;
 778                mmu->u_itlb = mmu3->u_itlb;
 779                mmu->sasid = mmu3->sasid;
 780        } else {
 781                mmu4 = (struct bcr_mmu_4 *)&tmp;
 782                mmu->pg_sz_k = 1 << (mmu4->sz0 - 1);
 783                mmu->s_pg_sz_m = 1 << (mmu4->sz1 - 11);
 784                mmu->sets = 64 << mmu4->n_entry;
 785                mmu->ways = mmu4->n_ways * 2;
 786                mmu->u_dtlb = mmu4->u_dtlb * 4;
 787                mmu->u_itlb = mmu4->u_itlb * 4;
 788                mmu->sasid = mmu4->sasid;
 789                pae_exists = mmu->pae = mmu4->pae;
 790        }
 791}
 792
 793char *arc_mmu_mumbojumbo(int cpu_id, char *buf, int len)
 794{
 795        int n = 0;
 796        struct cpuinfo_arc_mmu *p_mmu = &cpuinfo_arc700[cpu_id].mmu;
 797        char super_pg[64] = "";
 798
 799        if (p_mmu->s_pg_sz_m)
 800                scnprintf(super_pg, 64, "%dM Super Page %s",
 801                          p_mmu->s_pg_sz_m,
 802                          IS_USED_CFG(CONFIG_TRANSPARENT_HUGEPAGE));
 803
 804        n += scnprintf(buf + n, len - n,
 805                      "MMU [v%x]\t: %dk PAGE, %sJTLB %d (%dx%d), uDTLB %d, uITLB %d%s%s\n",
 806                       p_mmu->ver, p_mmu->pg_sz_k, super_pg,
 807                       p_mmu->sets * p_mmu->ways, p_mmu->sets, p_mmu->ways,
 808                       p_mmu->u_dtlb, p_mmu->u_itlb,
 809                       IS_AVAIL2(p_mmu->pae, ", PAE40 ", CONFIG_ARC_HAS_PAE40));
 810
 811        return buf;
 812}
 813
 814int pae40_exist_but_not_enab(void)
 815{
 816        return pae_exists && !is_pae40_enabled();
 817}
 818
 819void arc_mmu_init(void)
 820{
 821        char str[256];
 822        struct cpuinfo_arc_mmu *mmu = &cpuinfo_arc700[smp_processor_id()].mmu;
 823
 824        pr_info("%s", arc_mmu_mumbojumbo(0, str, sizeof(str)));
 825
 826        /*
 827         * Can't be done in processor.h due to header include depenedencies
 828         */
 829        BUILD_BUG_ON(!IS_ALIGNED((CONFIG_ARC_KVADDR_SIZE << 20), PMD_SIZE));
 830
 831        /*
 832         * stack top size sanity check,
 833         * Can't be done in processor.h due to header include depenedencies
 834         */
 835        BUILD_BUG_ON(!IS_ALIGNED(STACK_TOP, PMD_SIZE));
 836
 837        /* For efficiency sake, kernel is compile time built for a MMU ver
 838         * This must match the hardware it is running on.
 839         * Linux built for MMU V2, if run on MMU V1 will break down because V1
 840         *  hardware doesn't understand cmds such as WriteNI, or IVUTLB
 841         * On the other hand, Linux built for V1 if run on MMU V2 will do
 842         *   un-needed workarounds to prevent memcpy thrashing.
 843         * Similarly MMU V3 has new features which won't work on older MMU
 844         */
 845        if (mmu->ver != CONFIG_ARC_MMU_VER) {
 846                panic("MMU ver %d doesn't match kernel built for %d...\n",
 847                      mmu->ver, CONFIG_ARC_MMU_VER);
 848        }
 849
 850        if (mmu->pg_sz_k != TO_KB(PAGE_SIZE))
 851                panic("MMU pg size != PAGE_SIZE (%luk)\n", TO_KB(PAGE_SIZE));
 852
 853        if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
 854            mmu->s_pg_sz_m != TO_MB(HPAGE_PMD_SIZE))
 855                panic("MMU Super pg size != Linux HPAGE_PMD_SIZE (%luM)\n",
 856                      (unsigned long)TO_MB(HPAGE_PMD_SIZE));
 857
 858        if (IS_ENABLED(CONFIG_ARC_HAS_PAE40) && !mmu->pae)
 859                panic("Hardware doesn't support PAE40\n");
 860
 861        /* Enable the MMU */
 862        write_aux_reg(ARC_REG_PID, MMU_ENABLE);
 863
 864        /* In smp we use this reg for interrupt 1 scratch */
 865#ifndef CONFIG_SMP
 866        /* swapper_pg_dir is the pgd for the kernel, used by vmalloc */
 867        write_aux_reg(ARC_REG_SCRATCH_DATA0, swapper_pg_dir);
 868#endif
 869
 870        if (pae40_exist_but_not_enab())
 871                write_aux_reg(ARC_REG_TLBPD1HI, 0);
 872}
 873
 874/*
 875 * TLB Programmer's Model uses Linear Indexes: 0 to {255, 511} for 128 x {2,4}
 876 * The mapping is Column-first.
 877 *              ---------------------   -----------
 878 *              |way0|way1|way2|way3|   |way0|way1|
 879 *              ---------------------   -----------
 880 * [set0]       |  0 |  1 |  2 |  3 |   |  0 |  1 |
 881 * [set1]       |  4 |  5 |  6 |  7 |   |  2 |  3 |
 882 *              ~                   ~   ~         ~
 883 * [set127]     | 508| 509| 510| 511|   | 254| 255|
 884 *              ---------------------   -----------
 885 * For normal operations we don't(must not) care how above works since
 886 * MMU cmd getIndex(vaddr) abstracts that out.
 887 * However for walking WAYS of a SET, we need to know this
 888 */
 889#define SET_WAY_TO_IDX(mmu, set, way)  ((set) * mmu->ways + (way))
 890
 891/* Handling of Duplicate PD (TLB entry) in MMU.
 892 * -Could be due to buggy customer tapeouts or obscure kernel bugs
 893 * -MMU complaints not at the time of duplicate PD installation, but at the
 894 *      time of lookup matching multiple ways.
 895 * -Ideally these should never happen - but if they do - workaround by deleting
 896 *      the duplicate one.
 897 * -Knob to be verbose abt it.(TODO: hook them up to debugfs)
 898 */
 899volatile int dup_pd_silent; /* Be slient abt it or complain (default) */
 900
 901void do_tlb_overlap_fault(unsigned long cause, unsigned long address,
 902                          struct pt_regs *regs)
 903{
 904        struct cpuinfo_arc_mmu *mmu = &cpuinfo_arc700[smp_processor_id()].mmu;
 905        unsigned int pd0[mmu->ways];
 906        unsigned long flags;
 907        int set;
 908
 909        local_irq_save(flags);
 910
 911        /* loop thru all sets of TLB */
 912        for (set = 0; set < mmu->sets; set++) {
 913
 914                int is_valid, way;
 915
 916                /* read out all the ways of current set */
 917                for (way = 0, is_valid = 0; way < mmu->ways; way++) {
 918                        write_aux_reg(ARC_REG_TLBINDEX,
 919                                          SET_WAY_TO_IDX(mmu, set, way));
 920                        write_aux_reg(ARC_REG_TLBCOMMAND, TLBRead);
 921                        pd0[way] = read_aux_reg(ARC_REG_TLBPD0);
 922                        is_valid |= pd0[way] & _PAGE_PRESENT;
 923                        pd0[way] &= PAGE_MASK;
 924                }
 925
 926                /* If all the WAYS in SET are empty, skip to next SET */
 927                if (!is_valid)
 928                        continue;
 929
 930                /* Scan the set for duplicate ways: needs a nested loop */
 931                for (way = 0; way < mmu->ways - 1; way++) {
 932
 933                        int n;
 934
 935                        if (!pd0[way])
 936                                continue;
 937
 938                        for (n = way + 1; n < mmu->ways; n++) {
 939                                if (pd0[way] != pd0[n])
 940                                        continue;
 941
 942                                if (!dup_pd_silent)
 943                                        pr_info("Dup TLB PD0 %08x @ set %d ways %d,%d\n",
 944                                                pd0[way], set, way, n);
 945
 946                                /*
 947                                 * clear entry @way and not @n.
 948                                 * This is critical to our optimised loop
 949                                 */
 950                                pd0[way] = 0;
 951                                write_aux_reg(ARC_REG_TLBINDEX,
 952                                                SET_WAY_TO_IDX(mmu, set, way));
 953                                __tlb_entry_erase();
 954                        }
 955                }
 956        }
 957
 958        local_irq_restore(flags);
 959}
 960
 961/***********************************************************************
 962 * Diagnostic Routines
 963 *  -Called from Low Level TLB Hanlders if things don;t look good
 964 **********************************************************************/
 965
 966#ifdef CONFIG_ARC_DBG_TLB_PARANOIA
 967
 968/*
 969 * Low Level ASM TLB handler calls this if it finds that HW and SW ASIDS
 970 * don't match
 971 */
 972void print_asid_mismatch(int mm_asid, int mmu_asid, int is_fast_path)
 973{
 974        pr_emerg("ASID Mismatch in %s Path Handler: sw-pid=0x%x hw-pid=0x%x\n",
 975               is_fast_path ? "Fast" : "Slow", mm_asid, mmu_asid);
 976
 977        __asm__ __volatile__("flag 1");
 978}
 979
 980void tlb_paranoid_check(unsigned int mm_asid, unsigned long addr)
 981{
 982        unsigned int mmu_asid;
 983
 984        mmu_asid = read_aux_reg(ARC_REG_PID) & 0xff;
 985
 986        /*
 987         * At the time of a TLB miss/installation
 988         *   - HW version needs to match SW version
 989         *   - SW needs to have a valid ASID
 990         */
 991        if (addr < 0x70000000 &&
 992            ((mm_asid == MM_CTXT_NO_ASID) ||
 993              (mmu_asid != (mm_asid & MM_CTXT_ASID_MASK))))
 994                print_asid_mismatch(mm_asid, mmu_asid, 0);
 995}
 996#endif
 997