LXR linux/arch/nds32/mm/proc.c

   1// SPDX-License-Identifier: GPL-2.0
   2// Copyright (C) 2005-2017 Andes Technology Corporation
   3
   4#include <linux/module.h>
   5#include <linux/sched.h>
   6#include <linux/mm.h>
   7#include <asm/nds32.h>
   8#include <asm/tlbflush.h>
   9#include <asm/cacheflush.h>
  10#include <asm/l2_cache.h>
  11#include <nds32_intrinsic.h>
  12
  13#include <asm/cache_info.h>
  14extern struct cache_info L1_cache_info[2];
  15
  16int va_kernel_present(unsigned long addr)
  17{
  18        pmd_t *pmd;
  19        pte_t *ptep, pte;
  20
  21        pmd = pmd_off_k(addr);
  22        if (!pmd_none(*pmd)) {
  23                ptep = pte_offset_map(pmd, addr);
  24                pte = *ptep;
  25                if (pte_present(pte))
  26                        return pte;
  27        }
  28        return 0;
  29}
  30
  31pte_t va_present(struct mm_struct * mm, unsigned long addr)
  32{
  33        pgd_t *pgd;
  34        p4d_t *p4d;
  35        pud_t *pud;
  36        pmd_t *pmd;
  37        pte_t *ptep, pte;
  38
  39        pgd = pgd_offset(mm, addr);
  40        if (!pgd_none(*pgd)) {
  41                p4d = p4d_offset(pgd, addr);
  42                if (!p4d_none(*p4d)) {
  43                        pud = pud_offset(p4d, addr);
  44                        if (!pud_none(*pud)) {
  45                                pmd = pmd_offset(pud, addr);
  46                                if (!pmd_none(*pmd)) {
  47                                        ptep = pte_offset_map(pmd, addr);
  48                                        pte = *ptep;
  49                                        if (pte_present(pte))
  50                                                return pte;
  51                                }
  52                        }
  53                }
  54        }
  55        return 0;
  56
  57}
  58
  59int va_readable(struct pt_regs *regs, unsigned long addr)
  60{
  61        struct mm_struct *mm = current->mm;
  62        pte_t pte;
  63        int ret = 0;
  64
  65        if (user_mode(regs)) {
  66                /* user mode */
  67                pte = va_present(mm, addr);
  68                if (!pte && pte_read(pte))
  69                        ret = 1;
  70        } else {
  71                /* superuser mode is always readable, so we can only
  72                 * check it is present or not*/
  73                return (! !va_kernel_present(addr));
  74        }
  75        return ret;
  76}
  77
  78int va_writable(struct pt_regs *regs, unsigned long addr)
  79{
  80        struct mm_struct *mm = current->mm;
  81        pte_t pte;
  82        int ret = 0;
  83
  84        if (user_mode(regs)) {
  85                /* user mode */
  86                pte = va_present(mm, addr);
  87                if (!pte && pte_write(pte))
  88                        ret = 1;
  89        } else {
  90                /* superuser mode */
  91                pte = va_kernel_present(addr);
  92                if (!pte && pte_kernel_write(pte))
  93                        ret = 1;
  94        }
  95        return ret;
  96}
  97
  98/*
  99 * All
 100 */
 101void cpu_icache_inval_all(void)
 102{
 103        unsigned long end, line_size;
 104
 105        line_size = L1_cache_info[ICACHE].line_size;
 106        end =
 107            line_size * L1_cache_info[ICACHE].ways * L1_cache_info[ICACHE].sets;
 108
 109        do {
 110                end -= line_size;
 111                __asm__ volatile ("\n\tcctl %0, L1I_IX_INVAL"::"r" (end));
 112                end -= line_size;
 113                __asm__ volatile ("\n\tcctl %0, L1I_IX_INVAL"::"r" (end));
 114                end -= line_size;
 115                __asm__ volatile ("\n\tcctl %0, L1I_IX_INVAL"::"r" (end));
 116                end -= line_size;
 117                __asm__ volatile ("\n\tcctl %0, L1I_IX_INVAL"::"r" (end));
 118        } while (end > 0);
 119        __nds32__isb();
 120}
 121
 122void cpu_dcache_inval_all(void)
 123{
 124        __nds32__cctl_l1d_invalall();
 125}
 126
 127#ifdef CONFIG_CACHE_L2
 128void dcache_wb_all_level(void)
 129{
 130        unsigned long flags, cmd;
 131        local_irq_save(flags);
 132        __nds32__cctl_l1d_wball_alvl();
 133        /* Section 1: Ensure the section 2 & 3 program code execution after */
 134        __nds32__cctlidx_read(NDS32_CCTL_L1D_IX_RWD,0);
 135
 136        /* Section 2: Confirm the writeback all level is done in CPU and L2C */
 137        cmd = CCTL_CMD_L2_SYNC;
 138        L2_CMD_RDY();
 139        L2C_W_REG(L2_CCTL_CMD_OFF, cmd);
 140        L2_CMD_RDY();
 141
 142        /* Section 3: Writeback whole L2 cache */
 143        cmd = CCTL_ALL_CMD | CCTL_CMD_L2_IX_WB;
 144        L2_CMD_RDY();
 145        L2C_W_REG(L2_CCTL_CMD_OFF, cmd);
 146        L2_CMD_RDY();
 147        __nds32__msync_all();
 148        local_irq_restore(flags);
 149}
 150EXPORT_SYMBOL(dcache_wb_all_level);
 151#endif
 152
 153void cpu_dcache_wb_all(void)
 154{
 155        __nds32__cctl_l1d_wball_one_lvl();
 156        __nds32__cctlidx_read(NDS32_CCTL_L1D_IX_RWD,0);
 157}
 158
 159void cpu_dcache_wbinval_all(void)
 160{
 161#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
 162        unsigned long flags;
 163        local_irq_save(flags);
 164#endif
 165        cpu_dcache_wb_all();
 166        cpu_dcache_inval_all();
 167#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
 168        local_irq_restore(flags);
 169#endif
 170}
 171
 172/*
 173 * Page
 174 */
 175void cpu_icache_inval_page(unsigned long start)
 176{
 177        unsigned long line_size, end;
 178
 179        line_size = L1_cache_info[ICACHE].line_size;
 180        end = start + PAGE_SIZE;
 181
 182        do {
 183                end -= line_size;
 184                __asm__ volatile ("\n\tcctl %0, L1I_VA_INVAL"::"r" (end));
 185                end -= line_size;
 186                __asm__ volatile ("\n\tcctl %0, L1I_VA_INVAL"::"r" (end));
 187                end -= line_size;
 188                __asm__ volatile ("\n\tcctl %0, L1I_VA_INVAL"::"r" (end));
 189                end -= line_size;
 190                __asm__ volatile ("\n\tcctl %0, L1I_VA_INVAL"::"r" (end));
 191        } while (end != start);
 192        __nds32__isb();
 193}
 194
 195void cpu_dcache_inval_page(unsigned long start)
 196{
 197        unsigned long line_size, end;
 198
 199        line_size = L1_cache_info[DCACHE].line_size;
 200        end = start + PAGE_SIZE;
 201
 202        do {
 203                end -= line_size;
 204                __asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end));
 205                end -= line_size;
 206                __asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end));
 207                end -= line_size;
 208                __asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end));
 209                end -= line_size;
 210                __asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end));
 211        } while (end != start);
 212}
 213
 214void cpu_dcache_wb_page(unsigned long start)
 215{
 216#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
 217        unsigned long line_size, end;
 218
 219        line_size = L1_cache_info[DCACHE].line_size;
 220        end = start + PAGE_SIZE;
 221
 222        do {
 223                end -= line_size;
 224                __asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end));
 225                end -= line_size;
 226                __asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end));
 227                end -= line_size;
 228                __asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end));
 229                end -= line_size;
 230                __asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end));
 231        } while (end != start);
 232        __nds32__cctlidx_read(NDS32_CCTL_L1D_IX_RWD,0);
 233#endif
 234}
 235
 236void cpu_dcache_wbinval_page(unsigned long start)
 237{
 238        unsigned long line_size, end;
 239
 240        line_size = L1_cache_info[DCACHE].line_size;
 241        end = start + PAGE_SIZE;
 242
 243        do {
 244                end -= line_size;
 245#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
 246                __asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end));
 247#endif
 248                __asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end));
 249                end -= line_size;
 250#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
 251                __asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end));
 252#endif
 253                __asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end));
 254                end -= line_size;
 255#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
 256                __asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end));
 257#endif
 258                __asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end));
 259                end -= line_size;
 260#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
 261                __asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end));
 262#endif
 263                __asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end));
 264        } while (end != start);
 265        __nds32__cctlidx_read(NDS32_CCTL_L1D_IX_RWD,0);
 266}
 267
 268void cpu_cache_wbinval_page(unsigned long page, int flushi)
 269{
 270        cpu_dcache_wbinval_page(page);
 271        if (flushi)
 272                cpu_icache_inval_page(page);
 273}
 274
 275/*
 276 * Range
 277 */
 278void cpu_icache_inval_range(unsigned long start, unsigned long end)
 279{
 280        unsigned long line_size;
 281
 282        line_size = L1_cache_info[ICACHE].line_size;
 283
 284        while (end > start) {
 285                __asm__ volatile ("\n\tcctl %0, L1I_VA_INVAL"::"r" (start));
 286                start += line_size;
 287        }
 288        __nds32__isb();
 289}
 290
 291void cpu_dcache_inval_range(unsigned long start, unsigned long end)
 292{
 293        unsigned long line_size;
 294
 295        line_size = L1_cache_info[DCACHE].line_size;
 296
 297        while (end > start) {
 298                __asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (start));
 299                start += line_size;
 300        }
 301}
 302
 303void cpu_dcache_wb_range(unsigned long start, unsigned long end)
 304{
 305#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
 306        unsigned long line_size;
 307
 308        line_size = L1_cache_info[DCACHE].line_size;
 309
 310        while (end > start) {
 311                __asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (start));
 312                start += line_size;
 313        }
 314        __nds32__cctlidx_read(NDS32_CCTL_L1D_IX_RWD,0);
 315#endif
 316}
 317
 318void cpu_dcache_wbinval_range(unsigned long start, unsigned long end)
 319{
 320        unsigned long line_size;
 321
 322        line_size = L1_cache_info[DCACHE].line_size;
 323
 324        while (end > start) {
 325#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
 326                __asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (start));
 327#endif
 328                __asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (start));
 329                start += line_size;
 330        }
 331        __nds32__cctlidx_read(NDS32_CCTL_L1D_IX_RWD,0);
 332}
 333
 334void cpu_cache_wbinval_range(unsigned long start, unsigned long end, int flushi)
 335{
 336        unsigned long line_size, align_start, align_end;
 337
 338        line_size = L1_cache_info[DCACHE].line_size;
 339        align_start = start & ~(line_size - 1);
 340        align_end = (end + line_size - 1) & ~(line_size - 1);
 341        cpu_dcache_wbinval_range(align_start, align_end);
 342
 343        if (flushi) {
 344                line_size = L1_cache_info[ICACHE].line_size;
 345                align_start = start & ~(line_size - 1);
 346                align_end = (end + line_size - 1) & ~(line_size - 1);
 347                cpu_icache_inval_range(align_start, align_end);
 348        }
 349}
 350
 351void cpu_cache_wbinval_range_check(struct vm_area_struct *vma,
 352                                   unsigned long start, unsigned long end,
 353                                   bool flushi, bool wbd)
 354{
 355        unsigned long line_size, t_start, t_end;
 356
 357        if (!flushi && !wbd)
 358                return;
 359        line_size = L1_cache_info[DCACHE].line_size;
 360        start = start & ~(line_size - 1);
 361        end = (end + line_size - 1) & ~(line_size - 1);
 362
 363        if ((end - start) > (8 * PAGE_SIZE)) {
 364                if (wbd)
 365                        cpu_dcache_wbinval_all();
 366                if (flushi)
 367                        cpu_icache_inval_all();
 368                return;
 369        }
 370
 371        t_start = (start + PAGE_SIZE) & PAGE_MASK;
 372        t_end = ((end - 1) & PAGE_MASK);
 373
 374        if ((start & PAGE_MASK) == t_end) {
 375                if (va_present(vma->vm_mm, start)) {
 376                        if (wbd)
 377                                cpu_dcache_wbinval_range(start, end);
 378                        if (flushi)
 379                                cpu_icache_inval_range(start, end);
 380                }
 381                return;
 382        }
 383
 384        if (va_present(vma->vm_mm, start)) {
 385                if (wbd)
 386                        cpu_dcache_wbinval_range(start, t_start);
 387                if (flushi)
 388                        cpu_icache_inval_range(start, t_start);
 389        }
 390
 391        if (va_present(vma->vm_mm, end - 1)) {
 392                if (wbd)
 393                        cpu_dcache_wbinval_range(t_end, end);
 394                if (flushi)
 395                        cpu_icache_inval_range(t_end, end);
 396        }
 397
 398        while (t_start < t_end) {
 399                if (va_present(vma->vm_mm, t_start)) {
 400                        if (wbd)
 401                                cpu_dcache_wbinval_page(t_start);
 402                        if (flushi)
 403                                cpu_icache_inval_page(t_start);
 404                }
 405                t_start += PAGE_SIZE;
 406        }
 407}
 408
 409#ifdef CONFIG_CACHE_L2
 410static inline void cpu_l2cache_op(unsigned long start, unsigned long end, unsigned long op)
 411{
 412        if (atl2c_base) {
 413                unsigned long p_start = __pa(start);
 414                unsigned long p_end = __pa(end);
 415                unsigned long cmd;
 416                unsigned long line_size;
 417                /* TODO Can Use PAGE Mode to optimize if range large than PAGE_SIZE */
 418                line_size = L2_CACHE_LINE_SIZE();
 419                p_start = p_start & (~(line_size - 1));
 420                p_end = (p_end + line_size - 1) & (~(line_size - 1));
 421                cmd =
 422                    (p_start & ~(line_size - 1)) | op |
 423                    CCTL_SINGLE_CMD;
 424                do {
 425                        L2_CMD_RDY();
 426                        L2C_W_REG(L2_CCTL_CMD_OFF, cmd);
 427                        cmd += line_size;
 428                        p_start += line_size;
 429                } while (p_end > p_start);
 430                cmd = CCTL_CMD_L2_SYNC;
 431                L2_CMD_RDY();
 432                L2C_W_REG(L2_CCTL_CMD_OFF, cmd);
 433                L2_CMD_RDY();
 434        }
 435}
 436#else
 437#define cpu_l2cache_op(start,end,op) do { } while (0)
 438#endif
 439/*
 440 * DMA
 441 */
 442void cpu_dma_wb_range(unsigned long start, unsigned long end)
 443{
 444        unsigned long line_size;
 445        unsigned long flags;
 446        line_size = L1_cache_info[DCACHE].line_size;
 447        start = start & (~(line_size - 1));
 448        end = (end + line_size - 1) & (~(line_size - 1));
 449        if (unlikely(start == end))
 450                return;
 451
 452        local_irq_save(flags);
 453        cpu_dcache_wb_range(start, end);
 454        cpu_l2cache_op(start, end, CCTL_CMD_L2_PA_WB);
 455        __nds32__msync_all();
 456        local_irq_restore(flags);
 457}
 458
 459void cpu_dma_inval_range(unsigned long start, unsigned long end)
 460{
 461        unsigned long line_size;
 462        unsigned long old_start = start;
 463        unsigned long old_end = end;
 464        unsigned long flags;
 465        line_size = L1_cache_info[DCACHE].line_size;
 466        start = start & (~(line_size - 1));
 467        end = (end + line_size - 1) & (~(line_size - 1));
 468        if (unlikely(start == end))
 469                return;
 470        local_irq_save(flags);
 471        if (start != old_start) {
 472                cpu_dcache_wbinval_range(start, start + line_size);
 473                cpu_l2cache_op(start, start + line_size, CCTL_CMD_L2_PA_WBINVAL);
 474        }
 475        if (end != old_end) {
 476                cpu_dcache_wbinval_range(end - line_size, end);
 477                cpu_l2cache_op(end - line_size, end, CCTL_CMD_L2_PA_WBINVAL);
 478        }
 479        cpu_dcache_inval_range(start, end);
 480        cpu_l2cache_op(start, end, CCTL_CMD_L2_PA_INVAL);
 481        __nds32__msync_all();
 482        local_irq_restore(flags);
 483
 484}
 485
 486void cpu_dma_wbinval_range(unsigned long start, unsigned long end)
 487{
 488        unsigned long line_size;
 489        unsigned long flags;
 490        line_size = L1_cache_info[DCACHE].line_size;
 491        start = start & (~(line_size - 1));
 492        end = (end + line_size - 1) & (~(line_size - 1));
 493        if (unlikely(start == end))
 494                return;
 495
 496        local_irq_save(flags);
 497        cpu_dcache_wbinval_range(start, end);
 498        cpu_l2cache_op(start, end, CCTL_CMD_L2_PA_WBINVAL);
 499        __nds32__msync_all();
 500        local_irq_restore(flags);
 501}
 502
 503void cpu_proc_init(void)
 504{
 505}
 506
 507void cpu_proc_fin(void)
 508{
 509}
 510
 511void cpu_do_idle(void)
 512{
 513        __nds32__standby_no_wake_grant();
 514}
 515
 516void cpu_reset(unsigned long reset)
 517{
 518        u32 tmp;
 519        GIE_DISABLE();
 520        tmp = __nds32__mfsr(NDS32_SR_CACHE_CTL);
 521        tmp &= ~(CACHE_CTL_mskIC_EN | CACHE_CTL_mskDC_EN);
 522        __nds32__mtsr_isb(tmp, NDS32_SR_CACHE_CTL);
 523        cpu_dcache_wbinval_all();
 524        cpu_icache_inval_all();
 525
 526        __asm__ __volatile__("jr.toff %0\n\t"::"r"(reset));
 527}
 528
 529void cpu_switch_mm(struct mm_struct *mm)
 530{
 531        unsigned long cid;
 532        cid = __nds32__mfsr(NDS32_SR_TLB_MISC);
 533        cid = (cid & ~TLB_MISC_mskCID) | mm->context.id;
 534        __nds32__mtsr_dsb(cid, NDS32_SR_TLB_MISC);
 535        __nds32__mtsr_isb(__pa(mm->pgd), NDS32_SR_L1_PPTB);
 536}
 537