linux/arch/arm/mm/cache-b15-rac.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Broadcom Brahma-B15 CPU read-ahead cache management functions
   4 *
   5 * Copyright (C) 2015-2016 Broadcom
   6 */
   7
   8#include <linux/err.h>
   9#include <linux/spinlock.h>
  10#include <linux/io.h>
  11#include <linux/bitops.h>
  12#include <linux/of_address.h>
  13#include <linux/notifier.h>
  14#include <linux/cpu.h>
  15#include <linux/syscore_ops.h>
  16#include <linux/reboot.h>
  17
  18#include <asm/cacheflush.h>
  19#include <asm/hardware/cache-b15-rac.h>
  20
  21extern void v7_flush_kern_cache_all(void);
  22
  23/* RAC register offsets, relative to the HIF_CPU_BIUCTRL register base */
  24#define RAC_CONFIG0_REG                 (0x78)
  25#define  RACENPREF_MASK                 (0x3)
  26#define  RACPREFINST_SHIFT              (0)
  27#define  RACENINST_SHIFT                (2)
  28#define  RACPREFDATA_SHIFT              (4)
  29#define  RACENDATA_SHIFT                (6)
  30#define  RAC_CPU_SHIFT                  (8)
  31#define  RACCFG_MASK                    (0xff)
  32#define RAC_CONFIG1_REG                 (0x7c)
  33/* Brahma-B15 is a quad-core only design */
  34#define B15_RAC_FLUSH_REG               (0x80)
  35/* Brahma-B53 is an octo-core design */
  36#define B53_RAC_FLUSH_REG               (0x84)
  37#define  FLUSH_RAC                      (1 << 0)
  38
  39/* Bitmask to enable instruction and data prefetching with a 256-bytes stride */
  40#define RAC_DATA_INST_EN_MASK           (1 << RACPREFINST_SHIFT | \
  41                                         RACENPREF_MASK << RACENINST_SHIFT | \
  42                                         1 << RACPREFDATA_SHIFT | \
  43                                         RACENPREF_MASK << RACENDATA_SHIFT)
  44
  45#define RAC_ENABLED                     0
  46/* Special state where we want to bypass the spinlock and call directly
  47 * into the v7 cache maintenance operations during suspend/resume
  48 */
  49#define RAC_SUSPENDED                   1
  50
  51static void __iomem *b15_rac_base;
  52static DEFINE_SPINLOCK(rac_lock);
  53
  54static u32 rac_config0_reg;
  55static u32 rac_flush_offset;
  56
  57/* Initialization flag to avoid checking for b15_rac_base, and to prevent
  58 * multi-platform kernels from crashing here as well.
  59 */
  60static unsigned long b15_rac_flags;
  61
  62static inline u32 __b15_rac_disable(void)
  63{
  64        u32 val = __raw_readl(b15_rac_base + RAC_CONFIG0_REG);
  65        __raw_writel(0, b15_rac_base + RAC_CONFIG0_REG);
  66        dmb();
  67        return val;
  68}
  69
  70static inline void __b15_rac_flush(void)
  71{
  72        u32 reg;
  73
  74        __raw_writel(FLUSH_RAC, b15_rac_base + rac_flush_offset);
  75        do {
  76                /* This dmb() is required to force the Bus Interface Unit
  77                 * to clean oustanding writes, and forces an idle cycle
  78                 * to be inserted.
  79                 */
  80                dmb();
  81                reg = __raw_readl(b15_rac_base + rac_flush_offset);
  82        } while (reg & FLUSH_RAC);
  83}
  84
  85static inline u32 b15_rac_disable_and_flush(void)
  86{
  87        u32 reg;
  88
  89        reg = __b15_rac_disable();
  90        __b15_rac_flush();
  91        return reg;
  92}
  93
  94static inline void __b15_rac_enable(u32 val)
  95{
  96        __raw_writel(val, b15_rac_base + RAC_CONFIG0_REG);
  97        /* dsb() is required here to be consistent with __flush_icache_all() */
  98        dsb();
  99}
 100
 101#define BUILD_RAC_CACHE_OP(name, bar)                           \
 102void b15_flush_##name(void)                                     \
 103{                                                               \
 104        unsigned int do_flush;                                  \
 105        u32 val = 0;                                            \
 106                                                                \
 107        if (test_bit(RAC_SUSPENDED, &b15_rac_flags)) {          \
 108                v7_flush_##name();                              \
 109                bar;                                            \
 110                return;                                         \
 111        }                                                       \
 112                                                                \
 113        spin_lock(&rac_lock);                                   \
 114        do_flush = test_bit(RAC_ENABLED, &b15_rac_flags);       \
 115        if (do_flush)                                           \
 116                val = b15_rac_disable_and_flush();              \
 117        v7_flush_##name();                                      \
 118        if (!do_flush)                                          \
 119                bar;                                            \
 120        else                                                    \
 121                __b15_rac_enable(val);                          \
 122        spin_unlock(&rac_lock);                                 \
 123}
 124
 125#define nobarrier
 126
 127/* The readahead cache present in the Brahma-B15 CPU is a special piece of
 128 * hardware after the integrated L2 cache of the B15 CPU complex whose purpose
 129 * is to prefetch instruction and/or data with a line size of either 64 bytes
 130 * or 256 bytes. The rationale is that the data-bus of the CPU interface is
 131 * optimized for 256-bytes transactions, and enabling the readahead cache
 132 * provides a significant performance boost we want it enabled (typically
 133 * twice the performance for a memcpy benchmark application).
 134 *
 135 * The readahead cache is transparent for Modified Virtual Addresses
 136 * cache maintenance operations: ICIMVAU, DCIMVAC, DCCMVAC, DCCMVAU and
 137 * DCCIMVAC.
 138 *
 139 * It is however not transparent for the following cache maintenance
 140 * operations: DCISW, DCCSW, DCCISW, ICIALLUIS and ICIALLU which is precisely
 141 * what we are patching here with our BUILD_RAC_CACHE_OP here.
 142 */
 143BUILD_RAC_CACHE_OP(kern_cache_all, nobarrier);
 144
 145static void b15_rac_enable(void)
 146{
 147        unsigned int cpu;
 148        u32 enable = 0;
 149
 150        for_each_possible_cpu(cpu)
 151                enable |= (RAC_DATA_INST_EN_MASK << (cpu * RAC_CPU_SHIFT));
 152
 153        b15_rac_disable_and_flush();
 154        __b15_rac_enable(enable);
 155}
 156
 157static int b15_rac_reboot_notifier(struct notifier_block *nb,
 158                                   unsigned long action,
 159                                   void *data)
 160{
 161        /* During kexec, we are not yet migrated on the boot CPU, so we need to
 162         * make sure we are SMP safe here. Once the RAC is disabled, flag it as
 163         * suspended such that the hotplug notifier returns early.
 164         */
 165        if (action == SYS_RESTART) {
 166                spin_lock(&rac_lock);
 167                b15_rac_disable_and_flush();
 168                clear_bit(RAC_ENABLED, &b15_rac_flags);
 169                set_bit(RAC_SUSPENDED, &b15_rac_flags);
 170                spin_unlock(&rac_lock);
 171        }
 172
 173        return NOTIFY_DONE;
 174}
 175
 176static struct notifier_block b15_rac_reboot_nb = {
 177        .notifier_call  = b15_rac_reboot_notifier,
 178};
 179
 180/* The CPU hotplug case is the most interesting one, we basically need to make
 181 * sure that the RAC is disabled for the entire system prior to having a CPU
 182 * die, in particular prior to this dying CPU having exited the coherency
 183 * domain.
 184 *
 185 * Once this CPU is marked dead, we can safely re-enable the RAC for the
 186 * remaining CPUs in the system which are still online.
 187 *
 188 * Offlining a CPU is the problematic case, onlining a CPU is not much of an
 189 * issue since the CPU and its cache-level hierarchy will start filling with
 190 * the RAC disabled, so L1 and L2 only.
 191 *
 192 * In this function, we should NOT have to verify any unsafe setting/condition
 193 * b15_rac_base:
 194 *
 195 *   It is protected by the RAC_ENABLED flag which is cleared by default, and
 196 *   being cleared when initial procedure is done. b15_rac_base had been set at
 197 *   that time.
 198 *
 199 * RAC_ENABLED:
 200 *   There is a small timing windows, in b15_rac_init(), between
 201 *      cpuhp_setup_state_*()
 202 *      ...
 203 *      set RAC_ENABLED
 204 *   However, there is no hotplug activity based on the Linux booting procedure.
 205 *
 206 * Since we have to disable RAC for all cores, we keep RAC on as long as as
 207 * possible (disable it as late as possible) to gain the cache benefit.
 208 *
 209 * Thus, dying/dead states are chosen here
 210 *
 211 * We are choosing not do disable the RAC on a per-CPU basis, here, if we did
 212 * we would want to consider disabling it as early as possible to benefit the
 213 * other active CPUs.
 214 */
 215
 216/* Running on the dying CPU */
 217static int b15_rac_dying_cpu(unsigned int cpu)
 218{
 219        /* During kexec/reboot, the RAC is disabled via the reboot notifier
 220         * return early here.
 221         */
 222        if (test_bit(RAC_SUSPENDED, &b15_rac_flags))
 223                return 0;
 224
 225        spin_lock(&rac_lock);
 226
 227        /* Indicate that we are starting a hotplug procedure */
 228        __clear_bit(RAC_ENABLED, &b15_rac_flags);
 229
 230        /* Disable the readahead cache and save its value to a global */
 231        rac_config0_reg = b15_rac_disable_and_flush();
 232
 233        spin_unlock(&rac_lock);
 234
 235        return 0;
 236}
 237
 238/* Running on a non-dying CPU */
 239static int b15_rac_dead_cpu(unsigned int cpu)
 240{
 241        /* During kexec/reboot, the RAC is disabled via the reboot notifier
 242         * return early here.
 243         */
 244        if (test_bit(RAC_SUSPENDED, &b15_rac_flags))
 245                return 0;
 246
 247        spin_lock(&rac_lock);
 248
 249        /* And enable it */
 250        __b15_rac_enable(rac_config0_reg);
 251        __set_bit(RAC_ENABLED, &b15_rac_flags);
 252
 253        spin_unlock(&rac_lock);
 254
 255        return 0;
 256}
 257
 258static int b15_rac_suspend(void)
 259{
 260        /* Suspend the read-ahead cache oeprations, forcing our cache
 261         * implementation to fallback to the regular ARMv7 calls.
 262         *
 263         * We are guaranteed to be running on the boot CPU at this point and
 264         * with every other CPU quiesced, so setting RAC_SUSPENDED is not racy
 265         * here.
 266         */
 267        rac_config0_reg = b15_rac_disable_and_flush();
 268        set_bit(RAC_SUSPENDED, &b15_rac_flags);
 269
 270        return 0;
 271}
 272
 273static void b15_rac_resume(void)
 274{
 275        /* Coming out of a S3 suspend/resume cycle, the read-ahead cache
 276         * register RAC_CONFIG0_REG will be restored to its default value, make
 277         * sure we re-enable it and set the enable flag, we are also guaranteed
 278         * to run on the boot CPU, so not racy again.
 279         */
 280        __b15_rac_enable(rac_config0_reg);
 281        clear_bit(RAC_SUSPENDED, &b15_rac_flags);
 282}
 283
 284static struct syscore_ops b15_rac_syscore_ops = {
 285        .suspend        = b15_rac_suspend,
 286        .resume         = b15_rac_resume,
 287};
 288
 289static int __init b15_rac_init(void)
 290{
 291        struct device_node *dn, *cpu_dn;
 292        int ret = 0, cpu;
 293        u32 reg, en_mask = 0;
 294
 295        dn = of_find_compatible_node(NULL, NULL, "brcm,brcmstb-cpu-biu-ctrl");
 296        if (!dn)
 297                return -ENODEV;
 298
 299        if (WARN(num_possible_cpus() > 4, "RAC only supports 4 CPUs\n"))
 300                goto out;
 301
 302        b15_rac_base = of_iomap(dn, 0);
 303        if (!b15_rac_base) {
 304                pr_err("failed to remap BIU control base\n");
 305                ret = -ENOMEM;
 306                goto out;
 307        }
 308
 309        cpu_dn = of_get_cpu_node(0, NULL);
 310        if (!cpu_dn) {
 311                ret = -ENODEV;
 312                goto out;
 313        }
 314
 315        if (of_device_is_compatible(cpu_dn, "brcm,brahma-b15"))
 316                rac_flush_offset = B15_RAC_FLUSH_REG;
 317        else if (of_device_is_compatible(cpu_dn, "brcm,brahma-b53"))
 318                rac_flush_offset = B53_RAC_FLUSH_REG;
 319        else {
 320                pr_err("Unsupported CPU\n");
 321                of_node_put(cpu_dn);
 322                ret = -EINVAL;
 323                goto out;
 324        }
 325        of_node_put(cpu_dn);
 326
 327        ret = register_reboot_notifier(&b15_rac_reboot_nb);
 328        if (ret) {
 329                pr_err("failed to register reboot notifier\n");
 330                iounmap(b15_rac_base);
 331                goto out;
 332        }
 333
 334        if (IS_ENABLED(CONFIG_HOTPLUG_CPU)) {
 335                ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CACHE_B15_RAC_DEAD,
 336                                        "arm/cache-b15-rac:dead",
 337                                        NULL, b15_rac_dead_cpu);
 338                if (ret)
 339                        goto out_unmap;
 340
 341                ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CACHE_B15_RAC_DYING,
 342                                        "arm/cache-b15-rac:dying",
 343                                        NULL, b15_rac_dying_cpu);
 344                if (ret)
 345                        goto out_cpu_dead;
 346        }
 347
 348        if (IS_ENABLED(CONFIG_PM_SLEEP))
 349                register_syscore_ops(&b15_rac_syscore_ops);
 350
 351        spin_lock(&rac_lock);
 352        reg = __raw_readl(b15_rac_base + RAC_CONFIG0_REG);
 353        for_each_possible_cpu(cpu)
 354                en_mask |= ((1 << RACPREFDATA_SHIFT) << (cpu * RAC_CPU_SHIFT));
 355        WARN(reg & en_mask, "Read-ahead cache not previously disabled\n");
 356
 357        b15_rac_enable();
 358        set_bit(RAC_ENABLED, &b15_rac_flags);
 359        spin_unlock(&rac_lock);
 360
 361        pr_info("Broadcom Brahma-B15 readahead cache at: 0x%p\n",
 362                b15_rac_base + RAC_CONFIG0_REG);
 363
 364        goto out;
 365
 366out_cpu_dead:
 367        cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CACHE_B15_RAC_DYING);
 368out_unmap:
 369        unregister_reboot_notifier(&b15_rac_reboot_nb);
 370        iounmap(b15_rac_base);
 371out:
 372        of_node_put(dn);
 373        return ret;
 374}
 375arch_initcall(b15_rac_init);
 376