linux/arch/arm/kernel/hw_breakpoint.c
<<
>>
Prefs
   1/*
   2 * This program is free software; you can redistribute it and/or modify
   3 * it under the terms of the GNU General Public License version 2 as
   4 * published by the Free Software Foundation.
   5 *
   6 * This program is distributed in the hope that it will be useful,
   7 * but WITHOUT ANY WARRANTY; without even the implied warranty of
   8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   9 * GNU General Public License for more details.
  10 *
  11 * You should have received a copy of the GNU General Public License
  12 * along with this program; if not, write to the Free Software
  13 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  14 *
  15 * Copyright (C) 2009, 2010 ARM Limited
  16 *
  17 * Author: Will Deacon <will.deacon@arm.com>
  18 */
  19
  20/*
  21 * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
  22 * using the CPU's debug registers.
  23 */
  24#define pr_fmt(fmt) "hw-breakpoint: " fmt
  25
  26#include <linux/errno.h>
  27#include <linux/hardirq.h>
  28#include <linux/perf_event.h>
  29#include <linux/hw_breakpoint.h>
  30#include <linux/smp.h>
  31#include <linux/cpu_pm.h>
  32#include <linux/coresight.h>
  33
  34#include <asm/cacheflush.h>
  35#include <asm/cputype.h>
  36#include <asm/current.h>
  37#include <asm/hw_breakpoint.h>
  38#include <asm/traps.h>
  39
  40/* Breakpoint currently in use for each BRP. */
  41static DEFINE_PER_CPU(struct perf_event *, bp_on_reg[ARM_MAX_BRP]);
  42
  43/* Watchpoint currently in use for each WRP. */
  44static DEFINE_PER_CPU(struct perf_event *, wp_on_reg[ARM_MAX_WRP]);
  45
  46/* Number of BRP/WRP registers on this CPU. */
  47static int core_num_brps __ro_after_init;
  48static int core_num_wrps __ro_after_init;
  49
  50/* Debug architecture version. */
  51static u8 debug_arch __ro_after_init;
  52
  53/* Does debug architecture support OS Save and Restore? */
  54static bool has_ossr __ro_after_init;
  55
  56/* Maximum supported watchpoint length. */
  57static u8 max_watchpoint_len __ro_after_init;
  58
  59#define READ_WB_REG_CASE(OP2, M, VAL)                   \
  60        case ((OP2 << 4) + M):                          \
  61                ARM_DBG_READ(c0, c ## M, OP2, VAL);     \
  62                break
  63
  64#define WRITE_WB_REG_CASE(OP2, M, VAL)                  \
  65        case ((OP2 << 4) + M):                          \
  66                ARM_DBG_WRITE(c0, c ## M, OP2, VAL);    \
  67                break
  68
  69#define GEN_READ_WB_REG_CASES(OP2, VAL)         \
  70        READ_WB_REG_CASE(OP2, 0, VAL);          \
  71        READ_WB_REG_CASE(OP2, 1, VAL);          \
  72        READ_WB_REG_CASE(OP2, 2, VAL);          \
  73        READ_WB_REG_CASE(OP2, 3, VAL);          \
  74        READ_WB_REG_CASE(OP2, 4, VAL);          \
  75        READ_WB_REG_CASE(OP2, 5, VAL);          \
  76        READ_WB_REG_CASE(OP2, 6, VAL);          \
  77        READ_WB_REG_CASE(OP2, 7, VAL);          \
  78        READ_WB_REG_CASE(OP2, 8, VAL);          \
  79        READ_WB_REG_CASE(OP2, 9, VAL);          \
  80        READ_WB_REG_CASE(OP2, 10, VAL);         \
  81        READ_WB_REG_CASE(OP2, 11, VAL);         \
  82        READ_WB_REG_CASE(OP2, 12, VAL);         \
  83        READ_WB_REG_CASE(OP2, 13, VAL);         \
  84        READ_WB_REG_CASE(OP2, 14, VAL);         \
  85        READ_WB_REG_CASE(OP2, 15, VAL)
  86
  87#define GEN_WRITE_WB_REG_CASES(OP2, VAL)        \
  88        WRITE_WB_REG_CASE(OP2, 0, VAL);         \
  89        WRITE_WB_REG_CASE(OP2, 1, VAL);         \
  90        WRITE_WB_REG_CASE(OP2, 2, VAL);         \
  91        WRITE_WB_REG_CASE(OP2, 3, VAL);         \
  92        WRITE_WB_REG_CASE(OP2, 4, VAL);         \
  93        WRITE_WB_REG_CASE(OP2, 5, VAL);         \
  94        WRITE_WB_REG_CASE(OP2, 6, VAL);         \
  95        WRITE_WB_REG_CASE(OP2, 7, VAL);         \
  96        WRITE_WB_REG_CASE(OP2, 8, VAL);         \
  97        WRITE_WB_REG_CASE(OP2, 9, VAL);         \
  98        WRITE_WB_REG_CASE(OP2, 10, VAL);        \
  99        WRITE_WB_REG_CASE(OP2, 11, VAL);        \
 100        WRITE_WB_REG_CASE(OP2, 12, VAL);        \
 101        WRITE_WB_REG_CASE(OP2, 13, VAL);        \
 102        WRITE_WB_REG_CASE(OP2, 14, VAL);        \
 103        WRITE_WB_REG_CASE(OP2, 15, VAL)
 104
 105static u32 read_wb_reg(int n)
 106{
 107        u32 val = 0;
 108
 109        switch (n) {
 110        GEN_READ_WB_REG_CASES(ARM_OP2_BVR, val);
 111        GEN_READ_WB_REG_CASES(ARM_OP2_BCR, val);
 112        GEN_READ_WB_REG_CASES(ARM_OP2_WVR, val);
 113        GEN_READ_WB_REG_CASES(ARM_OP2_WCR, val);
 114        default:
 115                pr_warn("attempt to read from unknown breakpoint register %d\n",
 116                        n);
 117        }
 118
 119        return val;
 120}
 121
 122static void write_wb_reg(int n, u32 val)
 123{
 124        switch (n) {
 125        GEN_WRITE_WB_REG_CASES(ARM_OP2_BVR, val);
 126        GEN_WRITE_WB_REG_CASES(ARM_OP2_BCR, val);
 127        GEN_WRITE_WB_REG_CASES(ARM_OP2_WVR, val);
 128        GEN_WRITE_WB_REG_CASES(ARM_OP2_WCR, val);
 129        default:
 130                pr_warn("attempt to write to unknown breakpoint register %d\n",
 131                        n);
 132        }
 133        isb();
 134}
 135
 136/* Determine debug architecture. */
 137static u8 get_debug_arch(void)
 138{
 139        u32 didr;
 140
 141        /* Do we implement the extended CPUID interface? */
 142        if (((read_cpuid_id() >> 16) & 0xf) != 0xf) {
 143                pr_warn_once("CPUID feature registers not supported. "
 144                             "Assuming v6 debug is present.\n");
 145                return ARM_DEBUG_ARCH_V6;
 146        }
 147
 148        ARM_DBG_READ(c0, c0, 0, didr);
 149        return (didr >> 16) & 0xf;
 150}
 151
 152u8 arch_get_debug_arch(void)
 153{
 154        return debug_arch;
 155}
 156
 157static int debug_arch_supported(void)
 158{
 159        u8 arch = get_debug_arch();
 160
 161        /* We don't support the memory-mapped interface. */
 162        return (arch >= ARM_DEBUG_ARCH_V6 && arch <= ARM_DEBUG_ARCH_V7_ECP14) ||
 163                arch >= ARM_DEBUG_ARCH_V7_1;
 164}
 165
 166/* Can we determine the watchpoint access type from the fsr? */
 167static int debug_exception_updates_fsr(void)
 168{
 169        return get_debug_arch() >= ARM_DEBUG_ARCH_V8;
 170}
 171
 172/* Determine number of WRP registers available. */
 173static int get_num_wrp_resources(void)
 174{
 175        u32 didr;
 176        ARM_DBG_READ(c0, c0, 0, didr);
 177        return ((didr >> 28) & 0xf) + 1;
 178}
 179
 180/* Determine number of BRP registers available. */
 181static int get_num_brp_resources(void)
 182{
 183        u32 didr;
 184        ARM_DBG_READ(c0, c0, 0, didr);
 185        return ((didr >> 24) & 0xf) + 1;
 186}
 187
 188/* Does this core support mismatch breakpoints? */
 189static int core_has_mismatch_brps(void)
 190{
 191        return (get_debug_arch() >= ARM_DEBUG_ARCH_V7_ECP14 &&
 192                get_num_brp_resources() > 1);
 193}
 194
 195/* Determine number of usable WRPs available. */
 196static int get_num_wrps(void)
 197{
 198        /*
 199         * On debug architectures prior to 7.1, when a watchpoint fires, the
 200         * only way to work out which watchpoint it was is by disassembling
 201         * the faulting instruction and working out the address of the memory
 202         * access.
 203         *
 204         * Furthermore, we can only do this if the watchpoint was precise
 205         * since imprecise watchpoints prevent us from calculating register
 206         * based addresses.
 207         *
 208         * Providing we have more than 1 breakpoint register, we only report
 209         * a single watchpoint register for the time being. This way, we always
 210         * know which watchpoint fired. In the future we can either add a
 211         * disassembler and address generation emulator, or we can insert a
 212         * check to see if the DFAR is set on watchpoint exception entry
 213         * [the ARM ARM states that the DFAR is UNKNOWN, but experience shows
 214         * that it is set on some implementations].
 215         */
 216        if (get_debug_arch() < ARM_DEBUG_ARCH_V7_1)
 217                return 1;
 218
 219        return get_num_wrp_resources();
 220}
 221
 222/* Determine number of usable BRPs available. */
 223static int get_num_brps(void)
 224{
 225        int brps = get_num_brp_resources();
 226        return core_has_mismatch_brps() ? brps - 1 : brps;
 227}
 228
 229/*
 230 * In order to access the breakpoint/watchpoint control registers,
 231 * we must be running in debug monitor mode. Unfortunately, we can
 232 * be put into halting debug mode at any time by an external debugger
 233 * but there is nothing we can do to prevent that.
 234 */
 235static int monitor_mode_enabled(void)
 236{
 237        u32 dscr;
 238        ARM_DBG_READ(c0, c1, 0, dscr);
 239        return !!(dscr & ARM_DSCR_MDBGEN);
 240}
 241
 242static int enable_monitor_mode(void)
 243{
 244        u32 dscr;
 245        ARM_DBG_READ(c0, c1, 0, dscr);
 246
 247        /* If monitor mode is already enabled, just return. */
 248        if (dscr & ARM_DSCR_MDBGEN)
 249                goto out;
 250
 251        /* Write to the corresponding DSCR. */
 252        switch (get_debug_arch()) {
 253        case ARM_DEBUG_ARCH_V6:
 254        case ARM_DEBUG_ARCH_V6_1:
 255                ARM_DBG_WRITE(c0, c1, 0, (dscr | ARM_DSCR_MDBGEN));
 256                break;
 257        case ARM_DEBUG_ARCH_V7_ECP14:
 258        case ARM_DEBUG_ARCH_V7_1:
 259        case ARM_DEBUG_ARCH_V8:
 260                ARM_DBG_WRITE(c0, c2, 2, (dscr | ARM_DSCR_MDBGEN));
 261                isb();
 262                break;
 263        default:
 264                return -ENODEV;
 265        }
 266
 267        /* Check that the write made it through. */
 268        ARM_DBG_READ(c0, c1, 0, dscr);
 269        if (!(dscr & ARM_DSCR_MDBGEN)) {
 270                pr_warn_once("Failed to enable monitor mode on CPU %d.\n",
 271                                smp_processor_id());
 272                return -EPERM;
 273        }
 274
 275out:
 276        return 0;
 277}
 278
 279int hw_breakpoint_slots(int type)
 280{
 281        if (!debug_arch_supported())
 282                return 0;
 283
 284        /*
 285         * We can be called early, so don't rely on
 286         * our static variables being initialised.
 287         */
 288        switch (type) {
 289        case TYPE_INST:
 290                return get_num_brps();
 291        case TYPE_DATA:
 292                return get_num_wrps();
 293        default:
 294                pr_warn("unknown slot type: %d\n", type);
 295                return 0;
 296        }
 297}
 298
 299/*
 300 * Check if 8-bit byte-address select is available.
 301 * This clobbers WRP 0.
 302 */
 303static u8 get_max_wp_len(void)
 304{
 305        u32 ctrl_reg;
 306        struct arch_hw_breakpoint_ctrl ctrl;
 307        u8 size = 4;
 308
 309        if (debug_arch < ARM_DEBUG_ARCH_V7_ECP14)
 310                goto out;
 311
 312        memset(&ctrl, 0, sizeof(ctrl));
 313        ctrl.len = ARM_BREAKPOINT_LEN_8;
 314        ctrl_reg = encode_ctrl_reg(ctrl);
 315
 316        write_wb_reg(ARM_BASE_WVR, 0);
 317        write_wb_reg(ARM_BASE_WCR, ctrl_reg);
 318        if ((read_wb_reg(ARM_BASE_WCR) & ctrl_reg) == ctrl_reg)
 319                size = 8;
 320
 321out:
 322        return size;
 323}
 324
 325u8 arch_get_max_wp_len(void)
 326{
 327        return max_watchpoint_len;
 328}
 329
 330/*
 331 * Install a perf counter breakpoint.
 332 */
 333int arch_install_hw_breakpoint(struct perf_event *bp)
 334{
 335        struct arch_hw_breakpoint *info = counter_arch_bp(bp);
 336        struct perf_event **slot, **slots;
 337        int i, max_slots, ctrl_base, val_base;
 338        u32 addr, ctrl;
 339
 340        addr = info->address;
 341        ctrl = encode_ctrl_reg(info->ctrl) | 0x1;
 342
 343        if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
 344                /* Breakpoint */
 345                ctrl_base = ARM_BASE_BCR;
 346                val_base = ARM_BASE_BVR;
 347                slots = this_cpu_ptr(bp_on_reg);
 348                max_slots = core_num_brps;
 349        } else {
 350                /* Watchpoint */
 351                ctrl_base = ARM_BASE_WCR;
 352                val_base = ARM_BASE_WVR;
 353                slots = this_cpu_ptr(wp_on_reg);
 354                max_slots = core_num_wrps;
 355        }
 356
 357        for (i = 0; i < max_slots; ++i) {
 358                slot = &slots[i];
 359
 360                if (!*slot) {
 361                        *slot = bp;
 362                        break;
 363                }
 364        }
 365
 366        if (i == max_slots) {
 367                pr_warn("Can't find any breakpoint slot\n");
 368                return -EBUSY;
 369        }
 370
 371        /* Override the breakpoint data with the step data. */
 372        if (info->step_ctrl.enabled) {
 373                addr = info->trigger & ~0x3;
 374                ctrl = encode_ctrl_reg(info->step_ctrl);
 375                if (info->ctrl.type != ARM_BREAKPOINT_EXECUTE) {
 376                        i = 0;
 377                        ctrl_base = ARM_BASE_BCR + core_num_brps;
 378                        val_base = ARM_BASE_BVR + core_num_brps;
 379                }
 380        }
 381
 382        /* Setup the address register. */
 383        write_wb_reg(val_base + i, addr);
 384
 385        /* Setup the control register. */
 386        write_wb_reg(ctrl_base + i, ctrl);
 387        return 0;
 388}
 389
 390void arch_uninstall_hw_breakpoint(struct perf_event *bp)
 391{
 392        struct arch_hw_breakpoint *info = counter_arch_bp(bp);
 393        struct perf_event **slot, **slots;
 394        int i, max_slots, base;
 395
 396        if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
 397                /* Breakpoint */
 398                base = ARM_BASE_BCR;
 399                slots = this_cpu_ptr(bp_on_reg);
 400                max_slots = core_num_brps;
 401        } else {
 402                /* Watchpoint */
 403                base = ARM_BASE_WCR;
 404                slots = this_cpu_ptr(wp_on_reg);
 405                max_slots = core_num_wrps;
 406        }
 407
 408        /* Remove the breakpoint. */
 409        for (i = 0; i < max_slots; ++i) {
 410                slot = &slots[i];
 411
 412                if (*slot == bp) {
 413                        *slot = NULL;
 414                        break;
 415                }
 416        }
 417
 418        if (i == max_slots) {
 419                pr_warn("Can't find any breakpoint slot\n");
 420                return;
 421        }
 422
 423        /* Ensure that we disable the mismatch breakpoint. */
 424        if (info->ctrl.type != ARM_BREAKPOINT_EXECUTE &&
 425            info->step_ctrl.enabled) {
 426                i = 0;
 427                base = ARM_BASE_BCR + core_num_brps;
 428        }
 429
 430        /* Reset the control register. */
 431        write_wb_reg(base + i, 0);
 432}
 433
 434static int get_hbp_len(u8 hbp_len)
 435{
 436        unsigned int len_in_bytes = 0;
 437
 438        switch (hbp_len) {
 439        case ARM_BREAKPOINT_LEN_1:
 440                len_in_bytes = 1;
 441                break;
 442        case ARM_BREAKPOINT_LEN_2:
 443                len_in_bytes = 2;
 444                break;
 445        case ARM_BREAKPOINT_LEN_4:
 446                len_in_bytes = 4;
 447                break;
 448        case ARM_BREAKPOINT_LEN_8:
 449                len_in_bytes = 8;
 450                break;
 451        }
 452
 453        return len_in_bytes;
 454}
 455
 456/*
 457 * Check whether bp virtual address is in kernel space.
 458 */
 459int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw)
 460{
 461        unsigned int len;
 462        unsigned long va;
 463
 464        va = hw->address;
 465        len = get_hbp_len(hw->ctrl.len);
 466
 467        return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE);
 468}
 469
 470/*
 471 * Extract generic type and length encodings from an arch_hw_breakpoint_ctrl.
 472 * Hopefully this will disappear when ptrace can bypass the conversion
 473 * to generic breakpoint descriptions.
 474 */
 475int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl,
 476                           int *gen_len, int *gen_type)
 477{
 478        /* Type */
 479        switch (ctrl.type) {
 480        case ARM_BREAKPOINT_EXECUTE:
 481                *gen_type = HW_BREAKPOINT_X;
 482                break;
 483        case ARM_BREAKPOINT_LOAD:
 484                *gen_type = HW_BREAKPOINT_R;
 485                break;
 486        case ARM_BREAKPOINT_STORE:
 487                *gen_type = HW_BREAKPOINT_W;
 488                break;
 489        case ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE:
 490                *gen_type = HW_BREAKPOINT_RW;
 491                break;
 492        default:
 493                return -EINVAL;
 494        }
 495
 496        /* Len */
 497        switch (ctrl.len) {
 498        case ARM_BREAKPOINT_LEN_1:
 499                *gen_len = HW_BREAKPOINT_LEN_1;
 500                break;
 501        case ARM_BREAKPOINT_LEN_2:
 502                *gen_len = HW_BREAKPOINT_LEN_2;
 503                break;
 504        case ARM_BREAKPOINT_LEN_4:
 505                *gen_len = HW_BREAKPOINT_LEN_4;
 506                break;
 507        case ARM_BREAKPOINT_LEN_8:
 508                *gen_len = HW_BREAKPOINT_LEN_8;
 509                break;
 510        default:
 511                return -EINVAL;
 512        }
 513
 514        return 0;
 515}
 516
 517/*
 518 * Construct an arch_hw_breakpoint from a perf_event.
 519 */
 520static int arch_build_bp_info(struct perf_event *bp,
 521                              const struct perf_event_attr *attr,
 522                              struct arch_hw_breakpoint *hw)
 523{
 524        /* Type */
 525        switch (attr->bp_type) {
 526        case HW_BREAKPOINT_X:
 527                hw->ctrl.type = ARM_BREAKPOINT_EXECUTE;
 528                break;
 529        case HW_BREAKPOINT_R:
 530                hw->ctrl.type = ARM_BREAKPOINT_LOAD;
 531                break;
 532        case HW_BREAKPOINT_W:
 533                hw->ctrl.type = ARM_BREAKPOINT_STORE;
 534                break;
 535        case HW_BREAKPOINT_RW:
 536                hw->ctrl.type = ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE;
 537                break;
 538        default:
 539                return -EINVAL;
 540        }
 541
 542        /* Len */
 543        switch (attr->bp_len) {
 544        case HW_BREAKPOINT_LEN_1:
 545                hw->ctrl.len = ARM_BREAKPOINT_LEN_1;
 546                break;
 547        case HW_BREAKPOINT_LEN_2:
 548                hw->ctrl.len = ARM_BREAKPOINT_LEN_2;
 549                break;
 550        case HW_BREAKPOINT_LEN_4:
 551                hw->ctrl.len = ARM_BREAKPOINT_LEN_4;
 552                break;
 553        case HW_BREAKPOINT_LEN_8:
 554                hw->ctrl.len = ARM_BREAKPOINT_LEN_8;
 555                if ((hw->ctrl.type != ARM_BREAKPOINT_EXECUTE)
 556                        && max_watchpoint_len >= 8)
 557                        break;
 558        default:
 559                return -EINVAL;
 560        }
 561
 562        /*
 563         * Breakpoints must be of length 2 (thumb) or 4 (ARM) bytes.
 564         * Watchpoints can be of length 1, 2, 4 or 8 bytes if supported
 565         * by the hardware and must be aligned to the appropriate number of
 566         * bytes.
 567         */
 568        if (hw->ctrl.type == ARM_BREAKPOINT_EXECUTE &&
 569            hw->ctrl.len != ARM_BREAKPOINT_LEN_2 &&
 570            hw->ctrl.len != ARM_BREAKPOINT_LEN_4)
 571                return -EINVAL;
 572
 573        /* Address */
 574        hw->address = attr->bp_addr;
 575
 576        /* Privilege */
 577        hw->ctrl.privilege = ARM_BREAKPOINT_USER;
 578        if (arch_check_bp_in_kernelspace(hw))
 579                hw->ctrl.privilege |= ARM_BREAKPOINT_PRIV;
 580
 581        /* Enabled? */
 582        hw->ctrl.enabled = !attr->disabled;
 583
 584        /* Mismatch */
 585        hw->ctrl.mismatch = 0;
 586
 587        return 0;
 588}
 589
 590/*
 591 * Validate the arch-specific HW Breakpoint register settings.
 592 */
 593int hw_breakpoint_arch_parse(struct perf_event *bp,
 594                             const struct perf_event_attr *attr,
 595                             struct arch_hw_breakpoint *hw)
 596{
 597        int ret = 0;
 598        u32 offset, alignment_mask = 0x3;
 599
 600        /* Ensure that we are in monitor debug mode. */
 601        if (!monitor_mode_enabled())
 602                return -ENODEV;
 603
 604        /* Build the arch_hw_breakpoint. */
 605        ret = arch_build_bp_info(bp, attr, hw);
 606        if (ret)
 607                goto out;
 608
 609        /* Check address alignment. */
 610        if (hw->ctrl.len == ARM_BREAKPOINT_LEN_8)
 611                alignment_mask = 0x7;
 612        offset = hw->address & alignment_mask;
 613        switch (offset) {
 614        case 0:
 615                /* Aligned */
 616                break;
 617        case 1:
 618        case 2:
 619                /* Allow halfword watchpoints and breakpoints. */
 620                if (hw->ctrl.len == ARM_BREAKPOINT_LEN_2)
 621                        break;
 622        case 3:
 623                /* Allow single byte watchpoint. */
 624                if (hw->ctrl.len == ARM_BREAKPOINT_LEN_1)
 625                        break;
 626        default:
 627                ret = -EINVAL;
 628                goto out;
 629        }
 630
 631        hw->address &= ~alignment_mask;
 632        hw->ctrl.len <<= offset;
 633
 634        if (is_default_overflow_handler(bp)) {
 635                /*
 636                 * Mismatch breakpoints are required for single-stepping
 637                 * breakpoints.
 638                 */
 639                if (!core_has_mismatch_brps())
 640                        return -EINVAL;
 641
 642                /* We don't allow mismatch breakpoints in kernel space. */
 643                if (arch_check_bp_in_kernelspace(hw))
 644                        return -EPERM;
 645
 646                /*
 647                 * Per-cpu breakpoints are not supported by our stepping
 648                 * mechanism.
 649                 */
 650                if (!bp->hw.target)
 651                        return -EINVAL;
 652
 653                /*
 654                 * We only support specific access types if the fsr
 655                 * reports them.
 656                 */
 657                if (!debug_exception_updates_fsr() &&
 658                    (hw->ctrl.type == ARM_BREAKPOINT_LOAD ||
 659                     hw->ctrl.type == ARM_BREAKPOINT_STORE))
 660                        return -EINVAL;
 661        }
 662
 663out:
 664        return ret;
 665}
 666
 667/*
 668 * Enable/disable single-stepping over the breakpoint bp at address addr.
 669 */
 670static void enable_single_step(struct perf_event *bp, u32 addr)
 671{
 672        struct arch_hw_breakpoint *info = counter_arch_bp(bp);
 673
 674        arch_uninstall_hw_breakpoint(bp);
 675        info->step_ctrl.mismatch  = 1;
 676        info->step_ctrl.len       = ARM_BREAKPOINT_LEN_4;
 677        info->step_ctrl.type      = ARM_BREAKPOINT_EXECUTE;
 678        info->step_ctrl.privilege = info->ctrl.privilege;
 679        info->step_ctrl.enabled   = 1;
 680        info->trigger             = addr;
 681        arch_install_hw_breakpoint(bp);
 682}
 683
 684static void disable_single_step(struct perf_event *bp)
 685{
 686        arch_uninstall_hw_breakpoint(bp);
 687        counter_arch_bp(bp)->step_ctrl.enabled = 0;
 688        arch_install_hw_breakpoint(bp);
 689}
 690
 691static void watchpoint_handler(unsigned long addr, unsigned int fsr,
 692                               struct pt_regs *regs)
 693{
 694        int i, access;
 695        u32 val, ctrl_reg, alignment_mask;
 696        struct perf_event *wp, **slots;
 697        struct arch_hw_breakpoint *info;
 698        struct arch_hw_breakpoint_ctrl ctrl;
 699
 700        slots = this_cpu_ptr(wp_on_reg);
 701
 702        for (i = 0; i < core_num_wrps; ++i) {
 703                rcu_read_lock();
 704
 705                wp = slots[i];
 706
 707                if (wp == NULL)
 708                        goto unlock;
 709
 710                info = counter_arch_bp(wp);
 711                /*
 712                 * The DFAR is an unknown value on debug architectures prior
 713                 * to 7.1. Since we only allow a single watchpoint on these
 714                 * older CPUs, we can set the trigger to the lowest possible
 715                 * faulting address.
 716                 */
 717                if (debug_arch < ARM_DEBUG_ARCH_V7_1) {
 718                        BUG_ON(i > 0);
 719                        info->trigger = wp->attr.bp_addr;
 720                } else {
 721                        if (info->ctrl.len == ARM_BREAKPOINT_LEN_8)
 722                                alignment_mask = 0x7;
 723                        else
 724                                alignment_mask = 0x3;
 725
 726                        /* Check if the watchpoint value matches. */
 727                        val = read_wb_reg(ARM_BASE_WVR + i);
 728                        if (val != (addr & ~alignment_mask))
 729                                goto unlock;
 730
 731                        /* Possible match, check the byte address select. */
 732                        ctrl_reg = read_wb_reg(ARM_BASE_WCR + i);
 733                        decode_ctrl_reg(ctrl_reg, &ctrl);
 734                        if (!((1 << (addr & alignment_mask)) & ctrl.len))
 735                                goto unlock;
 736
 737                        /* Check that the access type matches. */
 738                        if (debug_exception_updates_fsr()) {
 739                                access = (fsr & ARM_FSR_ACCESS_MASK) ?
 740                                          HW_BREAKPOINT_W : HW_BREAKPOINT_R;
 741                                if (!(access & hw_breakpoint_type(wp)))
 742                                        goto unlock;
 743                        }
 744
 745                        /* We have a winner. */
 746                        info->trigger = addr;
 747                }
 748
 749                pr_debug("watchpoint fired: address = 0x%x\n", info->trigger);
 750                perf_bp_event(wp, regs);
 751
 752                /*
 753                 * If no overflow handler is present, insert a temporary
 754                 * mismatch breakpoint so we can single-step over the
 755                 * watchpoint trigger.
 756                 */
 757                if (is_default_overflow_handler(wp))
 758                        enable_single_step(wp, instruction_pointer(regs));
 759
 760unlock:
 761                rcu_read_unlock();
 762        }
 763}
 764
 765static void watchpoint_single_step_handler(unsigned long pc)
 766{
 767        int i;
 768        struct perf_event *wp, **slots;
 769        struct arch_hw_breakpoint *info;
 770
 771        slots = this_cpu_ptr(wp_on_reg);
 772
 773        for (i = 0; i < core_num_wrps; ++i) {
 774                rcu_read_lock();
 775
 776                wp = slots[i];
 777
 778                if (wp == NULL)
 779                        goto unlock;
 780
 781                info = counter_arch_bp(wp);
 782                if (!info->step_ctrl.enabled)
 783                        goto unlock;
 784
 785                /*
 786                 * Restore the original watchpoint if we've completed the
 787                 * single-step.
 788                 */
 789                if (info->trigger != pc)
 790                        disable_single_step(wp);
 791
 792unlock:
 793                rcu_read_unlock();
 794        }
 795}
 796
 797static void breakpoint_handler(unsigned long unknown, struct pt_regs *regs)
 798{
 799        int i;
 800        u32 ctrl_reg, val, addr;
 801        struct perf_event *bp, **slots;
 802        struct arch_hw_breakpoint *info;
 803        struct arch_hw_breakpoint_ctrl ctrl;
 804
 805        slots = this_cpu_ptr(bp_on_reg);
 806
 807        /* The exception entry code places the amended lr in the PC. */
 808        addr = regs->ARM_pc;
 809
 810        /* Check the currently installed breakpoints first. */
 811        for (i = 0; i < core_num_brps; ++i) {
 812                rcu_read_lock();
 813
 814                bp = slots[i];
 815
 816                if (bp == NULL)
 817                        goto unlock;
 818
 819                info = counter_arch_bp(bp);
 820
 821                /* Check if the breakpoint value matches. */
 822                val = read_wb_reg(ARM_BASE_BVR + i);
 823                if (val != (addr & ~0x3))
 824                        goto mismatch;
 825
 826                /* Possible match, check the byte address select to confirm. */
 827                ctrl_reg = read_wb_reg(ARM_BASE_BCR + i);
 828                decode_ctrl_reg(ctrl_reg, &ctrl);
 829                if ((1 << (addr & 0x3)) & ctrl.len) {
 830                        info->trigger = addr;
 831                        pr_debug("breakpoint fired: address = 0x%x\n", addr);
 832                        perf_bp_event(bp, regs);
 833                        if (!bp->overflow_handler)
 834                                enable_single_step(bp, addr);
 835                        goto unlock;
 836                }
 837
 838mismatch:
 839                /* If we're stepping a breakpoint, it can now be restored. */
 840                if (info->step_ctrl.enabled)
 841                        disable_single_step(bp);
 842unlock:
 843                rcu_read_unlock();
 844        }
 845
 846        /* Handle any pending watchpoint single-step breakpoints. */
 847        watchpoint_single_step_handler(addr);
 848}
 849
 850/*
 851 * Called from either the Data Abort Handler [watchpoint] or the
 852 * Prefetch Abort Handler [breakpoint] with interrupts disabled.
 853 */
 854static int hw_breakpoint_pending(unsigned long addr, unsigned int fsr,
 855                                 struct pt_regs *regs)
 856{
 857        int ret = 0;
 858        u32 dscr;
 859
 860        preempt_disable();
 861
 862        if (interrupts_enabled(regs))
 863                local_irq_enable();
 864
 865        /* We only handle watchpoints and hardware breakpoints. */
 866        ARM_DBG_READ(c0, c1, 0, dscr);
 867
 868        /* Perform perf callbacks. */
 869        switch (ARM_DSCR_MOE(dscr)) {
 870        case ARM_ENTRY_BREAKPOINT:
 871                breakpoint_handler(addr, regs);
 872                break;
 873        case ARM_ENTRY_ASYNC_WATCHPOINT:
 874                WARN(1, "Asynchronous watchpoint exception taken. Debugging results may be unreliable\n");
 875        case ARM_ENTRY_SYNC_WATCHPOINT:
 876                watchpoint_handler(addr, fsr, regs);
 877                break;
 878        default:
 879                ret = 1; /* Unhandled fault. */
 880        }
 881
 882        preempt_enable();
 883
 884        return ret;
 885}
 886
 887/*
 888 * One-time initialisation.
 889 */
 890static cpumask_t debug_err_mask;
 891
 892static int debug_reg_trap(struct pt_regs *regs, unsigned int instr)
 893{
 894        int cpu = smp_processor_id();
 895
 896        pr_warn("Debug register access (0x%x) caused undefined instruction on CPU %d\n",
 897                instr, cpu);
 898
 899        /* Set the error flag for this CPU and skip the faulting instruction. */
 900        cpumask_set_cpu(cpu, &debug_err_mask);
 901        instruction_pointer(regs) += 4;
 902        return 0;
 903}
 904
 905static struct undef_hook debug_reg_hook = {
 906        .instr_mask     = 0x0fe80f10,
 907        .instr_val      = 0x0e000e10,
 908        .fn             = debug_reg_trap,
 909};
 910
 911/* Does this core support OS Save and Restore? */
 912static bool core_has_os_save_restore(void)
 913{
 914        u32 oslsr;
 915
 916        switch (get_debug_arch()) {
 917        case ARM_DEBUG_ARCH_V7_1:
 918                return true;
 919        case ARM_DEBUG_ARCH_V7_ECP14:
 920                ARM_DBG_READ(c1, c1, 4, oslsr);
 921                if (oslsr & ARM_OSLSR_OSLM0)
 922                        return true;
 923        default:
 924                return false;
 925        }
 926}
 927
 928static void reset_ctrl_regs(unsigned int cpu)
 929{
 930        int i, raw_num_brps, err = 0;
 931        u32 val;
 932
 933        /*
 934         * v7 debug contains save and restore registers so that debug state
 935         * can be maintained across low-power modes without leaving the debug
 936         * logic powered up. It is IMPLEMENTATION DEFINED whether we can access
 937         * the debug registers out of reset, so we must unlock the OS Lock
 938         * Access Register to avoid taking undefined instruction exceptions
 939         * later on.
 940         */
 941        switch (debug_arch) {
 942        case ARM_DEBUG_ARCH_V6:
 943        case ARM_DEBUG_ARCH_V6_1:
 944                /* ARMv6 cores clear the registers out of reset. */
 945                goto out_mdbgen;
 946        case ARM_DEBUG_ARCH_V7_ECP14:
 947                /*
 948                 * Ensure sticky power-down is clear (i.e. debug logic is
 949                 * powered up).
 950                 */
 951                ARM_DBG_READ(c1, c5, 4, val);
 952                if ((val & 0x1) == 0)
 953                        err = -EPERM;
 954
 955                if (!has_ossr)
 956                        goto clear_vcr;
 957                break;
 958        case ARM_DEBUG_ARCH_V7_1:
 959                /*
 960                 * Ensure the OS double lock is clear.
 961                 */
 962                ARM_DBG_READ(c1, c3, 4, val);
 963                if ((val & 0x1) == 1)
 964                        err = -EPERM;
 965                break;
 966        }
 967
 968        if (err) {
 969                pr_warn_once("CPU %d debug is powered down!\n", cpu);
 970                cpumask_or(&debug_err_mask, &debug_err_mask, cpumask_of(cpu));
 971                return;
 972        }
 973
 974        /*
 975         * Unconditionally clear the OS lock by writing a value
 976         * other than CS_LAR_KEY to the access register.
 977         */
 978        ARM_DBG_WRITE(c1, c0, 4, ~CORESIGHT_UNLOCK);
 979        isb();
 980
 981        /*
 982         * Clear any configured vector-catch events before
 983         * enabling monitor mode.
 984         */
 985clear_vcr:
 986        ARM_DBG_WRITE(c0, c7, 0, 0);
 987        isb();
 988
 989        if (cpumask_intersects(&debug_err_mask, cpumask_of(cpu))) {
 990                pr_warn_once("CPU %d failed to disable vector catch\n", cpu);
 991                return;
 992        }
 993
 994        /*
 995         * The control/value register pairs are UNKNOWN out of reset so
 996         * clear them to avoid spurious debug events.
 997         */
 998        raw_num_brps = get_num_brp_resources();
 999        for (i = 0; i < raw_num_brps; ++i) {
1000                write_wb_reg(ARM_BASE_BCR + i, 0UL);
1001                write_wb_reg(ARM_BASE_BVR + i, 0UL);
1002        }
1003
1004        for (i = 0; i < core_num_wrps; ++i) {
1005                write_wb_reg(ARM_BASE_WCR + i, 0UL);
1006                write_wb_reg(ARM_BASE_WVR + i, 0UL);
1007        }
1008
1009        if (cpumask_intersects(&debug_err_mask, cpumask_of(cpu))) {
1010                pr_warn_once("CPU %d failed to clear debug register pairs\n", cpu);
1011                return;
1012        }
1013
1014        /*
1015         * Have a crack at enabling monitor mode. We don't actually need
1016         * it yet, but reporting an error early is useful if it fails.
1017         */
1018out_mdbgen:
1019        if (enable_monitor_mode())
1020                cpumask_or(&debug_err_mask, &debug_err_mask, cpumask_of(cpu));
1021}
1022
1023static int dbg_reset_online(unsigned int cpu)
1024{
1025        local_irq_disable();
1026        reset_ctrl_regs(cpu);
1027        local_irq_enable();
1028        return 0;
1029}
1030
1031#ifdef CONFIG_CPU_PM
1032static int dbg_cpu_pm_notify(struct notifier_block *self, unsigned long action,
1033                             void *v)
1034{
1035        if (action == CPU_PM_EXIT)
1036                reset_ctrl_regs(smp_processor_id());
1037
1038        return NOTIFY_OK;
1039}
1040
1041static struct notifier_block dbg_cpu_pm_nb = {
1042        .notifier_call = dbg_cpu_pm_notify,
1043};
1044
1045static void __init pm_init(void)
1046{
1047        cpu_pm_register_notifier(&dbg_cpu_pm_nb);
1048}
1049#else
1050static inline void pm_init(void)
1051{
1052}
1053#endif
1054
1055static int __init arch_hw_breakpoint_init(void)
1056{
1057        int ret;
1058
1059        debug_arch = get_debug_arch();
1060
1061        if (!debug_arch_supported()) {
1062                pr_info("debug architecture 0x%x unsupported.\n", debug_arch);
1063                return 0;
1064        }
1065
1066        /*
1067         * Scorpion CPUs (at least those in APQ8060) seem to set DBGPRSR.SPD
1068         * whenever a WFI is issued, even if the core is not powered down, in
1069         * violation of the architecture.  When DBGPRSR.SPD is set, accesses to
1070         * breakpoint and watchpoint registers are treated as undefined, so
1071         * this results in boot time and runtime failures when these are
1072         * accessed and we unexpectedly take a trap.
1073         *
1074         * It's not clear if/how this can be worked around, so we blacklist
1075         * Scorpion CPUs to avoid these issues.
1076        */
1077        if (read_cpuid_part() == ARM_CPU_PART_SCORPION) {
1078                pr_info("Scorpion CPU detected. Hardware breakpoints and watchpoints disabled\n");
1079                return 0;
1080        }
1081
1082        has_ossr = core_has_os_save_restore();
1083
1084        /* Determine how many BRPs/WRPs are available. */
1085        core_num_brps = get_num_brps();
1086        core_num_wrps = get_num_wrps();
1087
1088        /*
1089         * We need to tread carefully here because DBGSWENABLE may be
1090         * driven low on this core and there isn't an architected way to
1091         * determine that.
1092         */
1093        cpus_read_lock();
1094        register_undef_hook(&debug_reg_hook);
1095
1096        /*
1097         * Register CPU notifier which resets the breakpoint resources. We
1098         * assume that a halting debugger will leave the world in a nice state
1099         * for us.
1100         */
1101        ret = cpuhp_setup_state_cpuslocked(CPUHP_AP_ONLINE_DYN,
1102                                           "arm/hw_breakpoint:online",
1103                                           dbg_reset_online, NULL);
1104        unregister_undef_hook(&debug_reg_hook);
1105        if (WARN_ON(ret < 0) || !cpumask_empty(&debug_err_mask)) {
1106                core_num_brps = 0;
1107                core_num_wrps = 0;
1108                if (ret > 0)
1109                        cpuhp_remove_state_nocalls_cpuslocked(ret);
1110                cpus_read_unlock();
1111                return 0;
1112        }
1113
1114        pr_info("found %d " "%s" "breakpoint and %d watchpoint registers.\n",
1115                core_num_brps, core_has_mismatch_brps() ? "(+1 reserved) " :
1116                "", core_num_wrps);
1117
1118        /* Work out the maximum supported watchpoint length. */
1119        max_watchpoint_len = get_max_wp_len();
1120        pr_info("maximum watchpoint size is %u bytes.\n",
1121                        max_watchpoint_len);
1122
1123        /* Register debug fault handler. */
1124        hook_fault_code(FAULT_CODE_DEBUG, hw_breakpoint_pending, SIGTRAP,
1125                        TRAP_HWBKPT, "watchpoint debug exception");
1126        hook_ifault_code(FAULT_CODE_DEBUG, hw_breakpoint_pending, SIGTRAP,
1127                        TRAP_HWBKPT, "breakpoint debug exception");
1128        cpus_read_unlock();
1129
1130        /* Register PM notifiers. */
1131        pm_init();
1132        return 0;
1133}
1134arch_initcall(arch_hw_breakpoint_init);
1135
1136void hw_breakpoint_pmu_read(struct perf_event *bp)
1137{
1138}
1139
1140/*
1141 * Dummy function to register with die_notifier.
1142 */
1143int hw_breakpoint_exceptions_notify(struct notifier_block *unused,
1144                                        unsigned long val, void *data)
1145{
1146        return NOTIFY_DONE;
1147}
1148