1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33#include <linux/irqflags.h>
34#include <linux/kallsyms.h>
35#include <linux/notifier.h>
36#include <linux/kprobes.h>
37#include <linux/kdebug.h>
38#include <linux/kernel.h>
39#include <linux/module.h>
40#include <linux/percpu.h>
41#include <linux/sched.h>
42#include <linux/init.h>
43#include <linux/slab.h>
44#include <linux/list.h>
45#include <linux/cpu.h>
46#include <linux/smp.h>
47#include <linux/bug.h>
48
49#include <linux/hw_breakpoint.h>
50
51
52
53struct bp_cpuinfo {
54
55 unsigned int cpu_pinned;
56
57 unsigned int *tsk_pinned;
58
59 unsigned int flexible;
60};
61
62static DEFINE_PER_CPU(struct bp_cpuinfo, bp_cpuinfo[TYPE_MAX]);
63static int nr_slots[TYPE_MAX];
64
65static struct bp_cpuinfo *get_bp_info(int cpu, enum bp_type_idx type)
66{
67 return per_cpu_ptr(bp_cpuinfo + type, cpu);
68}
69
70
71static LIST_HEAD(bp_task_head);
72
73static int constraints_initialized;
74
75
76struct bp_busy_slots {
77 unsigned int pinned;
78 unsigned int flexible;
79};
80
81
82static DEFINE_MUTEX(nr_bp_mutex);
83
84__weak int hw_breakpoint_weight(struct perf_event *bp)
85{
86 return 1;
87}
88
89static inline enum bp_type_idx find_slot_idx(u64 bp_type)
90{
91 if (bp_type & HW_BREAKPOINT_RW)
92 return TYPE_DATA;
93
94 return TYPE_INST;
95}
96
97
98
99
100
101static unsigned int max_task_bp_pinned(int cpu, enum bp_type_idx type)
102{
103 unsigned int *tsk_pinned = get_bp_info(cpu, type)->tsk_pinned;
104 int i;
105
106 for (i = nr_slots[type] - 1; i >= 0; i--) {
107 if (tsk_pinned[i] > 0)
108 return i + 1;
109 }
110
111 return 0;
112}
113
114
115
116
117
118static int task_bp_pinned(int cpu, struct perf_event *bp, enum bp_type_idx type)
119{
120 struct task_struct *tsk = bp->hw.target;
121 struct perf_event *iter;
122 int count = 0;
123
124 list_for_each_entry(iter, &bp_task_head, hw.bp_list) {
125 if (iter->hw.target == tsk &&
126 find_slot_idx(iter->attr.bp_type) == type &&
127 (iter->cpu < 0 || cpu == iter->cpu))
128 count += hw_breakpoint_weight(iter);
129 }
130
131 return count;
132}
133
134static const struct cpumask *cpumask_of_bp(struct perf_event *bp)
135{
136 if (bp->cpu >= 0)
137 return cpumask_of(bp->cpu);
138 return cpu_possible_mask;
139}
140
141
142
143
144
145static void
146fetch_bp_busy_slots(struct bp_busy_slots *slots, struct perf_event *bp,
147 enum bp_type_idx type)
148{
149 const struct cpumask *cpumask = cpumask_of_bp(bp);
150 int cpu;
151
152 for_each_cpu(cpu, cpumask) {
153 struct bp_cpuinfo *info = get_bp_info(cpu, type);
154 int nr;
155
156 nr = info->cpu_pinned;
157 if (!bp->hw.target)
158 nr += max_task_bp_pinned(cpu, type);
159 else
160 nr += task_bp_pinned(cpu, bp, type);
161
162 if (nr > slots->pinned)
163 slots->pinned = nr;
164
165 nr = info->flexible;
166 if (nr > slots->flexible)
167 slots->flexible = nr;
168 }
169}
170
171
172
173
174
175
176static void
177fetch_this_slot(struct bp_busy_slots *slots, int weight)
178{
179 slots->pinned += weight;
180}
181
182
183
184
185static void toggle_bp_task_slot(struct perf_event *bp, int cpu,
186 enum bp_type_idx type, int weight)
187{
188 unsigned int *tsk_pinned = get_bp_info(cpu, type)->tsk_pinned;
189 int old_idx, new_idx;
190
191 old_idx = task_bp_pinned(cpu, bp, type) - 1;
192 new_idx = old_idx + weight;
193
194 if (old_idx >= 0)
195 tsk_pinned[old_idx]--;
196 if (new_idx >= 0)
197 tsk_pinned[new_idx]++;
198}
199
200
201
202
203static void
204toggle_bp_slot(struct perf_event *bp, bool enable, enum bp_type_idx type,
205 int weight)
206{
207 const struct cpumask *cpumask = cpumask_of_bp(bp);
208 int cpu;
209
210 if (!enable)
211 weight = -weight;
212
213
214 if (!bp->hw.target) {
215 get_bp_info(bp->cpu, type)->cpu_pinned += weight;
216 return;
217 }
218
219
220 for_each_cpu(cpu, cpumask)
221 toggle_bp_task_slot(bp, cpu, type, weight);
222
223 if (enable)
224 list_add_tail(&bp->hw.bp_list, &bp_task_head);
225 else
226 list_del(&bp->hw.bp_list);
227}
228
229
230
231
232__weak void arch_unregister_hw_breakpoint(struct perf_event *bp)
233{
234
235
236
237
238}
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281static int __reserve_bp_slot(struct perf_event *bp, u64 bp_type)
282{
283 struct bp_busy_slots slots = {0};
284 enum bp_type_idx type;
285 int weight;
286
287
288 if (!constraints_initialized)
289 return -ENOMEM;
290
291
292 if (bp_type == HW_BREAKPOINT_EMPTY ||
293 bp_type == HW_BREAKPOINT_INVALID)
294 return -EINVAL;
295
296 type = find_slot_idx(bp_type);
297 weight = hw_breakpoint_weight(bp);
298
299 fetch_bp_busy_slots(&slots, bp, type);
300
301
302
303
304 fetch_this_slot(&slots, weight);
305
306
307 if (slots.pinned + (!!slots.flexible) > nr_slots[type])
308 return -ENOSPC;
309
310 toggle_bp_slot(bp, true, type, weight);
311
312 return 0;
313}
314
315int reserve_bp_slot(struct perf_event *bp)
316{
317 int ret;
318
319 mutex_lock(&nr_bp_mutex);
320
321 ret = __reserve_bp_slot(bp, bp->attr.bp_type);
322
323 mutex_unlock(&nr_bp_mutex);
324
325 return ret;
326}
327
328static void __release_bp_slot(struct perf_event *bp, u64 bp_type)
329{
330 enum bp_type_idx type;
331 int weight;
332
333 type = find_slot_idx(bp_type);
334 weight = hw_breakpoint_weight(bp);
335 toggle_bp_slot(bp, false, type, weight);
336}
337
338void release_bp_slot(struct perf_event *bp)
339{
340 mutex_lock(&nr_bp_mutex);
341
342 arch_unregister_hw_breakpoint(bp);
343 __release_bp_slot(bp, bp->attr.bp_type);
344
345 mutex_unlock(&nr_bp_mutex);
346}
347
348static int __modify_bp_slot(struct perf_event *bp, u64 old_type)
349{
350 int err;
351
352 __release_bp_slot(bp, old_type);
353
354 err = __reserve_bp_slot(bp, bp->attr.bp_type);
355 if (err) {
356
357
358
359
360
361
362
363
364 WARN_ON(__reserve_bp_slot(bp, old_type));
365 }
366
367 return err;
368}
369
370static int modify_bp_slot(struct perf_event *bp, u64 old_type)
371{
372 int ret;
373
374 mutex_lock(&nr_bp_mutex);
375 ret = __modify_bp_slot(bp, old_type);
376 mutex_unlock(&nr_bp_mutex);
377 return ret;
378}
379
380
381
382
383
384
385int dbg_reserve_bp_slot(struct perf_event *bp)
386{
387 if (mutex_is_locked(&nr_bp_mutex))
388 return -1;
389
390 return __reserve_bp_slot(bp, bp->attr.bp_type);
391}
392
393int dbg_release_bp_slot(struct perf_event *bp)
394{
395 if (mutex_is_locked(&nr_bp_mutex))
396 return -1;
397
398 __release_bp_slot(bp, bp->attr.bp_type);
399
400 return 0;
401}
402
403static int validate_hw_breakpoint(struct perf_event *bp)
404{
405 int ret;
406
407 ret = arch_validate_hwbkpt_settings(bp);
408 if (ret)
409 return ret;
410
411 if (arch_check_bp_in_kernelspace(bp)) {
412 if (bp->attr.exclude_kernel)
413 return -EINVAL;
414
415
416
417
418 if (!capable(CAP_SYS_ADMIN))
419 return -EPERM;
420 }
421
422 return 0;
423}
424
425int register_perf_hw_breakpoint(struct perf_event *bp)
426{
427 int ret;
428
429 ret = reserve_bp_slot(bp);
430 if (ret)
431 return ret;
432
433 ret = validate_hw_breakpoint(bp);
434
435
436 if (ret)
437 release_bp_slot(bp);
438
439 return ret;
440}
441
442
443
444
445
446
447
448struct perf_event *
449register_user_hw_breakpoint(struct perf_event_attr *attr,
450 perf_overflow_handler_t triggered,
451 void *context,
452 struct task_struct *tsk)
453{
454 return perf_event_create_kernel_counter(attr, -1, tsk, triggered,
455 context);
456}
457EXPORT_SYMBOL_GPL(register_user_hw_breakpoint);
458
459int
460modify_user_hw_breakpoint_check(struct perf_event *bp, struct perf_event_attr *attr,
461 bool check)
462{
463 u64 old_addr = bp->attr.bp_addr;
464 u64 old_len = bp->attr.bp_len;
465 int old_type = bp->attr.bp_type;
466 bool modify = attr->bp_type != old_type;
467 int err = 0;
468
469 bp->attr.bp_addr = attr->bp_addr;
470 bp->attr.bp_type = attr->bp_type;
471 bp->attr.bp_len = attr->bp_len;
472
473 if (check && memcmp(&bp->attr, attr, sizeof(*attr)))
474 return -EINVAL;
475
476 err = validate_hw_breakpoint(bp);
477 if (!err && modify)
478 err = modify_bp_slot(bp, old_type);
479
480 if (err) {
481 bp->attr.bp_addr = old_addr;
482 bp->attr.bp_type = old_type;
483 bp->attr.bp_len = old_len;
484 return err;
485 }
486
487 bp->attr.disabled = attr->disabled;
488 return 0;
489}
490
491
492
493
494
495
496int modify_user_hw_breakpoint(struct perf_event *bp, struct perf_event_attr *attr)
497{
498
499
500
501
502
503
504 if (irqs_disabled() && bp->ctx && bp->ctx->task == current)
505 perf_event_disable_local(bp);
506 else
507 perf_event_disable(bp);
508
509 if (!attr->disabled) {
510 int err = modify_user_hw_breakpoint_check(bp, attr, false);
511
512 if (err)
513 return err;
514 perf_event_enable(bp);
515 bp->attr.disabled = 0;
516 }
517 return 0;
518}
519EXPORT_SYMBOL_GPL(modify_user_hw_breakpoint);
520
521
522
523
524
525void unregister_hw_breakpoint(struct perf_event *bp)
526{
527 if (!bp)
528 return;
529 perf_event_release_kernel(bp);
530}
531EXPORT_SYMBOL_GPL(unregister_hw_breakpoint);
532
533
534
535
536
537
538
539
540struct perf_event * __percpu *
541register_wide_hw_breakpoint(struct perf_event_attr *attr,
542 perf_overflow_handler_t triggered,
543 void *context)
544{
545 struct perf_event * __percpu *cpu_events, *bp;
546 long err = 0;
547 int cpu;
548
549 cpu_events = alloc_percpu(typeof(*cpu_events));
550 if (!cpu_events)
551 return (void __percpu __force *)ERR_PTR(-ENOMEM);
552
553 get_online_cpus();
554 for_each_online_cpu(cpu) {
555 bp = perf_event_create_kernel_counter(attr, cpu, NULL,
556 triggered, context);
557 if (IS_ERR(bp)) {
558 err = PTR_ERR(bp);
559 break;
560 }
561
562 per_cpu(*cpu_events, cpu) = bp;
563 }
564 put_online_cpus();
565
566 if (likely(!err))
567 return cpu_events;
568
569 unregister_wide_hw_breakpoint(cpu_events);
570 return (void __percpu __force *)ERR_PTR(err);
571}
572EXPORT_SYMBOL_GPL(register_wide_hw_breakpoint);
573
574
575
576
577
578void unregister_wide_hw_breakpoint(struct perf_event * __percpu *cpu_events)
579{
580 int cpu;
581
582 for_each_possible_cpu(cpu)
583 unregister_hw_breakpoint(per_cpu(*cpu_events, cpu));
584
585 free_percpu(cpu_events);
586}
587EXPORT_SYMBOL_GPL(unregister_wide_hw_breakpoint);
588
589static struct notifier_block hw_breakpoint_exceptions_nb = {
590 .notifier_call = hw_breakpoint_exceptions_notify,
591
592 .priority = 0x7fffffff
593};
594
595static void bp_perf_event_destroy(struct perf_event *event)
596{
597 release_bp_slot(event);
598}
599
600static int hw_breakpoint_event_init(struct perf_event *bp)
601{
602 int err;
603
604 if (bp->attr.type != PERF_TYPE_BREAKPOINT)
605 return -ENOENT;
606
607
608
609
610 if (has_branch_stack(bp))
611 return -EOPNOTSUPP;
612
613 err = register_perf_hw_breakpoint(bp);
614 if (err)
615 return err;
616
617 bp->destroy = bp_perf_event_destroy;
618
619 return 0;
620}
621
622static int hw_breakpoint_add(struct perf_event *bp, int flags)
623{
624 if (!(flags & PERF_EF_START))
625 bp->hw.state = PERF_HES_STOPPED;
626
627 if (is_sampling_event(bp)) {
628 bp->hw.last_period = bp->hw.sample_period;
629 perf_swevent_set_period(bp);
630 }
631
632 return arch_install_hw_breakpoint(bp);
633}
634
635static void hw_breakpoint_del(struct perf_event *bp, int flags)
636{
637 arch_uninstall_hw_breakpoint(bp);
638}
639
640static void hw_breakpoint_start(struct perf_event *bp, int flags)
641{
642 bp->hw.state = 0;
643}
644
645static void hw_breakpoint_stop(struct perf_event *bp, int flags)
646{
647 bp->hw.state = PERF_HES_STOPPED;
648}
649
650static struct pmu perf_breakpoint = {
651 .task_ctx_nr = perf_sw_context,
652
653 .event_init = hw_breakpoint_event_init,
654 .add = hw_breakpoint_add,
655 .del = hw_breakpoint_del,
656 .start = hw_breakpoint_start,
657 .stop = hw_breakpoint_stop,
658 .read = hw_breakpoint_pmu_read,
659};
660
661int __init init_hw_breakpoint(void)
662{
663 int cpu, err_cpu;
664 int i;
665
666 for (i = 0; i < TYPE_MAX; i++)
667 nr_slots[i] = hw_breakpoint_slots(i);
668
669 for_each_possible_cpu(cpu) {
670 for (i = 0; i < TYPE_MAX; i++) {
671 struct bp_cpuinfo *info = get_bp_info(cpu, i);
672
673 info->tsk_pinned = kcalloc(nr_slots[i], sizeof(int),
674 GFP_KERNEL);
675 if (!info->tsk_pinned)
676 goto err_alloc;
677 }
678 }
679
680 constraints_initialized = 1;
681
682 perf_pmu_register(&perf_breakpoint, "breakpoint", PERF_TYPE_BREAKPOINT);
683
684 return register_die_notifier(&hw_breakpoint_exceptions_nb);
685
686 err_alloc:
687 for_each_possible_cpu(err_cpu) {
688 for (i = 0; i < TYPE_MAX; i++)
689 kfree(get_bp_info(err_cpu, i)->tsk_pinned);
690 if (err_cpu == cpu)
691 break;
692 }
693
694 return -ENOMEM;
695}
696
697
698