1
2#define pr_fmt(fmt) "kcov: " fmt
3
4#define DISABLE_BRANCH_PROFILING
5#include <linux/atomic.h>
6#include <linux/compiler.h>
7#include <linux/errno.h>
8#include <linux/export.h>
9#include <linux/types.h>
10#include <linux/file.h>
11#include <linux/fs.h>
12#include <linux/hashtable.h>
13#include <linux/init.h>
14#include <linux/mm.h>
15#include <linux/preempt.h>
16#include <linux/printk.h>
17#include <linux/sched.h>
18#include <linux/slab.h>
19#include <linux/spinlock.h>
20#include <linux/vmalloc.h>
21#include <linux/debugfs.h>
22#include <linux/uaccess.h>
23#include <linux/kcov.h>
24#include <linux/refcount.h>
25#include <linux/log2.h>
26#include <asm/setup.h>
27
28#define kcov_debug(fmt, ...) pr_debug("%s: " fmt, __func__, ##__VA_ARGS__)
29
30
31#define KCOV_WORDS_PER_CMP 4
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46struct kcov {
47
48
49
50
51
52
53 refcount_t refcount;
54
55 spinlock_t lock;
56 enum kcov_mode mode;
57
58 unsigned int size;
59
60 void *area;
61
62 struct task_struct *t;
63
64 bool remote;
65
66 unsigned int remote_size;
67
68
69
70
71 int sequence;
72};
73
74struct kcov_remote_area {
75 struct list_head list;
76 unsigned int size;
77};
78
79struct kcov_remote {
80 u64 handle;
81 struct kcov *kcov;
82 struct hlist_node hnode;
83};
84
85static DEFINE_SPINLOCK(kcov_remote_lock);
86static DEFINE_HASHTABLE(kcov_remote_map, 4);
87static struct list_head kcov_remote_areas = LIST_HEAD_INIT(kcov_remote_areas);
88
89struct kcov_percpu_data {
90 void *irq_area;
91
92 unsigned int saved_mode;
93 unsigned int saved_size;
94 void *saved_area;
95 struct kcov *saved_kcov;
96 int saved_sequence;
97};
98
99static DEFINE_PER_CPU(struct kcov_percpu_data, kcov_percpu_data);
100
101
102static struct kcov_remote *kcov_remote_find(u64 handle)
103{
104 struct kcov_remote *remote;
105
106 hash_for_each_possible(kcov_remote_map, remote, hnode, handle) {
107 if (remote->handle == handle)
108 return remote;
109 }
110 return NULL;
111}
112
113
114static struct kcov_remote *kcov_remote_add(struct kcov *kcov, u64 handle)
115{
116 struct kcov_remote *remote;
117
118 if (kcov_remote_find(handle))
119 return ERR_PTR(-EEXIST);
120 remote = kmalloc(sizeof(*remote), GFP_ATOMIC);
121 if (!remote)
122 return ERR_PTR(-ENOMEM);
123 remote->handle = handle;
124 remote->kcov = kcov;
125 hash_add(kcov_remote_map, &remote->hnode, handle);
126 return remote;
127}
128
129
130static struct kcov_remote_area *kcov_remote_area_get(unsigned int size)
131{
132 struct kcov_remote_area *area;
133 struct list_head *pos;
134
135 list_for_each(pos, &kcov_remote_areas) {
136 area = list_entry(pos, struct kcov_remote_area, list);
137 if (area->size == size) {
138 list_del(&area->list);
139 return area;
140 }
141 }
142 return NULL;
143}
144
145
146static void kcov_remote_area_put(struct kcov_remote_area *area,
147 unsigned int size)
148{
149 INIT_LIST_HEAD(&area->list);
150 area->size = size;
151 list_add(&area->list, &kcov_remote_areas);
152}
153
154static notrace bool check_kcov_mode(enum kcov_mode needed_mode, struct task_struct *t)
155{
156 unsigned int mode;
157
158
159
160
161
162
163 if (!in_task() && !(in_serving_softirq() && t->kcov_softirq))
164 return false;
165 mode = READ_ONCE(t->kcov_mode);
166
167
168
169
170
171
172
173 barrier();
174 return mode == needed_mode;
175}
176
177static notrace unsigned long canonicalize_ip(unsigned long ip)
178{
179#ifdef CONFIG_RANDOMIZE_BASE
180 ip -= kaslr_offset();
181#endif
182 return ip;
183}
184
185
186
187
188
189void notrace __sanitizer_cov_trace_pc(void)
190{
191 struct task_struct *t;
192 unsigned long *area;
193 unsigned long ip = canonicalize_ip(_RET_IP_);
194 unsigned long pos;
195
196 t = current;
197 if (!check_kcov_mode(KCOV_MODE_TRACE_PC, t))
198 return;
199
200 area = t->kcov_area;
201
202 pos = READ_ONCE(area[0]) + 1;
203 if (likely(pos < t->kcov_size)) {
204 area[pos] = ip;
205 WRITE_ONCE(area[0], pos);
206 }
207}
208EXPORT_SYMBOL(__sanitizer_cov_trace_pc);
209
210#ifdef CONFIG_KCOV_ENABLE_COMPARISONS
211static void notrace write_comp_data(u64 type, u64 arg1, u64 arg2, u64 ip)
212{
213 struct task_struct *t;
214 u64 *area;
215 u64 count, start_index, end_pos, max_pos;
216
217 t = current;
218 if (!check_kcov_mode(KCOV_MODE_TRACE_CMP, t))
219 return;
220
221 ip = canonicalize_ip(ip);
222
223
224
225
226
227 area = (u64 *)t->kcov_area;
228 max_pos = t->kcov_size * sizeof(unsigned long);
229
230 count = READ_ONCE(area[0]);
231
232
233 start_index = 1 + count * KCOV_WORDS_PER_CMP;
234 end_pos = (start_index + KCOV_WORDS_PER_CMP) * sizeof(u64);
235 if (likely(end_pos <= max_pos)) {
236 area[start_index] = type;
237 area[start_index + 1] = arg1;
238 area[start_index + 2] = arg2;
239 area[start_index + 3] = ip;
240 WRITE_ONCE(area[0], count + 1);
241 }
242}
243
244void notrace __sanitizer_cov_trace_cmp1(u8 arg1, u8 arg2)
245{
246 write_comp_data(KCOV_CMP_SIZE(0), arg1, arg2, _RET_IP_);
247}
248EXPORT_SYMBOL(__sanitizer_cov_trace_cmp1);
249
250void notrace __sanitizer_cov_trace_cmp2(u16 arg1, u16 arg2)
251{
252 write_comp_data(KCOV_CMP_SIZE(1), arg1, arg2, _RET_IP_);
253}
254EXPORT_SYMBOL(__sanitizer_cov_trace_cmp2);
255
256void notrace __sanitizer_cov_trace_cmp4(u32 arg1, u32 arg2)
257{
258 write_comp_data(KCOV_CMP_SIZE(2), arg1, arg2, _RET_IP_);
259}
260EXPORT_SYMBOL(__sanitizer_cov_trace_cmp4);
261
262void notrace __sanitizer_cov_trace_cmp8(u64 arg1, u64 arg2)
263{
264 write_comp_data(KCOV_CMP_SIZE(3), arg1, arg2, _RET_IP_);
265}
266EXPORT_SYMBOL(__sanitizer_cov_trace_cmp8);
267
268void notrace __sanitizer_cov_trace_const_cmp1(u8 arg1, u8 arg2)
269{
270 write_comp_data(KCOV_CMP_SIZE(0) | KCOV_CMP_CONST, arg1, arg2,
271 _RET_IP_);
272}
273EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp1);
274
275void notrace __sanitizer_cov_trace_const_cmp2(u16 arg1, u16 arg2)
276{
277 write_comp_data(KCOV_CMP_SIZE(1) | KCOV_CMP_CONST, arg1, arg2,
278 _RET_IP_);
279}
280EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp2);
281
282void notrace __sanitizer_cov_trace_const_cmp4(u32 arg1, u32 arg2)
283{
284 write_comp_data(KCOV_CMP_SIZE(2) | KCOV_CMP_CONST, arg1, arg2,
285 _RET_IP_);
286}
287EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp4);
288
289void notrace __sanitizer_cov_trace_const_cmp8(u64 arg1, u64 arg2)
290{
291 write_comp_data(KCOV_CMP_SIZE(3) | KCOV_CMP_CONST, arg1, arg2,
292 _RET_IP_);
293}
294EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp8);
295
296void notrace __sanitizer_cov_trace_switch(u64 val, u64 *cases)
297{
298 u64 i;
299 u64 count = cases[0];
300 u64 size = cases[1];
301 u64 type = KCOV_CMP_CONST;
302
303 switch (size) {
304 case 8:
305 type |= KCOV_CMP_SIZE(0);
306 break;
307 case 16:
308 type |= KCOV_CMP_SIZE(1);
309 break;
310 case 32:
311 type |= KCOV_CMP_SIZE(2);
312 break;
313 case 64:
314 type |= KCOV_CMP_SIZE(3);
315 break;
316 default:
317 return;
318 }
319 for (i = 0; i < count; i++)
320 write_comp_data(type, cases[i + 2], val, _RET_IP_);
321}
322EXPORT_SYMBOL(__sanitizer_cov_trace_switch);
323#endif
324
325static void kcov_start(struct task_struct *t, struct kcov *kcov,
326 unsigned int size, void *area, enum kcov_mode mode,
327 int sequence)
328{
329 kcov_debug("t = %px, size = %u, area = %px\n", t, size, area);
330 t->kcov = kcov;
331
332 t->kcov_size = size;
333 t->kcov_area = area;
334 t->kcov_sequence = sequence;
335
336 barrier();
337 WRITE_ONCE(t->kcov_mode, mode);
338}
339
340static void kcov_stop(struct task_struct *t)
341{
342 WRITE_ONCE(t->kcov_mode, KCOV_MODE_DISABLED);
343 barrier();
344 t->kcov = NULL;
345 t->kcov_size = 0;
346 t->kcov_area = NULL;
347}
348
349static void kcov_task_reset(struct task_struct *t)
350{
351 kcov_stop(t);
352 t->kcov_sequence = 0;
353 t->kcov_handle = 0;
354}
355
356void kcov_task_init(struct task_struct *t)
357{
358 kcov_task_reset(t);
359 t->kcov_handle = current->kcov_handle;
360}
361
362static void kcov_reset(struct kcov *kcov)
363{
364 kcov->t = NULL;
365 kcov->mode = KCOV_MODE_INIT;
366 kcov->remote = false;
367 kcov->remote_size = 0;
368 kcov->sequence++;
369}
370
371static void kcov_remote_reset(struct kcov *kcov)
372{
373 int bkt;
374 struct kcov_remote *remote;
375 struct hlist_node *tmp;
376 unsigned long flags;
377
378 spin_lock_irqsave(&kcov_remote_lock, flags);
379 hash_for_each_safe(kcov_remote_map, bkt, tmp, remote, hnode) {
380 if (remote->kcov != kcov)
381 continue;
382 hash_del(&remote->hnode);
383 kfree(remote);
384 }
385
386 kcov_reset(kcov);
387 spin_unlock_irqrestore(&kcov_remote_lock, flags);
388}
389
390static void kcov_disable(struct task_struct *t, struct kcov *kcov)
391{
392 kcov_task_reset(t);
393 if (kcov->remote)
394 kcov_remote_reset(kcov);
395 else
396 kcov_reset(kcov);
397}
398
399static void kcov_get(struct kcov *kcov)
400{
401 refcount_inc(&kcov->refcount);
402}
403
404static void kcov_put(struct kcov *kcov)
405{
406 if (refcount_dec_and_test(&kcov->refcount)) {
407 kcov_remote_reset(kcov);
408 vfree(kcov->area);
409 kfree(kcov);
410 }
411}
412
413void kcov_task_exit(struct task_struct *t)
414{
415 struct kcov *kcov;
416 unsigned long flags;
417
418 kcov = t->kcov;
419 if (kcov == NULL)
420 return;
421
422 spin_lock_irqsave(&kcov->lock, flags);
423 kcov_debug("t = %px, kcov->t = %px\n", t, kcov->t);
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446 if (WARN_ON(kcov->t != t)) {
447 spin_unlock_irqrestore(&kcov->lock, flags);
448 return;
449 }
450
451 kcov_disable(t, kcov);
452 spin_unlock_irqrestore(&kcov->lock, flags);
453 kcov_put(kcov);
454}
455
456static int kcov_mmap(struct file *filep, struct vm_area_struct *vma)
457{
458 int res = 0;
459 void *area;
460 struct kcov *kcov = vma->vm_file->private_data;
461 unsigned long size, off;
462 struct page *page;
463 unsigned long flags;
464
465 area = vmalloc_user(vma->vm_end - vma->vm_start);
466 if (!area)
467 return -ENOMEM;
468
469 spin_lock_irqsave(&kcov->lock, flags);
470 size = kcov->size * sizeof(unsigned long);
471 if (kcov->mode != KCOV_MODE_INIT || vma->vm_pgoff != 0 ||
472 vma->vm_end - vma->vm_start != size) {
473 res = -EINVAL;
474 goto exit;
475 }
476 if (!kcov->area) {
477 kcov->area = area;
478 vma->vm_flags |= VM_DONTEXPAND;
479 spin_unlock_irqrestore(&kcov->lock, flags);
480 for (off = 0; off < size; off += PAGE_SIZE) {
481 page = vmalloc_to_page(kcov->area + off);
482 if (vm_insert_page(vma, vma->vm_start + off, page))
483 WARN_ONCE(1, "vm_insert_page() failed");
484 }
485 return 0;
486 }
487exit:
488 spin_unlock_irqrestore(&kcov->lock, flags);
489 vfree(area);
490 return res;
491}
492
493static int kcov_open(struct inode *inode, struct file *filep)
494{
495 struct kcov *kcov;
496
497 kcov = kzalloc(sizeof(*kcov), GFP_KERNEL);
498 if (!kcov)
499 return -ENOMEM;
500 kcov->mode = KCOV_MODE_DISABLED;
501 kcov->sequence = 1;
502 refcount_set(&kcov->refcount, 1);
503 spin_lock_init(&kcov->lock);
504 filep->private_data = kcov;
505 return nonseekable_open(inode, filep);
506}
507
508static int kcov_close(struct inode *inode, struct file *filep)
509{
510 kcov_put(filep->private_data);
511 return 0;
512}
513
514static int kcov_get_mode(unsigned long arg)
515{
516 if (arg == KCOV_TRACE_PC)
517 return KCOV_MODE_TRACE_PC;
518 else if (arg == KCOV_TRACE_CMP)
519#ifdef CONFIG_KCOV_ENABLE_COMPARISONS
520 return KCOV_MODE_TRACE_CMP;
521#else
522 return -ENOTSUPP;
523#endif
524 else
525 return -EINVAL;
526}
527
528
529
530
531
532
533static void kcov_fault_in_area(struct kcov *kcov)
534{
535 unsigned long stride = PAGE_SIZE / sizeof(unsigned long);
536 unsigned long *area = kcov->area;
537 unsigned long offset;
538
539 for (offset = 0; offset < kcov->size; offset += stride)
540 READ_ONCE(area[offset]);
541}
542
543static inline bool kcov_check_handle(u64 handle, bool common_valid,
544 bool uncommon_valid, bool zero_valid)
545{
546 if (handle & ~(KCOV_SUBSYSTEM_MASK | KCOV_INSTANCE_MASK))
547 return false;
548 switch (handle & KCOV_SUBSYSTEM_MASK) {
549 case KCOV_SUBSYSTEM_COMMON:
550 return (handle & KCOV_INSTANCE_MASK) ?
551 common_valid : zero_valid;
552 case KCOV_SUBSYSTEM_USB:
553 return uncommon_valid;
554 default:
555 return false;
556 }
557 return false;
558}
559
560static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd,
561 unsigned long arg)
562{
563 struct task_struct *t;
564 unsigned long size, unused;
565 int mode, i;
566 struct kcov_remote_arg *remote_arg;
567 struct kcov_remote *remote;
568 unsigned long flags;
569
570 switch (cmd) {
571 case KCOV_INIT_TRACE:
572
573
574
575
576 if (kcov->mode != KCOV_MODE_DISABLED)
577 return -EBUSY;
578
579
580
581
582
583 size = arg;
584 if (size < 2 || size > INT_MAX / sizeof(unsigned long))
585 return -EINVAL;
586 kcov->size = size;
587 kcov->mode = KCOV_MODE_INIT;
588 return 0;
589 case KCOV_ENABLE:
590
591
592
593
594
595
596
597 if (kcov->mode != KCOV_MODE_INIT || !kcov->area)
598 return -EINVAL;
599 t = current;
600 if (kcov->t != NULL || t->kcov != NULL)
601 return -EBUSY;
602 mode = kcov_get_mode(arg);
603 if (mode < 0)
604 return mode;
605 kcov_fault_in_area(kcov);
606 kcov->mode = mode;
607 kcov_start(t, kcov, kcov->size, kcov->area, kcov->mode,
608 kcov->sequence);
609 kcov->t = t;
610
611 kcov_get(kcov);
612 return 0;
613 case KCOV_DISABLE:
614
615 unused = arg;
616 if (unused != 0 || current->kcov != kcov)
617 return -EINVAL;
618 t = current;
619 if (WARN_ON(kcov->t != t))
620 return -EINVAL;
621 kcov_disable(t, kcov);
622 kcov_put(kcov);
623 return 0;
624 case KCOV_REMOTE_ENABLE:
625 if (kcov->mode != KCOV_MODE_INIT || !kcov->area)
626 return -EINVAL;
627 t = current;
628 if (kcov->t != NULL || t->kcov != NULL)
629 return -EBUSY;
630 remote_arg = (struct kcov_remote_arg *)arg;
631 mode = kcov_get_mode(remote_arg->trace_mode);
632 if (mode < 0)
633 return mode;
634 if (remote_arg->area_size > LONG_MAX / sizeof(unsigned long))
635 return -EINVAL;
636 kcov->mode = mode;
637 t->kcov = kcov;
638 kcov->t = t;
639 kcov->remote = true;
640 kcov->remote_size = remote_arg->area_size;
641 spin_lock_irqsave(&kcov_remote_lock, flags);
642 for (i = 0; i < remote_arg->num_handles; i++) {
643 if (!kcov_check_handle(remote_arg->handles[i],
644 false, true, false)) {
645 spin_unlock_irqrestore(&kcov_remote_lock,
646 flags);
647 kcov_disable(t, kcov);
648 return -EINVAL;
649 }
650 remote = kcov_remote_add(kcov, remote_arg->handles[i]);
651 if (IS_ERR(remote)) {
652 spin_unlock_irqrestore(&kcov_remote_lock,
653 flags);
654 kcov_disable(t, kcov);
655 return PTR_ERR(remote);
656 }
657 }
658 if (remote_arg->common_handle) {
659 if (!kcov_check_handle(remote_arg->common_handle,
660 true, false, false)) {
661 spin_unlock_irqrestore(&kcov_remote_lock,
662 flags);
663 kcov_disable(t, kcov);
664 return -EINVAL;
665 }
666 remote = kcov_remote_add(kcov,
667 remote_arg->common_handle);
668 if (IS_ERR(remote)) {
669 spin_unlock_irqrestore(&kcov_remote_lock,
670 flags);
671 kcov_disable(t, kcov);
672 return PTR_ERR(remote);
673 }
674 t->kcov_handle = remote_arg->common_handle;
675 }
676 spin_unlock_irqrestore(&kcov_remote_lock, flags);
677
678 kcov_get(kcov);
679 return 0;
680 default:
681 return -ENOTTY;
682 }
683}
684
685static long kcov_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
686{
687 struct kcov *kcov;
688 int res;
689 struct kcov_remote_arg *remote_arg = NULL;
690 unsigned int remote_num_handles;
691 unsigned long remote_arg_size;
692 unsigned long flags;
693
694 if (cmd == KCOV_REMOTE_ENABLE) {
695 if (get_user(remote_num_handles, (unsigned __user *)(arg +
696 offsetof(struct kcov_remote_arg, num_handles))))
697 return -EFAULT;
698 if (remote_num_handles > KCOV_REMOTE_MAX_HANDLES)
699 return -EINVAL;
700 remote_arg_size = struct_size(remote_arg, handles,
701 remote_num_handles);
702 remote_arg = memdup_user((void __user *)arg, remote_arg_size);
703 if (IS_ERR(remote_arg))
704 return PTR_ERR(remote_arg);
705 if (remote_arg->num_handles != remote_num_handles) {
706 kfree(remote_arg);
707 return -EINVAL;
708 }
709 arg = (unsigned long)remote_arg;
710 }
711
712 kcov = filep->private_data;
713 spin_lock_irqsave(&kcov->lock, flags);
714 res = kcov_ioctl_locked(kcov, cmd, arg);
715 spin_unlock_irqrestore(&kcov->lock, flags);
716
717 kfree(remote_arg);
718
719 return res;
720}
721
722static const struct file_operations kcov_fops = {
723 .open = kcov_open,
724 .unlocked_ioctl = kcov_ioctl,
725 .compat_ioctl = kcov_ioctl,
726 .mmap = kcov_mmap,
727 .release = kcov_close,
728};
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773static inline bool kcov_mode_enabled(unsigned int mode)
774{
775 return (mode & ~KCOV_IN_CTXSW) != KCOV_MODE_DISABLED;
776}
777
778static void kcov_remote_softirq_start(struct task_struct *t)
779{
780 struct kcov_percpu_data *data = this_cpu_ptr(&kcov_percpu_data);
781 unsigned int mode;
782
783 mode = READ_ONCE(t->kcov_mode);
784 barrier();
785 if (kcov_mode_enabled(mode)) {
786 data->saved_mode = mode;
787 data->saved_size = t->kcov_size;
788 data->saved_area = t->kcov_area;
789 data->saved_sequence = t->kcov_sequence;
790 data->saved_kcov = t->kcov;
791 kcov_stop(t);
792 }
793}
794
795static void kcov_remote_softirq_stop(struct task_struct *t)
796{
797 struct kcov_percpu_data *data = this_cpu_ptr(&kcov_percpu_data);
798
799 if (data->saved_kcov) {
800 kcov_start(t, data->saved_kcov, data->saved_size,
801 data->saved_area, data->saved_mode,
802 data->saved_sequence);
803 data->saved_mode = 0;
804 data->saved_size = 0;
805 data->saved_area = NULL;
806 data->saved_sequence = 0;
807 data->saved_kcov = NULL;
808 }
809}
810
811void kcov_remote_start(u64 handle)
812{
813 struct task_struct *t = current;
814 struct kcov_remote *remote;
815 struct kcov *kcov;
816 unsigned int mode;
817 void *area;
818 unsigned int size;
819 int sequence;
820 unsigned long flags;
821
822 if (WARN_ON(!kcov_check_handle(handle, true, true, true)))
823 return;
824 if (!in_task() && !in_serving_softirq())
825 return;
826
827 local_irq_save(flags);
828
829
830
831
832
833 mode = READ_ONCE(t->kcov_mode);
834 if (WARN_ON(in_task() && kcov_mode_enabled(mode))) {
835 local_irq_restore(flags);
836 return;
837 }
838
839
840
841
842
843 if (WARN_ON(in_serving_softirq() && t->kcov_softirq)) {
844 local_irq_restore(flags);
845 return;
846 }
847
848 spin_lock(&kcov_remote_lock);
849 remote = kcov_remote_find(handle);
850 if (!remote) {
851 spin_unlock_irqrestore(&kcov_remote_lock, flags);
852 return;
853 }
854 kcov_debug("handle = %llx, context: %s\n", handle,
855 in_task() ? "task" : "softirq");
856 kcov = remote->kcov;
857
858 kcov_get(kcov);
859
860
861
862
863 mode = kcov->mode;
864 sequence = kcov->sequence;
865 if (in_task()) {
866 size = kcov->remote_size;
867 area = kcov_remote_area_get(size);
868 } else {
869 size = CONFIG_KCOV_IRQ_AREA_SIZE;
870 area = this_cpu_ptr(&kcov_percpu_data)->irq_area;
871 }
872 spin_unlock_irqrestore(&kcov_remote_lock, flags);
873
874
875 if (!area) {
876 area = vmalloc(size * sizeof(unsigned long));
877 if (!area) {
878 kcov_put(kcov);
879 return;
880 }
881 }
882
883 local_irq_save(flags);
884
885
886 *(u64 *)area = 0;
887
888 if (in_serving_softirq()) {
889 kcov_remote_softirq_start(t);
890 t->kcov_softirq = 1;
891 }
892 kcov_start(t, kcov, size, area, mode, sequence);
893
894 local_irq_restore(flags);
895
896}
897EXPORT_SYMBOL(kcov_remote_start);
898
899static void kcov_move_area(enum kcov_mode mode, void *dst_area,
900 unsigned int dst_area_size, void *src_area)
901{
902 u64 word_size = sizeof(unsigned long);
903 u64 count_size, entry_size_log;
904 u64 dst_len, src_len;
905 void *dst_entries, *src_entries;
906 u64 dst_occupied, dst_free, bytes_to_move, entries_moved;
907
908 kcov_debug("%px %u <= %px %lu\n",
909 dst_area, dst_area_size, src_area, *(unsigned long *)src_area);
910
911 switch (mode) {
912 case KCOV_MODE_TRACE_PC:
913 dst_len = READ_ONCE(*(unsigned long *)dst_area);
914 src_len = *(unsigned long *)src_area;
915 count_size = sizeof(unsigned long);
916 entry_size_log = __ilog2_u64(sizeof(unsigned long));
917 break;
918 case KCOV_MODE_TRACE_CMP:
919 dst_len = READ_ONCE(*(u64 *)dst_area);
920 src_len = *(u64 *)src_area;
921 count_size = sizeof(u64);
922 BUILD_BUG_ON(!is_power_of_2(KCOV_WORDS_PER_CMP));
923 entry_size_log = __ilog2_u64(sizeof(u64) * KCOV_WORDS_PER_CMP);
924 break;
925 default:
926 WARN_ON(1);
927 return;
928 }
929
930
931 if (dst_len > ((dst_area_size * word_size - count_size) >>
932 entry_size_log))
933 return;
934 dst_occupied = count_size + (dst_len << entry_size_log);
935 dst_free = dst_area_size * word_size - dst_occupied;
936 bytes_to_move = min(dst_free, src_len << entry_size_log);
937 dst_entries = dst_area + dst_occupied;
938 src_entries = src_area + count_size;
939 memcpy(dst_entries, src_entries, bytes_to_move);
940 entries_moved = bytes_to_move >> entry_size_log;
941
942 switch (mode) {
943 case KCOV_MODE_TRACE_PC:
944 WRITE_ONCE(*(unsigned long *)dst_area, dst_len + entries_moved);
945 break;
946 case KCOV_MODE_TRACE_CMP:
947 WRITE_ONCE(*(u64 *)dst_area, dst_len + entries_moved);
948 break;
949 default:
950 break;
951 }
952}
953
954
955void kcov_remote_stop(void)
956{
957 struct task_struct *t = current;
958 struct kcov *kcov;
959 unsigned int mode;
960 void *area;
961 unsigned int size;
962 int sequence;
963 unsigned long flags;
964
965 if (!in_task() && !in_serving_softirq())
966 return;
967
968 local_irq_save(flags);
969
970 mode = READ_ONCE(t->kcov_mode);
971 barrier();
972 if (!kcov_mode_enabled(mode)) {
973 local_irq_restore(flags);
974 return;
975 }
976
977
978
979
980 if (in_serving_softirq() && !t->kcov_softirq) {
981 local_irq_restore(flags);
982 return;
983 }
984
985 if (WARN_ON(!in_serving_softirq() && t->kcov_softirq)) {
986 local_irq_restore(flags);
987 return;
988 }
989
990 kcov = t->kcov;
991 area = t->kcov_area;
992 size = t->kcov_size;
993 sequence = t->kcov_sequence;
994
995 kcov_stop(t);
996 if (in_serving_softirq()) {
997 t->kcov_softirq = 0;
998 kcov_remote_softirq_stop(t);
999 }
1000
1001 spin_lock(&kcov->lock);
1002
1003
1004
1005
1006 if (sequence == kcov->sequence && kcov->remote)
1007 kcov_move_area(kcov->mode, kcov->area, kcov->size, area);
1008 spin_unlock(&kcov->lock);
1009
1010 if (in_task()) {
1011 spin_lock(&kcov_remote_lock);
1012 kcov_remote_area_put(area, size);
1013 spin_unlock(&kcov_remote_lock);
1014 }
1015
1016 local_irq_restore(flags);
1017
1018
1019 kcov_put(kcov);
1020}
1021EXPORT_SYMBOL(kcov_remote_stop);
1022
1023
1024u64 kcov_common_handle(void)
1025{
1026 if (!in_task())
1027 return 0;
1028 return current->kcov_handle;
1029}
1030EXPORT_SYMBOL(kcov_common_handle);
1031
1032static int __init kcov_init(void)
1033{
1034 int cpu;
1035
1036 for_each_possible_cpu(cpu) {
1037 void *area = vmalloc(CONFIG_KCOV_IRQ_AREA_SIZE *
1038 sizeof(unsigned long));
1039 if (!area)
1040 return -ENOMEM;
1041 per_cpu_ptr(&kcov_percpu_data, cpu)->irq_area = area;
1042 }
1043
1044
1045
1046
1047
1048
1049 debugfs_create_file_unsafe("kcov", 0600, NULL, NULL, &kcov_fops);
1050
1051 return 0;
1052}
1053
1054device_initcall(kcov_init);
1055