1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17#include <linux/export.h>
18#include <linux/profile.h>
19#include <linux/bootmem.h>
20#include <linux/notifier.h>
21#include <linux/mm.h>
22#include <linux/cpumask.h>
23#include <linux/cpu.h>
24#include <linux/highmem.h>
25#include <linux/mutex.h>
26#include <linux/slab.h>
27#include <linux/vmalloc.h>
28#include <linux/sched/stat.h>
29
30#include <asm/sections.h>
31#include <asm/irq_regs.h>
32#include <asm/ptrace.h>
33
34struct profile_hit {
35 u32 pc, hits;
36};
37#define PROFILE_GRPSHIFT 3
38#define PROFILE_GRPSZ (1 << PROFILE_GRPSHIFT)
39#define NR_PROFILE_HIT (PAGE_SIZE/sizeof(struct profile_hit))
40#define NR_PROFILE_GRP (NR_PROFILE_HIT/PROFILE_GRPSZ)
41
42static atomic_t *prof_buffer;
43static unsigned long prof_len, prof_shift;
44
45int prof_on __read_mostly;
46EXPORT_SYMBOL_GPL(prof_on);
47
48static cpumask_var_t prof_cpu_mask;
49#if defined(CONFIG_SMP) && defined(CONFIG_PROC_FS)
50static DEFINE_PER_CPU(struct profile_hit *[2], cpu_profile_hits);
51static DEFINE_PER_CPU(int, cpu_profile_flip);
52static DEFINE_MUTEX(profile_flip_mutex);
53#endif
54
55int profile_setup(char *str)
56{
57 static const char schedstr[] = "schedule";
58 static const char sleepstr[] = "sleep";
59 static const char kvmstr[] = "kvm";
60 int par;
61
62 if (!strncmp(str, sleepstr, strlen(sleepstr))) {
63#ifdef CONFIG_SCHEDSTATS
64 force_schedstat_enabled();
65 prof_on = SLEEP_PROFILING;
66 if (str[strlen(sleepstr)] == ',')
67 str += strlen(sleepstr) + 1;
68 if (get_option(&str, &par))
69 prof_shift = par;
70 pr_info("kernel sleep profiling enabled (shift: %ld)\n",
71 prof_shift);
72#else
73 pr_warn("kernel sleep profiling requires CONFIG_SCHEDSTATS\n");
74#endif
75 } else if (!strncmp(str, schedstr, strlen(schedstr))) {
76 prof_on = SCHED_PROFILING;
77 if (str[strlen(schedstr)] == ',')
78 str += strlen(schedstr) + 1;
79 if (get_option(&str, &par))
80 prof_shift = par;
81 pr_info("kernel schedule profiling enabled (shift: %ld)\n",
82 prof_shift);
83 } else if (!strncmp(str, kvmstr, strlen(kvmstr))) {
84 prof_on = KVM_PROFILING;
85 if (str[strlen(kvmstr)] == ',')
86 str += strlen(kvmstr) + 1;
87 if (get_option(&str, &par))
88 prof_shift = par;
89 pr_info("kernel KVM profiling enabled (shift: %ld)\n",
90 prof_shift);
91 } else if (get_option(&str, &par)) {
92 prof_shift = par;
93 prof_on = CPU_PROFILING;
94 pr_info("kernel profiling enabled (shift: %ld)\n",
95 prof_shift);
96 }
97 return 1;
98}
99__setup("profile=", profile_setup);
100
101
102int __ref profile_init(void)
103{
104 int buffer_bytes;
105 if (!prof_on)
106 return 0;
107
108
109 prof_len = (_etext - _stext) >> prof_shift;
110 buffer_bytes = prof_len*sizeof(atomic_t);
111
112 if (!alloc_cpumask_var(&prof_cpu_mask, GFP_KERNEL))
113 return -ENOMEM;
114
115 cpumask_copy(prof_cpu_mask, cpu_possible_mask);
116
117 prof_buffer = kzalloc(buffer_bytes, GFP_KERNEL|__GFP_NOWARN);
118 if (prof_buffer)
119 return 0;
120
121 prof_buffer = alloc_pages_exact(buffer_bytes,
122 GFP_KERNEL|__GFP_ZERO|__GFP_NOWARN);
123 if (prof_buffer)
124 return 0;
125
126 prof_buffer = vzalloc(buffer_bytes);
127 if (prof_buffer)
128 return 0;
129
130 free_cpumask_var(prof_cpu_mask);
131 return -ENOMEM;
132}
133
134
135
136static BLOCKING_NOTIFIER_HEAD(task_exit_notifier);
137static ATOMIC_NOTIFIER_HEAD(task_free_notifier);
138static BLOCKING_NOTIFIER_HEAD(munmap_notifier);
139
140void profile_task_exit(struct task_struct *task)
141{
142 blocking_notifier_call_chain(&task_exit_notifier, 0, task);
143}
144
145int profile_handoff_task(struct task_struct *task)
146{
147 int ret;
148 ret = atomic_notifier_call_chain(&task_free_notifier, 0, task);
149 return (ret == NOTIFY_OK) ? 1 : 0;
150}
151
152void profile_munmap(unsigned long addr)
153{
154 blocking_notifier_call_chain(&munmap_notifier, 0, (void *)addr);
155}
156
157int task_handoff_register(struct notifier_block *n)
158{
159 return atomic_notifier_chain_register(&task_free_notifier, n);
160}
161EXPORT_SYMBOL_GPL(task_handoff_register);
162
163int task_handoff_unregister(struct notifier_block *n)
164{
165 return atomic_notifier_chain_unregister(&task_free_notifier, n);
166}
167EXPORT_SYMBOL_GPL(task_handoff_unregister);
168
169int profile_event_register(enum profile_type type, struct notifier_block *n)
170{
171 int err = -EINVAL;
172
173 switch (type) {
174 case PROFILE_TASK_EXIT:
175 err = blocking_notifier_chain_register(
176 &task_exit_notifier, n);
177 break;
178 case PROFILE_MUNMAP:
179 err = blocking_notifier_chain_register(
180 &munmap_notifier, n);
181 break;
182 }
183
184 return err;
185}
186EXPORT_SYMBOL_GPL(profile_event_register);
187
188int profile_event_unregister(enum profile_type type, struct notifier_block *n)
189{
190 int err = -EINVAL;
191
192 switch (type) {
193 case PROFILE_TASK_EXIT:
194 err = blocking_notifier_chain_unregister(
195 &task_exit_notifier, n);
196 break;
197 case PROFILE_MUNMAP:
198 err = blocking_notifier_chain_unregister(
199 &munmap_notifier, n);
200 break;
201 }
202
203 return err;
204}
205EXPORT_SYMBOL_GPL(profile_event_unregister);
206
207#if defined(CONFIG_SMP) && defined(CONFIG_PROC_FS)
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239static void __profile_flip_buffers(void *unused)
240{
241 int cpu = smp_processor_id();
242
243 per_cpu(cpu_profile_flip, cpu) = !per_cpu(cpu_profile_flip, cpu);
244}
245
246static void profile_flip_buffers(void)
247{
248 int i, j, cpu;
249
250 mutex_lock(&profile_flip_mutex);
251 j = per_cpu(cpu_profile_flip, get_cpu());
252 put_cpu();
253 on_each_cpu(__profile_flip_buffers, NULL, 1);
254 for_each_online_cpu(cpu) {
255 struct profile_hit *hits = per_cpu(cpu_profile_hits, cpu)[j];
256 for (i = 0; i < NR_PROFILE_HIT; ++i) {
257 if (!hits[i].hits) {
258 if (hits[i].pc)
259 hits[i].pc = 0;
260 continue;
261 }
262 atomic_add(hits[i].hits, &prof_buffer[hits[i].pc]);
263 hits[i].hits = hits[i].pc = 0;
264 }
265 }
266 mutex_unlock(&profile_flip_mutex);
267}
268
269static void profile_discard_flip_buffers(void)
270{
271 int i, cpu;
272
273 mutex_lock(&profile_flip_mutex);
274 i = per_cpu(cpu_profile_flip, get_cpu());
275 put_cpu();
276 on_each_cpu(__profile_flip_buffers, NULL, 1);
277 for_each_online_cpu(cpu) {
278 struct profile_hit *hits = per_cpu(cpu_profile_hits, cpu)[i];
279 memset(hits, 0, NR_PROFILE_HIT*sizeof(struct profile_hit));
280 }
281 mutex_unlock(&profile_flip_mutex);
282}
283
284static void do_profile_hits(int type, void *__pc, unsigned int nr_hits)
285{
286 unsigned long primary, secondary, flags, pc = (unsigned long)__pc;
287 int i, j, cpu;
288 struct profile_hit *hits;
289
290 pc = min((pc - (unsigned long)_stext) >> prof_shift, prof_len - 1);
291 i = primary = (pc & (NR_PROFILE_GRP - 1)) << PROFILE_GRPSHIFT;
292 secondary = (~(pc << 1) & (NR_PROFILE_GRP - 1)) << PROFILE_GRPSHIFT;
293 cpu = get_cpu();
294 hits = per_cpu(cpu_profile_hits, cpu)[per_cpu(cpu_profile_flip, cpu)];
295 if (!hits) {
296 put_cpu();
297 return;
298 }
299
300
301
302
303
304 local_irq_save(flags);
305 do {
306 for (j = 0; j < PROFILE_GRPSZ; ++j) {
307 if (hits[i + j].pc == pc) {
308 hits[i + j].hits += nr_hits;
309 goto out;
310 } else if (!hits[i + j].hits) {
311 hits[i + j].pc = pc;
312 hits[i + j].hits = nr_hits;
313 goto out;
314 }
315 }
316 i = (i + secondary) & (NR_PROFILE_HIT - 1);
317 } while (i != primary);
318
319
320
321
322
323 atomic_add(nr_hits, &prof_buffer[pc]);
324 for (i = 0; i < NR_PROFILE_HIT; ++i) {
325 atomic_add(hits[i].hits, &prof_buffer[hits[i].pc]);
326 hits[i].pc = hits[i].hits = 0;
327 }
328out:
329 local_irq_restore(flags);
330 put_cpu();
331}
332
333static int profile_dead_cpu(unsigned int cpu)
334{
335 struct page *page;
336 int i;
337
338 if (prof_cpu_mask != NULL)
339 cpumask_clear_cpu(cpu, prof_cpu_mask);
340
341 for (i = 0; i < 2; i++) {
342 if (per_cpu(cpu_profile_hits, cpu)[i]) {
343 page = virt_to_page(per_cpu(cpu_profile_hits, cpu)[i]);
344 per_cpu(cpu_profile_hits, cpu)[i] = NULL;
345 __free_page(page);
346 }
347 }
348 return 0;
349}
350
351static int profile_prepare_cpu(unsigned int cpu)
352{
353 int i, node = cpu_to_mem(cpu);
354 struct page *page;
355
356 per_cpu(cpu_profile_flip, cpu) = 0;
357
358 for (i = 0; i < 2; i++) {
359 if (per_cpu(cpu_profile_hits, cpu)[i])
360 continue;
361
362 page = __alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0);
363 if (!page) {
364 profile_dead_cpu(cpu);
365 return -ENOMEM;
366 }
367 per_cpu(cpu_profile_hits, cpu)[i] = page_address(page);
368
369 }
370 return 0;
371}
372
373static int profile_online_cpu(unsigned int cpu)
374{
375 if (prof_cpu_mask != NULL)
376 cpumask_set_cpu(cpu, prof_cpu_mask);
377
378 return 0;
379}
380
381#else
382#define profile_flip_buffers() do { } while (0)
383#define profile_discard_flip_buffers() do { } while (0)
384
385static void do_profile_hits(int type, void *__pc, unsigned int nr_hits)
386{
387 unsigned long pc;
388 pc = ((unsigned long)__pc - (unsigned long)_stext) >> prof_shift;
389 atomic_add(nr_hits, &prof_buffer[min(pc, prof_len - 1)]);
390}
391#endif
392
393void profile_hits(int type, void *__pc, unsigned int nr_hits)
394{
395 if (prof_on != type || !prof_buffer)
396 return;
397 do_profile_hits(type, __pc, nr_hits);
398}
399EXPORT_SYMBOL_GPL(profile_hits);
400
401void profile_tick(int type)
402{
403 struct pt_regs *regs = get_irq_regs();
404
405 if (!user_mode(regs) && prof_cpu_mask != NULL &&
406 cpumask_test_cpu(smp_processor_id(), prof_cpu_mask))
407 profile_hit(type, (void *)profile_pc(regs));
408}
409
410#ifdef CONFIG_PROC_FS
411#include <linux/proc_fs.h>
412#include <linux/seq_file.h>
413#include <linux/uaccess.h>
414
415static int prof_cpu_mask_proc_show(struct seq_file *m, void *v)
416{
417 seq_printf(m, "%*pb\n", cpumask_pr_args(prof_cpu_mask));
418 return 0;
419}
420
421static int prof_cpu_mask_proc_open(struct inode *inode, struct file *file)
422{
423 return single_open(file, prof_cpu_mask_proc_show, NULL);
424}
425
426static ssize_t prof_cpu_mask_proc_write(struct file *file,
427 const char __user *buffer, size_t count, loff_t *pos)
428{
429 cpumask_var_t new_value;
430 int err;
431
432 if (!alloc_cpumask_var(&new_value, GFP_KERNEL))
433 return -ENOMEM;
434
435 err = cpumask_parse_user(buffer, count, new_value);
436 if (!err) {
437 cpumask_copy(prof_cpu_mask, new_value);
438 err = count;
439 }
440 free_cpumask_var(new_value);
441 return err;
442}
443
444static const struct file_operations prof_cpu_mask_proc_fops = {
445 .open = prof_cpu_mask_proc_open,
446 .read = seq_read,
447 .llseek = seq_lseek,
448 .release = single_release,
449 .write = prof_cpu_mask_proc_write,
450};
451
452void create_prof_cpu_mask(void)
453{
454
455 proc_create("irq/prof_cpu_mask", 0600, NULL, &prof_cpu_mask_proc_fops);
456}
457
458
459
460
461
462
463
464static ssize_t
465read_profile(struct file *file, char __user *buf, size_t count, loff_t *ppos)
466{
467 unsigned long p = *ppos;
468 ssize_t read;
469 char *pnt;
470 unsigned int sample_step = 1 << prof_shift;
471
472 profile_flip_buffers();
473 if (p >= (prof_len+1)*sizeof(unsigned int))
474 return 0;
475 if (count > (prof_len+1)*sizeof(unsigned int) - p)
476 count = (prof_len+1)*sizeof(unsigned int) - p;
477 read = 0;
478
479 while (p < sizeof(unsigned int) && count > 0) {
480 if (put_user(*((char *)(&sample_step)+p), buf))
481 return -EFAULT;
482 buf++; p++; count--; read++;
483 }
484 pnt = (char *)prof_buffer + p - sizeof(atomic_t);
485 if (copy_to_user(buf, (void *)pnt, count))
486 return -EFAULT;
487 read += count;
488 *ppos += read;
489 return read;
490}
491
492
493
494
495
496
497
498static ssize_t write_profile(struct file *file, const char __user *buf,
499 size_t count, loff_t *ppos)
500{
501#ifdef CONFIG_SMP
502 extern int setup_profiling_timer(unsigned int multiplier);
503
504 if (count == sizeof(int)) {
505 unsigned int multiplier;
506
507 if (copy_from_user(&multiplier, buf, sizeof(int)))
508 return -EFAULT;
509
510 if (setup_profiling_timer(multiplier))
511 return -EINVAL;
512 }
513#endif
514 profile_discard_flip_buffers();
515 memset(prof_buffer, 0, prof_len * sizeof(atomic_t));
516 return count;
517}
518
519static const struct file_operations proc_profile_operations = {
520 .read = read_profile,
521 .write = write_profile,
522 .llseek = default_llseek,
523};
524
525int __ref create_proc_profile(void)
526{
527 struct proc_dir_entry *entry;
528#ifdef CONFIG_SMP
529 enum cpuhp_state online_state;
530#endif
531
532 int err = 0;
533
534 if (!prof_on)
535 return 0;
536#ifdef CONFIG_SMP
537 err = cpuhp_setup_state(CPUHP_PROFILE_PREPARE, "PROFILE_PREPARE",
538 profile_prepare_cpu, profile_dead_cpu);
539 if (err)
540 return err;
541
542 err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "AP_PROFILE_ONLINE",
543 profile_online_cpu, NULL);
544 if (err < 0)
545 goto err_state_prep;
546 online_state = err;
547 err = 0;
548#endif
549 entry = proc_create("profile", S_IWUSR | S_IRUGO,
550 NULL, &proc_profile_operations);
551 if (!entry)
552 goto err_state_onl;
553 proc_set_size(entry, (1 + prof_len) * sizeof(atomic_t));
554
555 return err;
556err_state_onl:
557#ifdef CONFIG_SMP
558 cpuhp_remove_state(online_state);
559err_state_prep:
560 cpuhp_remove_state(CPUHP_PROFILE_PREPARE);
561#endif
562 return err;
563}
564subsys_initcall(create_proc_profile);
565#endif
566