1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21#include <linux/dcookies.h>
22#include <linux/kref.h>
23#include <linux/mm.h>
24#include <linux/fs.h>
25#include <linux/module.h>
26#include <linux/notifier.h>
27#include <linux/numa.h>
28#include <linux/oprofile.h>
29#include <linux/slab.h>
30#include <linux/spinlock.h>
31#include "pr_util.h"
32
33#define RELEASE_ALL 9999
34
35static DEFINE_SPINLOCK(buffer_lock);
36static DEFINE_SPINLOCK(cache_lock);
37static int num_spu_nodes;
38int spu_prof_num_nodes;
39
40struct spu_buffer spu_buff[MAX_NUMNODES * SPUS_PER_NODE];
41struct delayed_work spu_work;
42static unsigned max_spu_buff;
43
44static void spu_buff_add(unsigned long int value, int spu)
45{
46
47
48
49
50
51
52
53
54
55
56 int full = 1;
57
58 if (spu_buff[spu].head >= spu_buff[spu].tail) {
59 if ((spu_buff[spu].head - spu_buff[spu].tail)
60 < (max_spu_buff - 1))
61 full = 0;
62
63 } else if (spu_buff[spu].tail > spu_buff[spu].head) {
64 if ((spu_buff[spu].tail - spu_buff[spu].head)
65 > 1)
66 full = 0;
67 }
68
69 if (!full) {
70 spu_buff[spu].buff[spu_buff[spu].head] = value;
71 spu_buff[spu].head++;
72
73 if (spu_buff[spu].head >= max_spu_buff)
74 spu_buff[spu].head = 0;
75 } else {
76
77
78
79
80
81
82
83 oprofile_cpu_buffer_inc_smpl_lost();
84 }
85}
86
87
88
89
90void sync_spu_buff(void)
91{
92 int spu;
93 unsigned long flags;
94 int curr_head;
95
96 for (spu = 0; spu < num_spu_nodes; spu++) {
97
98
99
100 if (spu_buff[spu].buff == NULL)
101 continue;
102
103
104
105
106
107
108 spin_lock_irqsave(&buffer_lock, flags);
109 curr_head = spu_buff[spu].head;
110 spin_unlock_irqrestore(&buffer_lock, flags);
111
112
113
114
115 oprofile_put_buff(spu_buff[spu].buff,
116 spu_buff[spu].tail,
117 curr_head, max_spu_buff);
118
119 spin_lock_irqsave(&buffer_lock, flags);
120 spu_buff[spu].tail = curr_head;
121 spin_unlock_irqrestore(&buffer_lock, flags);
122 }
123
124}
125
126static void wq_sync_spu_buff(struct work_struct *work)
127{
128
129 sync_spu_buff();
130
131
132 if (spu_prof_running)
133 schedule_delayed_work(&spu_work, DEFAULT_TIMER_EXPIRE);
134}
135
136
137struct cached_info {
138 struct vma_to_fileoffset_map *map;
139 struct spu *the_spu;
140 struct kref cache_ref;
141};
142
143static struct cached_info *spu_info[MAX_NUMNODES * 8];
144
145static void destroy_cached_info(struct kref *kref)
146{
147 struct cached_info *info;
148
149 info = container_of(kref, struct cached_info, cache_ref);
150 vma_map_free(info->map);
151 kfree(info);
152 module_put(THIS_MODULE);
153}
154
155
156
157
158
159static struct cached_info *get_cached_info(struct spu *the_spu, int spu_num)
160{
161 struct kref *ref;
162 struct cached_info *ret_info;
163
164 if (spu_num >= num_spu_nodes) {
165 printk(KERN_ERR "SPU_PROF: "
166 "%s, line %d: Invalid index %d into spu info cache\n",
167 __func__, __LINE__, spu_num);
168 ret_info = NULL;
169 goto out;
170 }
171 if (!spu_info[spu_num] && the_spu) {
172 ref = spu_get_profile_private_kref(the_spu->ctx);
173 if (ref) {
174 spu_info[spu_num] = container_of(ref, struct cached_info, cache_ref);
175 kref_get(&spu_info[spu_num]->cache_ref);
176 }
177 }
178
179 ret_info = spu_info[spu_num];
180 out:
181 return ret_info;
182}
183
184
185
186
187
188
189static int
190prepare_cached_spu_info(struct spu *spu, unsigned long objectId)
191{
192 unsigned long flags;
193 struct vma_to_fileoffset_map *new_map;
194 int retval = 0;
195 struct cached_info *info;
196
197
198
199
200 info = get_cached_info(spu, spu->number);
201
202 if (info) {
203 pr_debug("Found cached SPU info.\n");
204 goto out;
205 }
206
207
208
209
210 info = kzalloc(sizeof(struct cached_info), GFP_KERNEL);
211 if (!info) {
212 printk(KERN_ERR "SPU_PROF: "
213 "%s, line %d: create vma_map failed\n",
214 __func__, __LINE__);
215 retval = -ENOMEM;
216 goto err_alloc;
217 }
218 new_map = create_vma_map(spu, objectId);
219 if (!new_map) {
220 printk(KERN_ERR "SPU_PROF: "
221 "%s, line %d: create vma_map failed\n",
222 __func__, __LINE__);
223 retval = -ENOMEM;
224 goto err_alloc;
225 }
226
227 pr_debug("Created vma_map\n");
228 info->map = new_map;
229 info->the_spu = spu;
230 kref_init(&info->cache_ref);
231 spin_lock_irqsave(&cache_lock, flags);
232 spu_info[spu->number] = info;
233
234 kref_get(&info->cache_ref);
235
236
237
238
239
240
241
242 try_module_get(THIS_MODULE);
243 spu_set_profile_private_kref(spu->ctx, &info->cache_ref,
244 destroy_cached_info);
245 spin_unlock_irqrestore(&cache_lock, flags);
246 goto out;
247
248err_alloc:
249 kfree(info);
250out:
251 return retval;
252}
253
254
255
256
257
258static int release_cached_info(int spu_index)
259{
260 int index, end;
261
262 if (spu_index == RELEASE_ALL) {
263 end = num_spu_nodes;
264 index = 0;
265 } else {
266 if (spu_index >= num_spu_nodes) {
267 printk(KERN_ERR "SPU_PROF: "
268 "%s, line %d: "
269 "Invalid index %d into spu info cache\n",
270 __func__, __LINE__, spu_index);
271 goto out;
272 }
273 end = spu_index + 1;
274 index = spu_index;
275 }
276 for (; index < end; index++) {
277 if (spu_info[index]) {
278 kref_put(&spu_info[index]->cache_ref,
279 destroy_cached_info);
280 spu_info[index] = NULL;
281 }
282 }
283
284out:
285 return 0;
286}
287
288
289
290
291
292
293
294
295
296
297static inline unsigned long fast_get_dcookie(struct path *path)
298{
299 unsigned long cookie;
300
301 if (path->dentry->d_flags & DCACHE_COOKIE)
302 return (unsigned long)path->dentry;
303 get_dcookie(path, &cookie);
304 return cookie;
305}
306
307
308
309
310
311
312
313
314
315
316
317static unsigned long
318get_exec_dcookie_and_offset(struct spu *spu, unsigned int *offsetp,
319 unsigned long *spu_bin_dcookie,
320 unsigned long spu_ref)
321{
322 unsigned long app_cookie = 0;
323 unsigned int my_offset = 0;
324 struct file *app = NULL;
325 struct vm_area_struct *vma;
326 struct mm_struct *mm = spu->mm;
327
328 if (!mm)
329 goto out;
330
331 down_read(&mm->mmap_sem);
332
333 for (vma = mm->mmap; vma; vma = vma->vm_next) {
334 if (!vma->vm_file)
335 continue;
336 if (!(vma->vm_flags & VM_EXECUTABLE))
337 continue;
338 app_cookie = fast_get_dcookie(&vma->vm_file->f_path);
339 pr_debug("got dcookie for %s\n",
340 vma->vm_file->f_dentry->d_name.name);
341 app = vma->vm_file;
342 break;
343 }
344
345 for (vma = mm->mmap; vma; vma = vma->vm_next) {
346 if (vma->vm_start > spu_ref || vma->vm_end <= spu_ref)
347 continue;
348 my_offset = spu_ref - vma->vm_start;
349 if (!vma->vm_file)
350 goto fail_no_image_cookie;
351
352 pr_debug("Found spu ELF at %X(object-id:%lx) for file %s\n",
353 my_offset, spu_ref,
354 vma->vm_file->f_dentry->d_name.name);
355 *offsetp = my_offset;
356 break;
357 }
358
359 *spu_bin_dcookie = fast_get_dcookie(&vma->vm_file->f_path);
360 pr_debug("got dcookie for %s\n", vma->vm_file->f_dentry->d_name.name);
361
362 up_read(&mm->mmap_sem);
363
364out:
365 return app_cookie;
366
367fail_no_image_cookie:
368 up_read(&mm->mmap_sem);
369
370 printk(KERN_ERR "SPU_PROF: "
371 "%s, line %d: Cannot find dcookie for SPU binary\n",
372 __func__, __LINE__);
373 goto out;
374}
375
376
377
378
379
380
381
382static int process_context_switch(struct spu *spu, unsigned long objectId)
383{
384 unsigned long flags;
385 int retval;
386 unsigned int offset = 0;
387 unsigned long spu_cookie = 0, app_dcookie;
388
389 retval = prepare_cached_spu_info(spu, objectId);
390 if (retval)
391 goto out;
392
393
394
395
396 app_dcookie = get_exec_dcookie_and_offset(spu, &offset, &spu_cookie, objectId);
397 if (!app_dcookie || !spu_cookie) {
398 retval = -ENOENT;
399 goto out;
400 }
401
402
403 spin_lock_irqsave(&buffer_lock, flags);
404 spu_buff_add(ESCAPE_CODE, spu->number);
405 spu_buff_add(SPU_CTX_SWITCH_CODE, spu->number);
406 spu_buff_add(spu->number, spu->number);
407 spu_buff_add(spu->pid, spu->number);
408 spu_buff_add(spu->tgid, spu->number);
409 spu_buff_add(app_dcookie, spu->number);
410 spu_buff_add(spu_cookie, spu->number);
411 spu_buff_add(offset, spu->number);
412
413
414
415
416
417 spu_buff[spu->number].ctx_sw_seen = 1;
418
419 spin_unlock_irqrestore(&buffer_lock, flags);
420 smp_wmb();
421
422out:
423 return retval;
424}
425
426
427
428
429
430
431
432static int spu_active_notify(struct notifier_block *self, unsigned long val,
433 void *data)
434{
435 int retval;
436 unsigned long flags;
437 struct spu *the_spu = data;
438
439 pr_debug("SPU event notification arrived\n");
440 if (!val) {
441 spin_lock_irqsave(&cache_lock, flags);
442 retval = release_cached_info(the_spu->number);
443 spin_unlock_irqrestore(&cache_lock, flags);
444 } else {
445 retval = process_context_switch(the_spu, val);
446 }
447 return retval;
448}
449
450static struct notifier_block spu_active = {
451 .notifier_call = spu_active_notify,
452};
453
454static int number_of_online_nodes(void)
455{
456 u32 cpu; u32 tmp;
457 int nodes = 0;
458 for_each_online_cpu(cpu) {
459 tmp = cbe_cpu_to_node(cpu) + 1;
460 if (tmp > nodes)
461 nodes++;
462 }
463 return nodes;
464}
465
466static int oprofile_spu_buff_create(void)
467{
468 int spu;
469
470 max_spu_buff = oprofile_get_cpu_buffer_size();
471
472 for (spu = 0; spu < num_spu_nodes; spu++) {
473
474
475
476 spu_buff[spu].head = 0;
477 spu_buff[spu].tail = 0;
478
479
480
481
482
483
484
485 spu_buff[spu].buff = kzalloc((max_spu_buff
486 * sizeof(unsigned long)),
487 GFP_KERNEL);
488
489 if (!spu_buff[spu].buff) {
490 printk(KERN_ERR "SPU_PROF: "
491 "%s, line %d: oprofile_spu_buff_create "
492 "failed to allocate spu buffer %d.\n",
493 __func__, __LINE__, spu);
494
495
496 while (spu >= 0) {
497 kfree(spu_buff[spu].buff);
498 spu_buff[spu].buff = 0;
499 spu--;
500 }
501 return -ENOMEM;
502 }
503 }
504 return 0;
505}
506
507
508
509
510
511
512
513
514
515
516
517int spu_sync_start(void)
518{
519 int spu;
520 int ret = SKIP_GENERIC_SYNC;
521 int register_ret;
522 unsigned long flags = 0;
523
524 spu_prof_num_nodes = number_of_online_nodes();
525 num_spu_nodes = spu_prof_num_nodes * 8;
526 INIT_DELAYED_WORK(&spu_work, wq_sync_spu_buff);
527
528
529
530
531 ret = oprofile_spu_buff_create();
532 if (ret)
533 goto out;
534
535 spin_lock_irqsave(&buffer_lock, flags);
536 for (spu = 0; spu < num_spu_nodes; spu++) {
537 spu_buff_add(ESCAPE_CODE, spu);
538 spu_buff_add(SPU_PROFILING_CODE, spu);
539 spu_buff_add(num_spu_nodes, spu);
540 }
541 spin_unlock_irqrestore(&buffer_lock, flags);
542
543 for (spu = 0; spu < num_spu_nodes; spu++) {
544 spu_buff[spu].ctx_sw_seen = 0;
545 spu_buff[spu].last_guard_val = 0;
546 }
547
548
549 register_ret = spu_switch_event_register(&spu_active);
550 if (register_ret) {
551 ret = SYNC_START_ERROR;
552 goto out;
553 }
554
555 pr_debug("spu_sync_start -- running.\n");
556out:
557 return ret;
558}
559
560
561void spu_sync_buffer(int spu_num, unsigned int *samples,
562 int num_samples)
563{
564 unsigned long long file_offset;
565 unsigned long flags;
566 int i;
567 struct vma_to_fileoffset_map *map;
568 struct spu *the_spu;
569 unsigned long long spu_num_ll = spu_num;
570 unsigned long long spu_num_shifted = spu_num_ll << 32;
571 struct cached_info *c_info;
572
573
574
575
576
577
578 spin_lock_irqsave(&cache_lock, flags);
579 c_info = get_cached_info(NULL, spu_num);
580 if (!c_info) {
581
582
583
584
585 pr_debug("SPU_PROF: No cached SPU contex "
586 "for SPU #%d. Dropping samples.\n", spu_num);
587 goto out;
588 }
589
590 map = c_info->map;
591 the_spu = c_info->the_spu;
592 spin_lock(&buffer_lock);
593 for (i = 0; i < num_samples; i++) {
594 unsigned int sample = *(samples+i);
595 int grd_val = 0;
596 file_offset = 0;
597 if (sample == 0)
598 continue;
599 file_offset = vma_map_lookup( map, sample, the_spu, &grd_val);
600
601
602
603
604
605
606 if (grd_val && grd_val != spu_buff[spu_num].last_guard_val) {
607 spu_buff[spu_num].last_guard_val = grd_val;
608
609 break;
610 }
611
612
613
614
615
616
617 if (spu_buff[spu_num].ctx_sw_seen)
618 spu_buff_add((file_offset | spu_num_shifted),
619 spu_num);
620 }
621 spin_unlock(&buffer_lock);
622out:
623 spin_unlock_irqrestore(&cache_lock, flags);
624}
625
626
627int spu_sync_stop(void)
628{
629 unsigned long flags = 0;
630 int ret;
631 int k;
632
633 ret = spu_switch_event_unregister(&spu_active);
634
635 if (ret)
636 printk(KERN_ERR "SPU_PROF: "
637 "%s, line %d: spu_switch_event_unregister " \
638 "returned %d\n",
639 __func__, __LINE__, ret);
640
641
642 sync_spu_buff();
643
644 spin_lock_irqsave(&cache_lock, flags);
645 ret = release_cached_info(RELEASE_ALL);
646 spin_unlock_irqrestore(&cache_lock, flags);
647
648
649
650
651
652 cancel_delayed_work(&spu_work);
653
654 for (k = 0; k < num_spu_nodes; k++) {
655 spu_buff[k].ctx_sw_seen = 0;
656
657
658
659
660
661 kfree(spu_buff[k].buff);
662 spu_buff[k].buff = 0;
663 }
664 pr_debug("spu_sync_stop -- done.\n");
665 return ret;
666}
667
668