1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21#include <linux/dcookies.h>
22#include <linux/kref.h>
23#include <linux/mm.h>
24#include <linux/fs.h>
25#include <linux/module.h>
26#include <linux/notifier.h>
27#include <linux/numa.h>
28#include <linux/oprofile.h>
29#include <linux/spinlock.h>
30#include "pr_util.h"
31
32#define RELEASE_ALL 9999
33
34static DEFINE_SPINLOCK(buffer_lock);
35static DEFINE_SPINLOCK(cache_lock);
36static int num_spu_nodes;
37int spu_prof_num_nodes;
38
39struct spu_buffer spu_buff[MAX_NUMNODES * SPUS_PER_NODE];
40struct delayed_work spu_work;
41static unsigned max_spu_buff;
42
43static void spu_buff_add(unsigned long int value, int spu)
44{
45
46
47
48
49
50
51
52
53
54
55 int full = 1;
56
57 if (spu_buff[spu].head >= spu_buff[spu].tail) {
58 if ((spu_buff[spu].head - spu_buff[spu].tail)
59 < (max_spu_buff - 1))
60 full = 0;
61
62 } else if (spu_buff[spu].tail > spu_buff[spu].head) {
63 if ((spu_buff[spu].tail - spu_buff[spu].head)
64 > 1)
65 full = 0;
66 }
67
68 if (!full) {
69 spu_buff[spu].buff[spu_buff[spu].head] = value;
70 spu_buff[spu].head++;
71
72 if (spu_buff[spu].head >= max_spu_buff)
73 spu_buff[spu].head = 0;
74 } else {
75
76
77
78
79
80
81
82 oprofile_cpu_buffer_inc_smpl_lost();
83 }
84}
85
86
87
88
89void sync_spu_buff(void)
90{
91 int spu;
92 unsigned long flags;
93 int curr_head;
94
95 for (spu = 0; spu < num_spu_nodes; spu++) {
96
97
98
99 if (spu_buff[spu].buff == NULL)
100 continue;
101
102
103
104
105
106
107 spin_lock_irqsave(&buffer_lock, flags);
108 curr_head = spu_buff[spu].head;
109 spin_unlock_irqrestore(&buffer_lock, flags);
110
111
112
113
114 oprofile_put_buff(spu_buff[spu].buff,
115 spu_buff[spu].tail,
116 curr_head, max_spu_buff);
117
118 spin_lock_irqsave(&buffer_lock, flags);
119 spu_buff[spu].tail = curr_head;
120 spin_unlock_irqrestore(&buffer_lock, flags);
121 }
122
123}
124
125static void wq_sync_spu_buff(struct work_struct *work)
126{
127
128 sync_spu_buff();
129
130
131 if (spu_prof_running)
132 schedule_delayed_work(&spu_work, DEFAULT_TIMER_EXPIRE);
133}
134
135
136struct cached_info {
137 struct vma_to_fileoffset_map *map;
138 struct spu *the_spu;
139 struct kref cache_ref;
140};
141
142static struct cached_info *spu_info[MAX_NUMNODES * 8];
143
144static void destroy_cached_info(struct kref *kref)
145{
146 struct cached_info *info;
147
148 info = container_of(kref, struct cached_info, cache_ref);
149 vma_map_free(info->map);
150 kfree(info);
151 module_put(THIS_MODULE);
152}
153
154
155
156
157
158static struct cached_info *get_cached_info(struct spu *the_spu, int spu_num)
159{
160 struct kref *ref;
161 struct cached_info *ret_info;
162
163 if (spu_num >= num_spu_nodes) {
164 printk(KERN_ERR "SPU_PROF: "
165 "%s, line %d: Invalid index %d into spu info cache\n",
166 __func__, __LINE__, spu_num);
167 ret_info = NULL;
168 goto out;
169 }
170 if (!spu_info[spu_num] && the_spu) {
171 ref = spu_get_profile_private_kref(the_spu->ctx);
172 if (ref) {
173 spu_info[spu_num] = container_of(ref, struct cached_info, cache_ref);
174 kref_get(&spu_info[spu_num]->cache_ref);
175 }
176 }
177
178 ret_info = spu_info[spu_num];
179 out:
180 return ret_info;
181}
182
183
184
185
186
187
188static int
189prepare_cached_spu_info(struct spu *spu, unsigned long objectId)
190{
191 unsigned long flags;
192 struct vma_to_fileoffset_map *new_map;
193 int retval = 0;
194 struct cached_info *info;
195
196
197
198
199 info = get_cached_info(spu, spu->number);
200
201 if (info) {
202 pr_debug("Found cached SPU info.\n");
203 goto out;
204 }
205
206
207
208
209 info = kzalloc(sizeof(struct cached_info), GFP_KERNEL);
210 if (!info) {
211 printk(KERN_ERR "SPU_PROF: "
212 "%s, line %d: create vma_map failed\n",
213 __func__, __LINE__);
214 retval = -ENOMEM;
215 goto err_alloc;
216 }
217 new_map = create_vma_map(spu, objectId);
218 if (!new_map) {
219 printk(KERN_ERR "SPU_PROF: "
220 "%s, line %d: create vma_map failed\n",
221 __func__, __LINE__);
222 retval = -ENOMEM;
223 goto err_alloc;
224 }
225
226 pr_debug("Created vma_map\n");
227 info->map = new_map;
228 info->the_spu = spu;
229 kref_init(&info->cache_ref);
230 spin_lock_irqsave(&cache_lock, flags);
231 spu_info[spu->number] = info;
232
233 kref_get(&info->cache_ref);
234
235
236
237
238
239
240
241 try_module_get(THIS_MODULE);
242 spu_set_profile_private_kref(spu->ctx, &info->cache_ref,
243 destroy_cached_info);
244 spin_unlock_irqrestore(&cache_lock, flags);
245 goto out;
246
247err_alloc:
248 kfree(info);
249out:
250 return retval;
251}
252
253
254
255
256
257static int release_cached_info(int spu_index)
258{
259 int index, end;
260
261 if (spu_index == RELEASE_ALL) {
262 end = num_spu_nodes;
263 index = 0;
264 } else {
265 if (spu_index >= num_spu_nodes) {
266 printk(KERN_ERR "SPU_PROF: "
267 "%s, line %d: "
268 "Invalid index %d into spu info cache\n",
269 __func__, __LINE__, spu_index);
270 goto out;
271 }
272 end = spu_index + 1;
273 index = spu_index;
274 }
275 for (; index < end; index++) {
276 if (spu_info[index]) {
277 kref_put(&spu_info[index]->cache_ref,
278 destroy_cached_info);
279 spu_info[index] = NULL;
280 }
281 }
282
283out:
284 return 0;
285}
286
287
288
289
290
291
292
293
294
295
296static inline unsigned long fast_get_dcookie(struct path *path)
297{
298 unsigned long cookie;
299
300 if (path->dentry->d_flags & DCACHE_COOKIE)
301 return (unsigned long)path->dentry;
302 get_dcookie(path, &cookie);
303 return cookie;
304}
305
306
307
308
309
310
311
312
313
314
315
316static unsigned long
317get_exec_dcookie_and_offset(struct spu *spu, unsigned int *offsetp,
318 unsigned long *spu_bin_dcookie,
319 unsigned long spu_ref)
320{
321 unsigned long app_cookie = 0;
322 unsigned int my_offset = 0;
323 struct file *app = NULL;
324 struct vm_area_struct *vma;
325 struct mm_struct *mm = spu->mm;
326
327 if (!mm)
328 goto out;
329
330 down_read(&mm->mmap_sem);
331
332 for (vma = mm->mmap; vma; vma = vma->vm_next) {
333 if (!vma->vm_file)
334 continue;
335 if (!(vma->vm_flags & VM_EXECUTABLE))
336 continue;
337 app_cookie = fast_get_dcookie(&vma->vm_file->f_path);
338 pr_debug("got dcookie for %s\n",
339 vma->vm_file->f_dentry->d_name.name);
340 app = vma->vm_file;
341 break;
342 }
343
344 for (vma = mm->mmap; vma; vma = vma->vm_next) {
345 if (vma->vm_start > spu_ref || vma->vm_end <= spu_ref)
346 continue;
347 my_offset = spu_ref - vma->vm_start;
348 if (!vma->vm_file)
349 goto fail_no_image_cookie;
350
351 pr_debug("Found spu ELF at %X(object-id:%lx) for file %s\n",
352 my_offset, spu_ref,
353 vma->vm_file->f_dentry->d_name.name);
354 *offsetp = my_offset;
355 break;
356 }
357
358 *spu_bin_dcookie = fast_get_dcookie(&vma->vm_file->f_path);
359 pr_debug("got dcookie for %s\n", vma->vm_file->f_dentry->d_name.name);
360
361 up_read(&mm->mmap_sem);
362
363out:
364 return app_cookie;
365
366fail_no_image_cookie:
367 up_read(&mm->mmap_sem);
368
369 printk(KERN_ERR "SPU_PROF: "
370 "%s, line %d: Cannot find dcookie for SPU binary\n",
371 __func__, __LINE__);
372 goto out;
373}
374
375
376
377
378
379
380
381static int process_context_switch(struct spu *spu, unsigned long objectId)
382{
383 unsigned long flags;
384 int retval;
385 unsigned int offset = 0;
386 unsigned long spu_cookie = 0, app_dcookie;
387
388 retval = prepare_cached_spu_info(spu, objectId);
389 if (retval)
390 goto out;
391
392
393
394
395 app_dcookie = get_exec_dcookie_and_offset(spu, &offset, &spu_cookie, objectId);
396 if (!app_dcookie || !spu_cookie) {
397 retval = -ENOENT;
398 goto out;
399 }
400
401
402 spin_lock_irqsave(&buffer_lock, flags);
403 spu_buff_add(ESCAPE_CODE, spu->number);
404 spu_buff_add(SPU_CTX_SWITCH_CODE, spu->number);
405 spu_buff_add(spu->number, spu->number);
406 spu_buff_add(spu->pid, spu->number);
407 spu_buff_add(spu->tgid, spu->number);
408 spu_buff_add(app_dcookie, spu->number);
409 spu_buff_add(spu_cookie, spu->number);
410 spu_buff_add(offset, spu->number);
411
412
413
414
415
416 spu_buff[spu->number].ctx_sw_seen = 1;
417
418 spin_unlock_irqrestore(&buffer_lock, flags);
419 smp_wmb();
420
421out:
422 return retval;
423}
424
425
426
427
428
429
430
431static int spu_active_notify(struct notifier_block *self, unsigned long val,
432 void *data)
433{
434 int retval;
435 unsigned long flags;
436 struct spu *the_spu = data;
437
438 pr_debug("SPU event notification arrived\n");
439 if (!val) {
440 spin_lock_irqsave(&cache_lock, flags);
441 retval = release_cached_info(the_spu->number);
442 spin_unlock_irqrestore(&cache_lock, flags);
443 } else {
444 retval = process_context_switch(the_spu, val);
445 }
446 return retval;
447}
448
449static struct notifier_block spu_active = {
450 .notifier_call = spu_active_notify,
451};
452
453static int number_of_online_nodes(void)
454{
455 u32 cpu; u32 tmp;
456 int nodes = 0;
457 for_each_online_cpu(cpu) {
458 tmp = cbe_cpu_to_node(cpu) + 1;
459 if (tmp > nodes)
460 nodes++;
461 }
462 return nodes;
463}
464
465static int oprofile_spu_buff_create(void)
466{
467 int spu;
468
469 max_spu_buff = oprofile_get_cpu_buffer_size();
470
471 for (spu = 0; spu < num_spu_nodes; spu++) {
472
473
474
475 spu_buff[spu].head = 0;
476 spu_buff[spu].tail = 0;
477
478
479
480
481
482
483
484 spu_buff[spu].buff = kzalloc((max_spu_buff
485 * sizeof(unsigned long)),
486 GFP_KERNEL);
487
488 if (!spu_buff[spu].buff) {
489 printk(KERN_ERR "SPU_PROF: "
490 "%s, line %d: oprofile_spu_buff_create "
491 "failed to allocate spu buffer %d.\n",
492 __func__, __LINE__, spu);
493
494
495 while (spu >= 0) {
496 kfree(spu_buff[spu].buff);
497 spu_buff[spu].buff = 0;
498 spu--;
499 }
500 return -ENOMEM;
501 }
502 }
503 return 0;
504}
505
506
507
508
509
510
511
512
513
514
515
516int spu_sync_start(void)
517{
518 int spu;
519 int ret = SKIP_GENERIC_SYNC;
520 int register_ret;
521 unsigned long flags = 0;
522
523 spu_prof_num_nodes = number_of_online_nodes();
524 num_spu_nodes = spu_prof_num_nodes * 8;
525 INIT_DELAYED_WORK(&spu_work, wq_sync_spu_buff);
526
527
528
529
530 ret = oprofile_spu_buff_create();
531 if (ret)
532 goto out;
533
534 spin_lock_irqsave(&buffer_lock, flags);
535 for (spu = 0; spu < num_spu_nodes; spu++) {
536 spu_buff_add(ESCAPE_CODE, spu);
537 spu_buff_add(SPU_PROFILING_CODE, spu);
538 spu_buff_add(num_spu_nodes, spu);
539 }
540 spin_unlock_irqrestore(&buffer_lock, flags);
541
542 for (spu = 0; spu < num_spu_nodes; spu++) {
543 spu_buff[spu].ctx_sw_seen = 0;
544 spu_buff[spu].last_guard_val = 0;
545 }
546
547
548 register_ret = spu_switch_event_register(&spu_active);
549 if (register_ret) {
550 ret = SYNC_START_ERROR;
551 goto out;
552 }
553
554 pr_debug("spu_sync_start -- running.\n");
555out:
556 return ret;
557}
558
559
560void spu_sync_buffer(int spu_num, unsigned int *samples,
561 int num_samples)
562{
563 unsigned long long file_offset;
564 unsigned long flags;
565 int i;
566 struct vma_to_fileoffset_map *map;
567 struct spu *the_spu;
568 unsigned long long spu_num_ll = spu_num;
569 unsigned long long spu_num_shifted = spu_num_ll << 32;
570 struct cached_info *c_info;
571
572
573
574
575
576
577 spin_lock_irqsave(&cache_lock, flags);
578 c_info = get_cached_info(NULL, spu_num);
579 if (!c_info) {
580
581
582
583
584 pr_debug("SPU_PROF: No cached SPU contex "
585 "for SPU #%d. Dropping samples.\n", spu_num);
586 goto out;
587 }
588
589 map = c_info->map;
590 the_spu = c_info->the_spu;
591 spin_lock(&buffer_lock);
592 for (i = 0; i < num_samples; i++) {
593 unsigned int sample = *(samples+i);
594 int grd_val = 0;
595 file_offset = 0;
596 if (sample == 0)
597 continue;
598 file_offset = vma_map_lookup( map, sample, the_spu, &grd_val);
599
600
601
602
603
604
605 if (grd_val && grd_val != spu_buff[spu_num].last_guard_val) {
606 spu_buff[spu_num].last_guard_val = grd_val;
607
608 break;
609 }
610
611
612
613
614
615
616 if (spu_buff[spu_num].ctx_sw_seen)
617 spu_buff_add((file_offset | spu_num_shifted),
618 spu_num);
619 }
620 spin_unlock(&buffer_lock);
621out:
622 spin_unlock_irqrestore(&cache_lock, flags);
623}
624
625
626int spu_sync_stop(void)
627{
628 unsigned long flags = 0;
629 int ret;
630 int k;
631
632 ret = spu_switch_event_unregister(&spu_active);
633
634 if (ret)
635 printk(KERN_ERR "SPU_PROF: "
636 "%s, line %d: spu_switch_event_unregister " \
637 "returned %d\n",
638 __func__, __LINE__, ret);
639
640
641 sync_spu_buff();
642
643 spin_lock_irqsave(&cache_lock, flags);
644 ret = release_cached_info(RELEASE_ALL);
645 spin_unlock_irqrestore(&cache_lock, flags);
646
647
648
649
650
651 cancel_delayed_work(&spu_work);
652
653 for (k = 0; k < num_spu_nodes; k++) {
654 spu_buff[k].ctx_sw_seen = 0;
655
656
657
658
659
660 kfree(spu_buff[k].buff);
661 spu_buff[k].buff = 0;
662 }
663 pr_debug("spu_sync_stop -- done.\n");
664 return ret;
665}
666
667