1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22#include <linux/module.h>
23#include <linux/kernel.h>
24#include <linux/sched.h>
25#include <linux/sched/task.h>
26#include <linux/sched/task_stack.h>
27#include <linux/interrupt.h>
28#include <linux/proc_fs.h>
29#include <linux/seq_file.h>
30#include <linux/init.h>
31#include <linux/vmalloc.h>
32#include <linux/mm.h>
33#include <linux/sysctl.h>
34#include <linux/list.h>
35#include <linux/file.h>
36#include <linux/poll.h>
37#include <linux/vfs.h>
38#include <linux/smp.h>
39#include <linux/pagemap.h>
40#include <linux/mount.h>
41#include <linux/bitops.h>
42#include <linux/capability.h>
43#include <linux/rcupdate.h>
44#include <linux/completion.h>
45#include <linux/tracehook.h>
46#include <linux/slab.h>
47#include <linux/cpu.h>
48
49#include <asm/errno.h>
50#include <asm/intrinsics.h>
51#include <asm/page.h>
52#include <asm/perfmon.h>
53#include <asm/processor.h>
54#include <asm/signal.h>
55#include <linux/uaccess.h>
56#include <asm/delay.h>
57
58#ifdef CONFIG_PERFMON
59
60
61
62#define PFM_CTX_UNLOADED 1
63#define PFM_CTX_LOADED 2
64#define PFM_CTX_MASKED 3
65#define PFM_CTX_ZOMBIE 4
66
67#define PFM_INVALID_ACTIVATION (~0UL)
68
69#define PFM_NUM_PMC_REGS 64
70#define PFM_NUM_PMD_REGS 64
71
72
73
74
75#define PFM_MAX_MSGS 32
76#define PFM_CTXQ_EMPTY(g) ((g)->ctx_msgq_head == (g)->ctx_msgq_tail)
77
78
79
80
81
82
83
84
85
86
87
88
89#define PFM_REG_NOTIMPL 0x0
90#define PFM_REG_IMPL 0x1
91#define PFM_REG_END 0x2
92#define PFM_REG_MONITOR (0x1<<4|PFM_REG_IMPL)
93#define PFM_REG_COUNTING (0x2<<4|PFM_REG_MONITOR)
94#define PFM_REG_CONTROL (0x4<<4|PFM_REG_IMPL)
95#define PFM_REG_CONFIG (0x8<<4|PFM_REG_IMPL)
96#define PFM_REG_BUFFER (0xc<<4|PFM_REG_IMPL)
97
98#define PMC_IS_LAST(i) (pmu_conf->pmc_desc[i].type & PFM_REG_END)
99#define PMD_IS_LAST(i) (pmu_conf->pmd_desc[i].type & PFM_REG_END)
100
101#define PMC_OVFL_NOTIFY(ctx, i) ((ctx)->ctx_pmds[i].flags & PFM_REGFL_OVFL_NOTIFY)
102
103
104#define PMC_IS_IMPL(i) (i< PMU_MAX_PMCS && (pmu_conf->pmc_desc[i].type & PFM_REG_IMPL))
105#define PMD_IS_IMPL(i) (i< PMU_MAX_PMDS && (pmu_conf->pmd_desc[i].type & PFM_REG_IMPL))
106
107
108#define PMD_IS_COUNTING(i) ((pmu_conf->pmd_desc[i].type & PFM_REG_COUNTING) == PFM_REG_COUNTING)
109#define PMC_IS_COUNTING(i) ((pmu_conf->pmc_desc[i].type & PFM_REG_COUNTING) == PFM_REG_COUNTING)
110#define PMC_IS_MONITOR(i) ((pmu_conf->pmc_desc[i].type & PFM_REG_MONITOR) == PFM_REG_MONITOR)
111#define PMC_IS_CONTROL(i) ((pmu_conf->pmc_desc[i].type & PFM_REG_CONTROL) == PFM_REG_CONTROL)
112
113#define PMC_DFL_VAL(i) pmu_conf->pmc_desc[i].default_value
114#define PMC_RSVD_MASK(i) pmu_conf->pmc_desc[i].reserved_mask
115#define PMD_PMD_DEP(i) pmu_conf->pmd_desc[i].dep_pmd[0]
116#define PMC_PMD_DEP(i) pmu_conf->pmc_desc[i].dep_pmd[0]
117
118#define PFM_NUM_IBRS IA64_NUM_DBG_REGS
119#define PFM_NUM_DBRS IA64_NUM_DBG_REGS
120
121#define CTX_OVFL_NOBLOCK(c) ((c)->ctx_fl_block == 0)
122#define CTX_HAS_SMPL(c) ((c)->ctx_fl_is_sampling)
123#define PFM_CTX_TASK(h) (h)->ctx_task
124
125#define PMU_PMC_OI 5
126
127
128#define CTX_USED_PMD(ctx, mask) (ctx)->ctx_used_pmds[0] |= (mask)
129#define CTX_IS_USED_PMD(ctx, c) (((ctx)->ctx_used_pmds[0] & (1UL << (c))) != 0UL)
130
131#define CTX_USED_MONITOR(ctx, mask) (ctx)->ctx_used_monitors[0] |= (mask)
132
133#define CTX_USED_IBR(ctx,n) (ctx)->ctx_used_ibrs[(n)>>6] |= 1UL<< ((n) % 64)
134#define CTX_USED_DBR(ctx,n) (ctx)->ctx_used_dbrs[(n)>>6] |= 1UL<< ((n) % 64)
135#define CTX_USES_DBREGS(ctx) (((pfm_context_t *)(ctx))->ctx_fl_using_dbreg==1)
136#define PFM_CODE_RR 0
137#define PFM_DATA_RR 1
138
139#define PFM_CPUINFO_CLEAR(v) pfm_get_cpu_var(pfm_syst_info) &= ~(v)
140#define PFM_CPUINFO_SET(v) pfm_get_cpu_var(pfm_syst_info) |= (v)
141#define PFM_CPUINFO_GET() pfm_get_cpu_var(pfm_syst_info)
142
143#define RDEP(x) (1UL<<(x))
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163#define PROTECT_CTX(c, f) \
164 do { \
165 DPRINT(("spinlock_irq_save ctx %p by [%d]\n", c, task_pid_nr(current))); \
166 spin_lock_irqsave(&(c)->ctx_lock, f); \
167 DPRINT(("spinlocked ctx %p by [%d]\n", c, task_pid_nr(current))); \
168 } while(0)
169
170#define UNPROTECT_CTX(c, f) \
171 do { \
172 DPRINT(("spinlock_irq_restore ctx %p by [%d]\n", c, task_pid_nr(current))); \
173 spin_unlock_irqrestore(&(c)->ctx_lock, f); \
174 } while(0)
175
176#define PROTECT_CTX_NOPRINT(c, f) \
177 do { \
178 spin_lock_irqsave(&(c)->ctx_lock, f); \
179 } while(0)
180
181
182#define UNPROTECT_CTX_NOPRINT(c, f) \
183 do { \
184 spin_unlock_irqrestore(&(c)->ctx_lock, f); \
185 } while(0)
186
187
188#define PROTECT_CTX_NOIRQ(c) \
189 do { \
190 spin_lock(&(c)->ctx_lock); \
191 } while(0)
192
193#define UNPROTECT_CTX_NOIRQ(c) \
194 do { \
195 spin_unlock(&(c)->ctx_lock); \
196 } while(0)
197
198
199#ifdef CONFIG_SMP
200
201#define GET_ACTIVATION() pfm_get_cpu_var(pmu_activation_number)
202#define INC_ACTIVATION() pfm_get_cpu_var(pmu_activation_number)++
203#define SET_ACTIVATION(c) (c)->ctx_last_activation = GET_ACTIVATION()
204
205#else
206#define SET_ACTIVATION(t) do {} while(0)
207#define GET_ACTIVATION(t) do {} while(0)
208#define INC_ACTIVATION(t) do {} while(0)
209#endif
210
211#define SET_PMU_OWNER(t, c) do { pfm_get_cpu_var(pmu_owner) = (t); pfm_get_cpu_var(pmu_ctx) = (c); } while(0)
212#define GET_PMU_OWNER() pfm_get_cpu_var(pmu_owner)
213#define GET_PMU_CTX() pfm_get_cpu_var(pmu_ctx)
214
215#define LOCK_PFS(g) spin_lock_irqsave(&pfm_sessions.pfs_lock, g)
216#define UNLOCK_PFS(g) spin_unlock_irqrestore(&pfm_sessions.pfs_lock, g)
217
218#define PFM_REG_RETFLAG_SET(flags, val) do { flags &= ~PFM_REG_RETFL_MASK; flags |= (val); } while(0)
219
220
221
222
223#define PMC0_HAS_OVFL(cmp0) (cmp0 & ~0x1UL)
224
225#define PFMFS_MAGIC 0xa0b4d889
226
227
228
229
230#define PFM_DEBUGGING 1
231#ifdef PFM_DEBUGGING
232#define DPRINT(a) \
233 do { \
234 if (unlikely(pfm_sysctl.debug >0)) { printk("%s.%d: CPU%d [%d] ", __func__, __LINE__, smp_processor_id(), task_pid_nr(current)); printk a; } \
235 } while (0)
236
237#define DPRINT_ovfl(a) \
238 do { \
239 if (unlikely(pfm_sysctl.debug > 0 && pfm_sysctl.debug_ovfl >0)) { printk("%s.%d: CPU%d [%d] ", __func__, __LINE__, smp_processor_id(), task_pid_nr(current)); printk a; } \
240 } while (0)
241#endif
242
243
244
245
246
247
248typedef struct {
249 unsigned long val;
250 unsigned long lval;
251 unsigned long long_reset;
252 unsigned long short_reset;
253 unsigned long reset_pmds[4];
254 unsigned long smpl_pmds[4];
255 unsigned long seed;
256 unsigned long mask;
257 unsigned int flags;
258 unsigned long eventid;
259} pfm_counter_t;
260
261
262
263
264typedef struct {
265 unsigned int block:1;
266 unsigned int system:1;
267 unsigned int using_dbreg:1;
268 unsigned int is_sampling:1;
269 unsigned int excl_idle:1;
270 unsigned int going_zombie:1;
271 unsigned int trap_reason:2;
272 unsigned int no_msg:1;
273 unsigned int can_restart:1;
274 unsigned int reserved:22;
275} pfm_context_flags_t;
276
277#define PFM_TRAP_REASON_NONE 0x0
278#define PFM_TRAP_REASON_BLOCK 0x1
279#define PFM_TRAP_REASON_RESET 0x2
280
281
282
283
284
285
286typedef struct pfm_context {
287 spinlock_t ctx_lock;
288
289 pfm_context_flags_t ctx_flags;
290 unsigned int ctx_state;
291
292 struct task_struct *ctx_task;
293
294 unsigned long ctx_ovfl_regs[4];
295
296 struct completion ctx_restart_done;
297
298 unsigned long ctx_used_pmds[4];
299 unsigned long ctx_all_pmds[4];
300 unsigned long ctx_reload_pmds[4];
301
302 unsigned long ctx_all_pmcs[4];
303 unsigned long ctx_reload_pmcs[4];
304 unsigned long ctx_used_monitors[4];
305
306 unsigned long ctx_pmcs[PFM_NUM_PMC_REGS];
307
308 unsigned int ctx_used_ibrs[1];
309 unsigned int ctx_used_dbrs[1];
310 unsigned long ctx_dbrs[IA64_NUM_DBG_REGS];
311 unsigned long ctx_ibrs[IA64_NUM_DBG_REGS];
312
313 pfm_counter_t ctx_pmds[PFM_NUM_PMD_REGS];
314
315 unsigned long th_pmcs[PFM_NUM_PMC_REGS];
316 unsigned long th_pmds[PFM_NUM_PMD_REGS];
317
318 unsigned long ctx_saved_psr_up;
319
320 unsigned long ctx_last_activation;
321 unsigned int ctx_last_cpu;
322 unsigned int ctx_cpu;
323
324 int ctx_fd;
325 pfm_ovfl_arg_t ctx_ovfl_arg;
326
327 pfm_buffer_fmt_t *ctx_buf_fmt;
328 void *ctx_smpl_hdr;
329 unsigned long ctx_smpl_size;
330 void *ctx_smpl_vaddr;
331
332 wait_queue_head_t ctx_msgq_wait;
333 pfm_msg_t ctx_msgq[PFM_MAX_MSGS];
334 int ctx_msgq_head;
335 int ctx_msgq_tail;
336 struct fasync_struct *ctx_async_queue;
337
338 wait_queue_head_t ctx_zombieq;
339} pfm_context_t;
340
341
342
343
344
345#define PFM_IS_FILE(f) ((f)->f_op == &pfm_file_ops)
346
347#define PFM_GET_CTX(t) ((pfm_context_t *)(t)->thread.pfm_context)
348
349#ifdef CONFIG_SMP
350#define SET_LAST_CPU(ctx, v) (ctx)->ctx_last_cpu = (v)
351#define GET_LAST_CPU(ctx) (ctx)->ctx_last_cpu
352#else
353#define SET_LAST_CPU(ctx, v) do {} while(0)
354#define GET_LAST_CPU(ctx) do {} while(0)
355#endif
356
357
358#define ctx_fl_block ctx_flags.block
359#define ctx_fl_system ctx_flags.system
360#define ctx_fl_using_dbreg ctx_flags.using_dbreg
361#define ctx_fl_is_sampling ctx_flags.is_sampling
362#define ctx_fl_excl_idle ctx_flags.excl_idle
363#define ctx_fl_going_zombie ctx_flags.going_zombie
364#define ctx_fl_trap_reason ctx_flags.trap_reason
365#define ctx_fl_no_msg ctx_flags.no_msg
366#define ctx_fl_can_restart ctx_flags.can_restart
367
368#define PFM_SET_WORK_PENDING(t, v) do { (t)->thread.pfm_needs_checking = v; } while(0);
369#define PFM_GET_WORK_PENDING(t) (t)->thread.pfm_needs_checking
370
371
372
373
374
375typedef struct {
376 spinlock_t pfs_lock;
377
378 unsigned int pfs_task_sessions;
379 unsigned int pfs_sys_sessions;
380 unsigned int pfs_sys_use_dbregs;
381 unsigned int pfs_ptrace_use_dbregs;
382 struct task_struct *pfs_sys_session[NR_CPUS];
383} pfm_session_t;
384
385
386
387
388
389
390typedef int (*pfm_reg_check_t)(struct task_struct *task, pfm_context_t *ctx, unsigned int cnum, unsigned long *val, struct pt_regs *regs);
391typedef struct {
392 unsigned int type;
393 int pm_pos;
394 unsigned long default_value;
395 unsigned long reserved_mask;
396 pfm_reg_check_t read_check;
397 pfm_reg_check_t write_check;
398 unsigned long dep_pmd[4];
399 unsigned long dep_pmc[4];
400} pfm_reg_desc_t;
401
402
403#define PMC_PM(cnum, val) (((val) >> (pmu_conf->pmc_desc[cnum].pm_pos)) & 0x1)
404
405
406
407
408
409
410
411
412
413
414
415
416
417typedef struct {
418 unsigned long ovfl_val;
419
420 pfm_reg_desc_t *pmc_desc;
421 pfm_reg_desc_t *pmd_desc;
422
423 unsigned int num_pmcs;
424 unsigned int num_pmds;
425 unsigned long impl_pmcs[4];
426 unsigned long impl_pmds[4];
427
428 char *pmu_name;
429 unsigned int pmu_family;
430 unsigned int flags;
431 unsigned int num_ibrs;
432 unsigned int num_dbrs;
433 unsigned int num_counters;
434 int (*probe)(void);
435 unsigned int use_rr_dbregs:1;
436} pmu_config_t;
437
438
439
440#define PFM_PMU_IRQ_RESEND 1
441
442
443
444
445typedef struct {
446 unsigned long ibr_mask:56;
447 unsigned long ibr_plm:4;
448 unsigned long ibr_ig:3;
449 unsigned long ibr_x:1;
450} ibr_mask_reg_t;
451
452typedef struct {
453 unsigned long dbr_mask:56;
454 unsigned long dbr_plm:4;
455 unsigned long dbr_ig:2;
456 unsigned long dbr_w:1;
457 unsigned long dbr_r:1;
458} dbr_mask_reg_t;
459
460typedef union {
461 unsigned long val;
462 ibr_mask_reg_t ibr;
463 dbr_mask_reg_t dbr;
464} dbreg_t;
465
466
467
468
469
470typedef struct {
471 int (*cmd_func)(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs);
472 char *cmd_name;
473 int cmd_flags;
474 unsigned int cmd_narg;
475 size_t cmd_argsize;
476 int (*cmd_getsize)(void *arg, size_t *sz);
477} pfm_cmd_desc_t;
478
479#define PFM_CMD_FD 0x01
480#define PFM_CMD_ARG_READ 0x02
481#define PFM_CMD_ARG_RW 0x04
482#define PFM_CMD_STOP 0x08
483
484
485#define PFM_CMD_NAME(cmd) pfm_cmd_tab[(cmd)].cmd_name
486#define PFM_CMD_READ_ARG(cmd) (pfm_cmd_tab[(cmd)].cmd_flags & PFM_CMD_ARG_READ)
487#define PFM_CMD_RW_ARG(cmd) (pfm_cmd_tab[(cmd)].cmd_flags & PFM_CMD_ARG_RW)
488#define PFM_CMD_USE_FD(cmd) (pfm_cmd_tab[(cmd)].cmd_flags & PFM_CMD_FD)
489#define PFM_CMD_STOPPED(cmd) (pfm_cmd_tab[(cmd)].cmd_flags & PFM_CMD_STOP)
490
491#define PFM_CMD_ARG_MANY -1
492
493typedef struct {
494 unsigned long pfm_spurious_ovfl_intr_count;
495 unsigned long pfm_replay_ovfl_intr_count;
496 unsigned long pfm_ovfl_intr_count;
497 unsigned long pfm_ovfl_intr_cycles;
498 unsigned long pfm_ovfl_intr_cycles_min;
499 unsigned long pfm_ovfl_intr_cycles_max;
500 unsigned long pfm_smpl_handler_calls;
501 unsigned long pfm_smpl_handler_cycles;
502 char pad[SMP_CACHE_BYTES] ____cacheline_aligned;
503} pfm_stats_t;
504
505
506
507
508static pfm_stats_t pfm_stats[NR_CPUS];
509static pfm_session_t pfm_sessions;
510
511static DEFINE_SPINLOCK(pfm_alt_install_check);
512static pfm_intr_handler_desc_t *pfm_alt_intr_handler;
513
514static struct proc_dir_entry *perfmon_dir;
515static pfm_uuid_t pfm_null_uuid = {0,};
516
517static spinlock_t pfm_buffer_fmt_lock;
518static LIST_HEAD(pfm_buffer_fmt_list);
519
520static pmu_config_t *pmu_conf;
521
522
523pfm_sysctl_t pfm_sysctl;
524EXPORT_SYMBOL(pfm_sysctl);
525
526static struct ctl_table pfm_ctl_table[] = {
527 {
528 .procname = "debug",
529 .data = &pfm_sysctl.debug,
530 .maxlen = sizeof(int),
531 .mode = 0666,
532 .proc_handler = proc_dointvec,
533 },
534 {
535 .procname = "debug_ovfl",
536 .data = &pfm_sysctl.debug_ovfl,
537 .maxlen = sizeof(int),
538 .mode = 0666,
539 .proc_handler = proc_dointvec,
540 },
541 {
542 .procname = "fastctxsw",
543 .data = &pfm_sysctl.fastctxsw,
544 .maxlen = sizeof(int),
545 .mode = 0600,
546 .proc_handler = proc_dointvec,
547 },
548 {
549 .procname = "expert_mode",
550 .data = &pfm_sysctl.expert_mode,
551 .maxlen = sizeof(int),
552 .mode = 0600,
553 .proc_handler = proc_dointvec,
554 },
555 {}
556};
557static struct ctl_table pfm_sysctl_dir[] = {
558 {
559 .procname = "perfmon",
560 .mode = 0555,
561 .child = pfm_ctl_table,
562 },
563 {}
564};
565static struct ctl_table pfm_sysctl_root[] = {
566 {
567 .procname = "kernel",
568 .mode = 0555,
569 .child = pfm_sysctl_dir,
570 },
571 {}
572};
573static struct ctl_table_header *pfm_sysctl_header;
574
575static int pfm_context_unload(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs);
576
577#define pfm_get_cpu_var(v) __ia64_per_cpu_var(v)
578#define pfm_get_cpu_data(a,b) per_cpu(a, b)
579
580static inline void
581pfm_put_task(struct task_struct *task)
582{
583 if (task != current) put_task_struct(task);
584}
585
586static inline void
587pfm_reserve_page(unsigned long a)
588{
589 SetPageReserved(vmalloc_to_page((void *)a));
590}
591static inline void
592pfm_unreserve_page(unsigned long a)
593{
594 ClearPageReserved(vmalloc_to_page((void*)a));
595}
596
597static inline unsigned long
598pfm_protect_ctx_ctxsw(pfm_context_t *x)
599{
600 spin_lock(&(x)->ctx_lock);
601 return 0UL;
602}
603
604static inline void
605pfm_unprotect_ctx_ctxsw(pfm_context_t *x, unsigned long f)
606{
607 spin_unlock(&(x)->ctx_lock);
608}
609
610
611static const struct dentry_operations pfmfs_dentry_operations;
612
613static struct dentry *
614pfmfs_mount(struct file_system_type *fs_type, int flags, const char *dev_name, void *data)
615{
616 return mount_pseudo(fs_type, "pfm:", NULL, &pfmfs_dentry_operations,
617 PFMFS_MAGIC);
618}
619
620static struct file_system_type pfm_fs_type = {
621 .name = "pfmfs",
622 .mount = pfmfs_mount,
623 .kill_sb = kill_anon_super,
624};
625MODULE_ALIAS_FS("pfmfs");
626
627DEFINE_PER_CPU(unsigned long, pfm_syst_info);
628DEFINE_PER_CPU(struct task_struct *, pmu_owner);
629DEFINE_PER_CPU(pfm_context_t *, pmu_ctx);
630DEFINE_PER_CPU(unsigned long, pmu_activation_number);
631EXPORT_PER_CPU_SYMBOL_GPL(pfm_syst_info);
632
633
634
635static const struct file_operations pfm_file_ops;
636
637
638
639
640#ifndef CONFIG_SMP
641static void pfm_lazy_save_regs (struct task_struct *ta);
642#endif
643
644void dump_pmu_state(const char *);
645static int pfm_write_ibr_dbr(int mode, pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs);
646
647#include "perfmon_itanium.h"
648#include "perfmon_mckinley.h"
649#include "perfmon_montecito.h"
650#include "perfmon_generic.h"
651
652static pmu_config_t *pmu_confs[]={
653 &pmu_conf_mont,
654 &pmu_conf_mck,
655 &pmu_conf_ita,
656 &pmu_conf_gen,
657 NULL
658};
659
660
661static int pfm_end_notify_user(pfm_context_t *ctx);
662
663static inline void
664pfm_clear_psr_pp(void)
665{
666 ia64_rsm(IA64_PSR_PP);
667 ia64_srlz_i();
668}
669
670static inline void
671pfm_set_psr_pp(void)
672{
673 ia64_ssm(IA64_PSR_PP);
674 ia64_srlz_i();
675}
676
677static inline void
678pfm_clear_psr_up(void)
679{
680 ia64_rsm(IA64_PSR_UP);
681 ia64_srlz_i();
682}
683
684static inline void
685pfm_set_psr_up(void)
686{
687 ia64_ssm(IA64_PSR_UP);
688 ia64_srlz_i();
689}
690
691static inline unsigned long
692pfm_get_psr(void)
693{
694 unsigned long tmp;
695 tmp = ia64_getreg(_IA64_REG_PSR);
696 ia64_srlz_i();
697 return tmp;
698}
699
700static inline void
701pfm_set_psr_l(unsigned long val)
702{
703 ia64_setreg(_IA64_REG_PSR_L, val);
704 ia64_srlz_i();
705}
706
707static inline void
708pfm_freeze_pmu(void)
709{
710 ia64_set_pmc(0,1UL);
711 ia64_srlz_d();
712}
713
714static inline void
715pfm_unfreeze_pmu(void)
716{
717 ia64_set_pmc(0,0UL);
718 ia64_srlz_d();
719}
720
721static inline void
722pfm_restore_ibrs(unsigned long *ibrs, unsigned int nibrs)
723{
724 int i;
725
726 for (i=0; i < nibrs; i++) {
727 ia64_set_ibr(i, ibrs[i]);
728 ia64_dv_serialize_instruction();
729 }
730 ia64_srlz_i();
731}
732
733static inline void
734pfm_restore_dbrs(unsigned long *dbrs, unsigned int ndbrs)
735{
736 int i;
737
738 for (i=0; i < ndbrs; i++) {
739 ia64_set_dbr(i, dbrs[i]);
740 ia64_dv_serialize_data();
741 }
742 ia64_srlz_d();
743}
744
745
746
747
748static inline unsigned long
749pfm_read_soft_counter(pfm_context_t *ctx, int i)
750{
751 return ctx->ctx_pmds[i].val + (ia64_get_pmd(i) & pmu_conf->ovfl_val);
752}
753
754
755
756
757static inline void
758pfm_write_soft_counter(pfm_context_t *ctx, int i, unsigned long val)
759{
760 unsigned long ovfl_val = pmu_conf->ovfl_val;
761
762 ctx->ctx_pmds[i].val = val & ~ovfl_val;
763
764
765
766
767 ia64_set_pmd(i, val & ovfl_val);
768}
769
770static pfm_msg_t *
771pfm_get_new_msg(pfm_context_t *ctx)
772{
773 int idx, next;
774
775 next = (ctx->ctx_msgq_tail+1) % PFM_MAX_MSGS;
776
777 DPRINT(("ctx_fd=%p head=%d tail=%d\n", ctx, ctx->ctx_msgq_head, ctx->ctx_msgq_tail));
778 if (next == ctx->ctx_msgq_head) return NULL;
779
780 idx = ctx->ctx_msgq_tail;
781 ctx->ctx_msgq_tail = next;
782
783 DPRINT(("ctx=%p head=%d tail=%d msg=%d\n", ctx, ctx->ctx_msgq_head, ctx->ctx_msgq_tail, idx));
784
785 return ctx->ctx_msgq+idx;
786}
787
788static pfm_msg_t *
789pfm_get_next_msg(pfm_context_t *ctx)
790{
791 pfm_msg_t *msg;
792
793 DPRINT(("ctx=%p head=%d tail=%d\n", ctx, ctx->ctx_msgq_head, ctx->ctx_msgq_tail));
794
795 if (PFM_CTXQ_EMPTY(ctx)) return NULL;
796
797
798
799
800 msg = ctx->ctx_msgq+ctx->ctx_msgq_head;
801
802
803
804
805 ctx->ctx_msgq_head = (ctx->ctx_msgq_head+1) % PFM_MAX_MSGS;
806
807 DPRINT(("ctx=%p head=%d tail=%d type=%d\n", ctx, ctx->ctx_msgq_head, ctx->ctx_msgq_tail, msg->pfm_gen_msg.msg_type));
808
809 return msg;
810}
811
812static void
813pfm_reset_msgq(pfm_context_t *ctx)
814{
815 ctx->ctx_msgq_head = ctx->ctx_msgq_tail = 0;
816 DPRINT(("ctx=%p msgq reset\n", ctx));
817}
818
819static void *
820pfm_rvmalloc(unsigned long size)
821{
822 void *mem;
823 unsigned long addr;
824
825 size = PAGE_ALIGN(size);
826 mem = vzalloc(size);
827 if (mem) {
828
829 addr = (unsigned long)mem;
830 while (size > 0) {
831 pfm_reserve_page(addr);
832 addr+=PAGE_SIZE;
833 size-=PAGE_SIZE;
834 }
835 }
836 return mem;
837}
838
839static void
840pfm_rvfree(void *mem, unsigned long size)
841{
842 unsigned long addr;
843
844 if (mem) {
845 DPRINT(("freeing physical buffer @%p size=%lu\n", mem, size));
846 addr = (unsigned long) mem;
847 while ((long) size > 0) {
848 pfm_unreserve_page(addr);
849 addr+=PAGE_SIZE;
850 size-=PAGE_SIZE;
851 }
852 vfree(mem);
853 }
854 return;
855}
856
857static pfm_context_t *
858pfm_context_alloc(int ctx_flags)
859{
860 pfm_context_t *ctx;
861
862
863
864
865
866 ctx = kzalloc(sizeof(pfm_context_t), GFP_KERNEL);
867 if (ctx) {
868 DPRINT(("alloc ctx @%p\n", ctx));
869
870
871
872
873 spin_lock_init(&ctx->ctx_lock);
874
875
876
877
878 ctx->ctx_state = PFM_CTX_UNLOADED;
879
880
881
882
883 ctx->ctx_fl_block = (ctx_flags & PFM_FL_NOTIFY_BLOCK) ? 1 : 0;
884 ctx->ctx_fl_system = (ctx_flags & PFM_FL_SYSTEM_WIDE) ? 1: 0;
885 ctx->ctx_fl_no_msg = (ctx_flags & PFM_FL_OVFL_NO_MSG) ? 1: 0;
886
887
888
889
890
891
892
893
894 init_completion(&ctx->ctx_restart_done);
895
896
897
898
899 ctx->ctx_last_activation = PFM_INVALID_ACTIVATION;
900 SET_LAST_CPU(ctx, -1);
901
902
903
904
905 ctx->ctx_msgq_head = ctx->ctx_msgq_tail = 0;
906 init_waitqueue_head(&ctx->ctx_msgq_wait);
907 init_waitqueue_head(&ctx->ctx_zombieq);
908
909 }
910 return ctx;
911}
912
913static void
914pfm_context_free(pfm_context_t *ctx)
915{
916 if (ctx) {
917 DPRINT(("free ctx @%p\n", ctx));
918 kfree(ctx);
919 }
920}
921
922static void
923pfm_mask_monitoring(struct task_struct *task)
924{
925 pfm_context_t *ctx = PFM_GET_CTX(task);
926 unsigned long mask, val, ovfl_mask;
927 int i;
928
929 DPRINT_ovfl(("masking monitoring for [%d]\n", task_pid_nr(task)));
930
931 ovfl_mask = pmu_conf->ovfl_val;
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951 mask = ctx->ctx_used_pmds[0];
952 for (i = 0; mask; i++, mask>>=1) {
953
954 if ((mask & 0x1) == 0) continue;
955 val = ia64_get_pmd(i);
956
957 if (PMD_IS_COUNTING(i)) {
958
959
960
961 ctx->ctx_pmds[i].val += (val & ovfl_mask);
962 } else {
963 ctx->ctx_pmds[i].val = val;
964 }
965 DPRINT_ovfl(("pmd[%d]=0x%lx hw_pmd=0x%lx\n",
966 i,
967 ctx->ctx_pmds[i].val,
968 val & ovfl_mask));
969 }
970
971
972
973
974
975
976
977
978 mask = ctx->ctx_used_monitors[0] >> PMU_FIRST_COUNTER;
979 for(i= PMU_FIRST_COUNTER; mask; i++, mask>>=1) {
980 if ((mask & 0x1) == 0UL) continue;
981 ia64_set_pmc(i, ctx->th_pmcs[i] & ~0xfUL);
982 ctx->th_pmcs[i] &= ~0xfUL;
983 DPRINT_ovfl(("pmc[%d]=0x%lx\n", i, ctx->th_pmcs[i]));
984 }
985
986
987
988 ia64_srlz_d();
989}
990
991
992
993
994
995
996static void
997pfm_restore_monitoring(struct task_struct *task)
998{
999 pfm_context_t *ctx = PFM_GET_CTX(task);
1000 unsigned long mask, ovfl_mask;
1001 unsigned long psr, val;
1002 int i, is_system;
1003
1004 is_system = ctx->ctx_fl_system;
1005 ovfl_mask = pmu_conf->ovfl_val;
1006
1007 if (task != current) {
1008 printk(KERN_ERR "perfmon.%d: invalid task[%d] current[%d]\n", __LINE__, task_pid_nr(task), task_pid_nr(current));
1009 return;
1010 }
1011 if (ctx->ctx_state != PFM_CTX_MASKED) {
1012 printk(KERN_ERR "perfmon.%d: task[%d] current[%d] invalid state=%d\n", __LINE__,
1013 task_pid_nr(task), task_pid_nr(current), ctx->ctx_state);
1014 return;
1015 }
1016 psr = pfm_get_psr();
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027 if (is_system && (PFM_CPUINFO_GET() & PFM_CPUINFO_DCR_PP)) {
1028
1029 ia64_setreg(_IA64_REG_CR_DCR, ia64_getreg(_IA64_REG_CR_DCR) & ~IA64_DCR_PP);
1030 pfm_clear_psr_pp();
1031 } else {
1032 pfm_clear_psr_up();
1033 }
1034
1035
1036
1037 mask = ctx->ctx_used_pmds[0];
1038 for (i = 0; mask; i++, mask>>=1) {
1039
1040 if ((mask & 0x1) == 0) continue;
1041
1042 if (PMD_IS_COUNTING(i)) {
1043
1044
1045
1046
1047 val = ctx->ctx_pmds[i].val & ovfl_mask;
1048 ctx->ctx_pmds[i].val &= ~ovfl_mask;
1049 } else {
1050 val = ctx->ctx_pmds[i].val;
1051 }
1052 ia64_set_pmd(i, val);
1053
1054 DPRINT(("pmd[%d]=0x%lx hw_pmd=0x%lx\n",
1055 i,
1056 ctx->ctx_pmds[i].val,
1057 val));
1058 }
1059
1060
1061
1062 mask = ctx->ctx_used_monitors[0] >> PMU_FIRST_COUNTER;
1063 for(i= PMU_FIRST_COUNTER; mask; i++, mask>>=1) {
1064 if ((mask & 0x1) == 0UL) continue;
1065 ctx->th_pmcs[i] = ctx->ctx_pmcs[i];
1066 ia64_set_pmc(i, ctx->th_pmcs[i]);
1067 DPRINT(("[%d] pmc[%d]=0x%lx\n",
1068 task_pid_nr(task), i, ctx->th_pmcs[i]));
1069 }
1070 ia64_srlz_d();
1071
1072
1073
1074
1075
1076 if (ctx->ctx_fl_using_dbreg) {
1077 pfm_restore_ibrs(ctx->ctx_ibrs, pmu_conf->num_ibrs);
1078 pfm_restore_dbrs(ctx->ctx_dbrs, pmu_conf->num_dbrs);
1079 }
1080
1081
1082
1083
1084 if (is_system && (PFM_CPUINFO_GET() & PFM_CPUINFO_DCR_PP)) {
1085
1086 ia64_setreg(_IA64_REG_CR_DCR, ia64_getreg(_IA64_REG_CR_DCR) | IA64_DCR_PP);
1087 ia64_srlz_i();
1088 }
1089 pfm_set_psr_l(psr);
1090}
1091
1092static inline void
1093pfm_save_pmds(unsigned long *pmds, unsigned long mask)
1094{
1095 int i;
1096
1097 ia64_srlz_d();
1098
1099 for (i=0; mask; i++, mask>>=1) {
1100 if (mask & 0x1) pmds[i] = ia64_get_pmd(i);
1101 }
1102}
1103
1104
1105
1106
1107static inline void
1108pfm_restore_pmds(unsigned long *pmds, unsigned long mask)
1109{
1110 int i;
1111 unsigned long val, ovfl_val = pmu_conf->ovfl_val;
1112
1113 for (i=0; mask; i++, mask>>=1) {
1114 if ((mask & 0x1) == 0) continue;
1115 val = PMD_IS_COUNTING(i) ? pmds[i] & ovfl_val : pmds[i];
1116 ia64_set_pmd(i, val);
1117 }
1118 ia64_srlz_d();
1119}
1120
1121
1122
1123
1124static inline void
1125pfm_copy_pmds(struct task_struct *task, pfm_context_t *ctx)
1126{
1127 unsigned long ovfl_val = pmu_conf->ovfl_val;
1128 unsigned long mask = ctx->ctx_all_pmds[0];
1129 unsigned long val;
1130 int i;
1131
1132 DPRINT(("mask=0x%lx\n", mask));
1133
1134 for (i=0; mask; i++, mask>>=1) {
1135
1136 val = ctx->ctx_pmds[i].val;
1137
1138
1139
1140
1141
1142
1143
1144 if (PMD_IS_COUNTING(i)) {
1145 ctx->ctx_pmds[i].val = val & ~ovfl_val;
1146 val &= ovfl_val;
1147 }
1148 ctx->th_pmds[i] = val;
1149
1150 DPRINT(("pmd[%d]=0x%lx soft_val=0x%lx\n",
1151 i,
1152 ctx->th_pmds[i],
1153 ctx->ctx_pmds[i].val));
1154 }
1155}
1156
1157
1158
1159
1160static inline void
1161pfm_copy_pmcs(struct task_struct *task, pfm_context_t *ctx)
1162{
1163 unsigned long mask = ctx->ctx_all_pmcs[0];
1164 int i;
1165
1166 DPRINT(("mask=0x%lx\n", mask));
1167
1168 for (i=0; mask; i++, mask>>=1) {
1169
1170 ctx->th_pmcs[i] = ctx->ctx_pmcs[i];
1171 DPRINT(("pmc[%d]=0x%lx\n", i, ctx->th_pmcs[i]));
1172 }
1173}
1174
1175
1176
1177static inline void
1178pfm_restore_pmcs(unsigned long *pmcs, unsigned long mask)
1179{
1180 int i;
1181
1182 for (i=0; mask; i++, mask>>=1) {
1183 if ((mask & 0x1) == 0) continue;
1184 ia64_set_pmc(i, pmcs[i]);
1185 }
1186 ia64_srlz_d();
1187}
1188
1189static inline int
1190pfm_uuid_cmp(pfm_uuid_t a, pfm_uuid_t b)
1191{
1192 return memcmp(a, b, sizeof(pfm_uuid_t));
1193}
1194
1195static inline int
1196pfm_buf_fmt_exit(pfm_buffer_fmt_t *fmt, struct task_struct *task, void *buf, struct pt_regs *regs)
1197{
1198 int ret = 0;
1199 if (fmt->fmt_exit) ret = (*fmt->fmt_exit)(task, buf, regs);
1200 return ret;
1201}
1202
1203static inline int
1204pfm_buf_fmt_getsize(pfm_buffer_fmt_t *fmt, struct task_struct *task, unsigned int flags, int cpu, void *arg, unsigned long *size)
1205{
1206 int ret = 0;
1207 if (fmt->fmt_getsize) ret = (*fmt->fmt_getsize)(task, flags, cpu, arg, size);
1208 return ret;
1209}
1210
1211
1212static inline int
1213pfm_buf_fmt_validate(pfm_buffer_fmt_t *fmt, struct task_struct *task, unsigned int flags,
1214 int cpu, void *arg)
1215{
1216 int ret = 0;
1217 if (fmt->fmt_validate) ret = (*fmt->fmt_validate)(task, flags, cpu, arg);
1218 return ret;
1219}
1220
1221static inline int
1222pfm_buf_fmt_init(pfm_buffer_fmt_t *fmt, struct task_struct *task, void *buf, unsigned int flags,
1223 int cpu, void *arg)
1224{
1225 int ret = 0;
1226 if (fmt->fmt_init) ret = (*fmt->fmt_init)(task, buf, flags, cpu, arg);
1227 return ret;
1228}
1229
1230static inline int
1231pfm_buf_fmt_restart(pfm_buffer_fmt_t *fmt, struct task_struct *task, pfm_ovfl_ctrl_t *ctrl, void *buf, struct pt_regs *regs)
1232{
1233 int ret = 0;
1234 if (fmt->fmt_restart) ret = (*fmt->fmt_restart)(task, ctrl, buf, regs);
1235 return ret;
1236}
1237
1238static inline int
1239pfm_buf_fmt_restart_active(pfm_buffer_fmt_t *fmt, struct task_struct *task, pfm_ovfl_ctrl_t *ctrl, void *buf, struct pt_regs *regs)
1240{
1241 int ret = 0;
1242 if (fmt->fmt_restart_active) ret = (*fmt->fmt_restart_active)(task, ctrl, buf, regs);
1243 return ret;
1244}
1245
1246static pfm_buffer_fmt_t *
1247__pfm_find_buffer_fmt(pfm_uuid_t uuid)
1248{
1249 struct list_head * pos;
1250 pfm_buffer_fmt_t * entry;
1251
1252 list_for_each(pos, &pfm_buffer_fmt_list) {
1253 entry = list_entry(pos, pfm_buffer_fmt_t, fmt_list);
1254 if (pfm_uuid_cmp(uuid, entry->fmt_uuid) == 0)
1255 return entry;
1256 }
1257 return NULL;
1258}
1259
1260
1261
1262
1263static pfm_buffer_fmt_t *
1264pfm_find_buffer_fmt(pfm_uuid_t uuid)
1265{
1266 pfm_buffer_fmt_t * fmt;
1267 spin_lock(&pfm_buffer_fmt_lock);
1268 fmt = __pfm_find_buffer_fmt(uuid);
1269 spin_unlock(&pfm_buffer_fmt_lock);
1270 return fmt;
1271}
1272
1273int
1274pfm_register_buffer_fmt(pfm_buffer_fmt_t *fmt)
1275{
1276 int ret = 0;
1277
1278
1279 if (fmt == NULL || fmt->fmt_name == NULL) return -EINVAL;
1280
1281
1282 if (fmt->fmt_handler == NULL) return -EINVAL;
1283
1284
1285
1286
1287
1288 spin_lock(&pfm_buffer_fmt_lock);
1289
1290 if (__pfm_find_buffer_fmt(fmt->fmt_uuid)) {
1291 printk(KERN_ERR "perfmon: duplicate sampling format: %s\n", fmt->fmt_name);
1292 ret = -EBUSY;
1293 goto out;
1294 }
1295 list_add(&fmt->fmt_list, &pfm_buffer_fmt_list);
1296 printk(KERN_INFO "perfmon: added sampling format %s\n", fmt->fmt_name);
1297
1298out:
1299 spin_unlock(&pfm_buffer_fmt_lock);
1300 return ret;
1301}
1302EXPORT_SYMBOL(pfm_register_buffer_fmt);
1303
1304int
1305pfm_unregister_buffer_fmt(pfm_uuid_t uuid)
1306{
1307 pfm_buffer_fmt_t *fmt;
1308 int ret = 0;
1309
1310 spin_lock(&pfm_buffer_fmt_lock);
1311
1312 fmt = __pfm_find_buffer_fmt(uuid);
1313 if (!fmt) {
1314 printk(KERN_ERR "perfmon: cannot unregister format, not found\n");
1315 ret = -EINVAL;
1316 goto out;
1317 }
1318 list_del_init(&fmt->fmt_list);
1319 printk(KERN_INFO "perfmon: removed sampling format: %s\n", fmt->fmt_name);
1320
1321out:
1322 spin_unlock(&pfm_buffer_fmt_lock);
1323 return ret;
1324
1325}
1326EXPORT_SYMBOL(pfm_unregister_buffer_fmt);
1327
1328static int
1329pfm_reserve_session(struct task_struct *task, int is_syswide, unsigned int cpu)
1330{
1331 unsigned long flags;
1332
1333
1334
1335 LOCK_PFS(flags);
1336
1337 DPRINT(("in sys_sessions=%u task_sessions=%u dbregs=%u syswide=%d cpu=%u\n",
1338 pfm_sessions.pfs_sys_sessions,
1339 pfm_sessions.pfs_task_sessions,
1340 pfm_sessions.pfs_sys_use_dbregs,
1341 is_syswide,
1342 cpu));
1343
1344 if (is_syswide) {
1345
1346
1347
1348 if (pfm_sessions.pfs_task_sessions > 0UL) {
1349 DPRINT(("system wide not possible, %u conflicting task_sessions\n",
1350 pfm_sessions.pfs_task_sessions));
1351 goto abort;
1352 }
1353
1354 if (pfm_sessions.pfs_sys_session[cpu]) goto error_conflict;
1355
1356 DPRINT(("reserving system wide session on CPU%u currently on CPU%u\n", cpu, smp_processor_id()));
1357
1358 pfm_sessions.pfs_sys_session[cpu] = task;
1359
1360 pfm_sessions.pfs_sys_sessions++ ;
1361
1362 } else {
1363 if (pfm_sessions.pfs_sys_sessions) goto abort;
1364 pfm_sessions.pfs_task_sessions++;
1365 }
1366
1367 DPRINT(("out sys_sessions=%u task_sessions=%u dbregs=%u syswide=%d cpu=%u\n",
1368 pfm_sessions.pfs_sys_sessions,
1369 pfm_sessions.pfs_task_sessions,
1370 pfm_sessions.pfs_sys_use_dbregs,
1371 is_syswide,
1372 cpu));
1373
1374
1375
1376
1377 cpu_idle_poll_ctrl(true);
1378
1379 UNLOCK_PFS(flags);
1380
1381 return 0;
1382
1383error_conflict:
1384 DPRINT(("system wide not possible, conflicting session [%d] on CPU%d\n",
1385 task_pid_nr(pfm_sessions.pfs_sys_session[cpu]),
1386 cpu));
1387abort:
1388 UNLOCK_PFS(flags);
1389
1390 return -EBUSY;
1391
1392}
1393
1394static int
1395pfm_unreserve_session(pfm_context_t *ctx, int is_syswide, unsigned int cpu)
1396{
1397 unsigned long flags;
1398
1399
1400
1401 LOCK_PFS(flags);
1402
1403 DPRINT(("in sys_sessions=%u task_sessions=%u dbregs=%u syswide=%d cpu=%u\n",
1404 pfm_sessions.pfs_sys_sessions,
1405 pfm_sessions.pfs_task_sessions,
1406 pfm_sessions.pfs_sys_use_dbregs,
1407 is_syswide,
1408 cpu));
1409
1410
1411 if (is_syswide) {
1412 pfm_sessions.pfs_sys_session[cpu] = NULL;
1413
1414
1415
1416 if (ctx && ctx->ctx_fl_using_dbreg) {
1417 if (pfm_sessions.pfs_sys_use_dbregs == 0) {
1418 printk(KERN_ERR "perfmon: invalid release for ctx %p sys_use_dbregs=0\n", ctx);
1419 } else {
1420 pfm_sessions.pfs_sys_use_dbregs--;
1421 }
1422 }
1423 pfm_sessions.pfs_sys_sessions--;
1424 } else {
1425 pfm_sessions.pfs_task_sessions--;
1426 }
1427 DPRINT(("out sys_sessions=%u task_sessions=%u dbregs=%u syswide=%d cpu=%u\n",
1428 pfm_sessions.pfs_sys_sessions,
1429 pfm_sessions.pfs_task_sessions,
1430 pfm_sessions.pfs_sys_use_dbregs,
1431 is_syswide,
1432 cpu));
1433
1434
1435 cpu_idle_poll_ctrl(false);
1436
1437 UNLOCK_PFS(flags);
1438
1439 return 0;
1440}
1441
1442
1443
1444
1445
1446
1447static int
1448pfm_remove_smpl_mapping(void *vaddr, unsigned long size)
1449{
1450 struct task_struct *task = current;
1451 int r;
1452
1453
1454 if (task->mm == NULL || size == 0UL || vaddr == NULL) {
1455 printk(KERN_ERR "perfmon: pfm_remove_smpl_mapping [%d] invalid context mm=%p\n", task_pid_nr(task), task->mm);
1456 return -EINVAL;
1457 }
1458
1459 DPRINT(("smpl_vaddr=%p size=%lu\n", vaddr, size));
1460
1461
1462
1463
1464 r = vm_munmap((unsigned long)vaddr, size);
1465
1466 if (r !=0) {
1467 printk(KERN_ERR "perfmon: [%d] unable to unmap sampling buffer @%p size=%lu\n", task_pid_nr(task), vaddr, size);
1468 }
1469
1470 DPRINT(("do_unmap(%p, %lu)=%d\n", vaddr, size, r));
1471
1472 return 0;
1473}
1474
1475
1476
1477
1478#if 0
1479static int
1480pfm_free_smpl_buffer(pfm_context_t *ctx)
1481{
1482 pfm_buffer_fmt_t *fmt;
1483
1484 if (ctx->ctx_smpl_hdr == NULL) goto invalid_free;
1485
1486
1487
1488
1489 fmt = ctx->ctx_buf_fmt;
1490
1491 DPRINT(("sampling buffer @%p size %lu vaddr=%p\n",
1492 ctx->ctx_smpl_hdr,
1493 ctx->ctx_smpl_size,
1494 ctx->ctx_smpl_vaddr));
1495
1496 pfm_buf_fmt_exit(fmt, current, NULL, NULL);
1497
1498
1499
1500
1501 pfm_rvfree(ctx->ctx_smpl_hdr, ctx->ctx_smpl_size);
1502
1503 ctx->ctx_smpl_hdr = NULL;
1504 ctx->ctx_smpl_size = 0UL;
1505
1506 return 0;
1507
1508invalid_free:
1509 printk(KERN_ERR "perfmon: pfm_free_smpl_buffer [%d] no buffer\n", task_pid_nr(current));
1510 return -EINVAL;
1511}
1512#endif
1513
1514static inline void
1515pfm_exit_smpl_buffer(pfm_buffer_fmt_t *fmt)
1516{
1517 if (fmt == NULL) return;
1518
1519 pfm_buf_fmt_exit(fmt, current, NULL, NULL);
1520
1521}
1522
1523
1524
1525
1526
1527
1528
1529static struct vfsmount *pfmfs_mnt __read_mostly;
1530
1531static int __init
1532init_pfm_fs(void)
1533{
1534 int err = register_filesystem(&pfm_fs_type);
1535 if (!err) {
1536 pfmfs_mnt = kern_mount(&pfm_fs_type);
1537 err = PTR_ERR(pfmfs_mnt);
1538 if (IS_ERR(pfmfs_mnt))
1539 unregister_filesystem(&pfm_fs_type);
1540 else
1541 err = 0;
1542 }
1543 return err;
1544}
1545
1546static ssize_t
1547pfm_read(struct file *filp, char __user *buf, size_t size, loff_t *ppos)
1548{
1549 pfm_context_t *ctx;
1550 pfm_msg_t *msg;
1551 ssize_t ret;
1552 unsigned long flags;
1553 DECLARE_WAITQUEUE(wait, current);
1554 if (PFM_IS_FILE(filp) == 0) {
1555 printk(KERN_ERR "perfmon: pfm_poll: bad magic [%d]\n", task_pid_nr(current));
1556 return -EINVAL;
1557 }
1558
1559 ctx = filp->private_data;
1560 if (ctx == NULL) {
1561 printk(KERN_ERR "perfmon: pfm_read: NULL ctx [%d]\n", task_pid_nr(current));
1562 return -EINVAL;
1563 }
1564
1565
1566
1567
1568 if (size < sizeof(pfm_msg_t)) {
1569 DPRINT(("message is too small ctx=%p (>=%ld)\n", ctx, sizeof(pfm_msg_t)));
1570 return -EINVAL;
1571 }
1572
1573 PROTECT_CTX(ctx, flags);
1574
1575
1576
1577
1578 add_wait_queue(&ctx->ctx_msgq_wait, &wait);
1579
1580
1581 for(;;) {
1582
1583
1584
1585
1586 set_current_state(TASK_INTERRUPTIBLE);
1587
1588 DPRINT(("head=%d tail=%d\n", ctx->ctx_msgq_head, ctx->ctx_msgq_tail));
1589
1590 ret = 0;
1591 if(PFM_CTXQ_EMPTY(ctx) == 0) break;
1592
1593 UNPROTECT_CTX(ctx, flags);
1594
1595
1596
1597
1598 ret = -EAGAIN;
1599 if(filp->f_flags & O_NONBLOCK) break;
1600
1601
1602
1603
1604 if(signal_pending(current)) {
1605 ret = -EINTR;
1606 break;
1607 }
1608
1609
1610
1611 schedule();
1612
1613 PROTECT_CTX(ctx, flags);
1614 }
1615 DPRINT(("[%d] back to running ret=%ld\n", task_pid_nr(current), ret));
1616 set_current_state(TASK_RUNNING);
1617 remove_wait_queue(&ctx->ctx_msgq_wait, &wait);
1618
1619 if (ret < 0) goto abort;
1620
1621 ret = -EINVAL;
1622 msg = pfm_get_next_msg(ctx);
1623 if (msg == NULL) {
1624 printk(KERN_ERR "perfmon: pfm_read no msg for ctx=%p [%d]\n", ctx, task_pid_nr(current));
1625 goto abort_locked;
1626 }
1627
1628 DPRINT(("fd=%d type=%d\n", msg->pfm_gen_msg.msg_ctx_fd, msg->pfm_gen_msg.msg_type));
1629
1630 ret = -EFAULT;
1631 if(copy_to_user(buf, msg, sizeof(pfm_msg_t)) == 0) ret = sizeof(pfm_msg_t);
1632
1633abort_locked:
1634 UNPROTECT_CTX(ctx, flags);
1635abort:
1636 return ret;
1637}
1638
1639static ssize_t
1640pfm_write(struct file *file, const char __user *ubuf,
1641 size_t size, loff_t *ppos)
1642{
1643 DPRINT(("pfm_write called\n"));
1644 return -EINVAL;
1645}
1646
1647static __poll_t
1648pfm_poll(struct file *filp, poll_table * wait)
1649{
1650 pfm_context_t *ctx;
1651 unsigned long flags;
1652 __poll_t mask = 0;
1653
1654 if (PFM_IS_FILE(filp) == 0) {
1655 printk(KERN_ERR "perfmon: pfm_poll: bad magic [%d]\n", task_pid_nr(current));
1656 return 0;
1657 }
1658
1659 ctx = filp->private_data;
1660 if (ctx == NULL) {
1661 printk(KERN_ERR "perfmon: pfm_poll: NULL ctx [%d]\n", task_pid_nr(current));
1662 return 0;
1663 }
1664
1665
1666 DPRINT(("pfm_poll ctx_fd=%d before poll_wait\n", ctx->ctx_fd));
1667
1668 poll_wait(filp, &ctx->ctx_msgq_wait, wait);
1669
1670 PROTECT_CTX(ctx, flags);
1671
1672 if (PFM_CTXQ_EMPTY(ctx) == 0)
1673 mask = EPOLLIN | EPOLLRDNORM;
1674
1675 UNPROTECT_CTX(ctx, flags);
1676
1677 DPRINT(("pfm_poll ctx_fd=%d mask=0x%x\n", ctx->ctx_fd, mask));
1678
1679 return mask;
1680}
1681
1682static long
1683pfm_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1684{
1685 DPRINT(("pfm_ioctl called\n"));
1686 return -EINVAL;
1687}
1688
1689
1690
1691
1692static inline int
1693pfm_do_fasync(int fd, struct file *filp, pfm_context_t *ctx, int on)
1694{
1695 int ret;
1696
1697 ret = fasync_helper (fd, filp, on, &ctx->ctx_async_queue);
1698
1699 DPRINT(("pfm_fasync called by [%d] on ctx_fd=%d on=%d async_queue=%p ret=%d\n",
1700 task_pid_nr(current),
1701 fd,
1702 on,
1703 ctx->ctx_async_queue, ret));
1704
1705 return ret;
1706}
1707
1708static int
1709pfm_fasync(int fd, struct file *filp, int on)
1710{
1711 pfm_context_t *ctx;
1712 int ret;
1713
1714 if (PFM_IS_FILE(filp) == 0) {
1715 printk(KERN_ERR "perfmon: pfm_fasync bad magic [%d]\n", task_pid_nr(current));
1716 return -EBADF;
1717 }
1718
1719 ctx = filp->private_data;
1720 if (ctx == NULL) {
1721 printk(KERN_ERR "perfmon: pfm_fasync NULL ctx [%d]\n", task_pid_nr(current));
1722 return -EBADF;
1723 }
1724
1725
1726
1727
1728
1729
1730
1731 ret = pfm_do_fasync(fd, filp, ctx, on);
1732
1733
1734 DPRINT(("pfm_fasync called on ctx_fd=%d on=%d async_queue=%p ret=%d\n",
1735 fd,
1736 on,
1737 ctx->ctx_async_queue, ret));
1738
1739 return ret;
1740}
1741
1742#ifdef CONFIG_SMP
1743
1744
1745
1746
1747
1748static void
1749pfm_syswide_force_stop(void *info)
1750{
1751 pfm_context_t *ctx = (pfm_context_t *)info;
1752 struct pt_regs *regs = task_pt_regs(current);
1753 struct task_struct *owner;
1754 unsigned long flags;
1755 int ret;
1756
1757 if (ctx->ctx_cpu != smp_processor_id()) {
1758 printk(KERN_ERR "perfmon: pfm_syswide_force_stop for CPU%d but on CPU%d\n",
1759 ctx->ctx_cpu,
1760 smp_processor_id());
1761 return;
1762 }
1763 owner = GET_PMU_OWNER();
1764 if (owner != ctx->ctx_task) {
1765 printk(KERN_ERR "perfmon: pfm_syswide_force_stop CPU%d unexpected owner [%d] instead of [%d]\n",
1766 smp_processor_id(),
1767 task_pid_nr(owner), task_pid_nr(ctx->ctx_task));
1768 return;
1769 }
1770 if (GET_PMU_CTX() != ctx) {
1771 printk(KERN_ERR "perfmon: pfm_syswide_force_stop CPU%d unexpected ctx %p instead of %p\n",
1772 smp_processor_id(),
1773 GET_PMU_CTX(), ctx);
1774 return;
1775 }
1776
1777 DPRINT(("on CPU%d forcing system wide stop for [%d]\n", smp_processor_id(), task_pid_nr(ctx->ctx_task)));
1778
1779
1780
1781
1782
1783 local_irq_save(flags);
1784
1785 ret = pfm_context_unload(ctx, NULL, 0, regs);
1786 if (ret) {
1787 DPRINT(("context_unload returned %d\n", ret));
1788 }
1789
1790
1791
1792
1793 local_irq_restore(flags);
1794}
1795
1796static void
1797pfm_syswide_cleanup_other_cpu(pfm_context_t *ctx)
1798{
1799 int ret;
1800
1801 DPRINT(("calling CPU%d for cleanup\n", ctx->ctx_cpu));
1802 ret = smp_call_function_single(ctx->ctx_cpu, pfm_syswide_force_stop, ctx, 1);
1803 DPRINT(("called CPU%d for cleanup ret=%d\n", ctx->ctx_cpu, ret));
1804}
1805#endif
1806
1807
1808
1809
1810
1811static int
1812pfm_flush(struct file *filp, fl_owner_t id)
1813{
1814 pfm_context_t *ctx;
1815 struct task_struct *task;
1816 struct pt_regs *regs;
1817 unsigned long flags;
1818 unsigned long smpl_buf_size = 0UL;
1819 void *smpl_buf_vaddr = NULL;
1820 int state, is_system;
1821
1822 if (PFM_IS_FILE(filp) == 0) {
1823 DPRINT(("bad magic for\n"));
1824 return -EBADF;
1825 }
1826
1827 ctx = filp->private_data;
1828 if (ctx == NULL) {
1829 printk(KERN_ERR "perfmon: pfm_flush: NULL ctx [%d]\n", task_pid_nr(current));
1830 return -EBADF;
1831 }
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846 PROTECT_CTX(ctx, flags);
1847
1848 state = ctx->ctx_state;
1849 is_system = ctx->ctx_fl_system;
1850
1851 task = PFM_CTX_TASK(ctx);
1852 regs = task_pt_regs(task);
1853
1854 DPRINT(("ctx_state=%d is_current=%d\n",
1855 state,
1856 task == current ? 1 : 0));
1857
1858
1859
1860
1861
1862
1863
1864
1865 if (task == current) {
1866#ifdef CONFIG_SMP
1867
1868
1869
1870
1871
1872
1873
1874 if (is_system && ctx->ctx_cpu != smp_processor_id()) {
1875
1876 DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
1877
1878
1879
1880 local_irq_restore(flags);
1881
1882 pfm_syswide_cleanup_other_cpu(ctx);
1883
1884
1885
1886
1887 local_irq_save(flags);
1888
1889
1890
1891
1892 } else
1893#endif
1894 {
1895
1896 DPRINT(("forcing unload\n"));
1897
1898
1899
1900
1901 pfm_context_unload(ctx, NULL, 0, regs);
1902
1903 DPRINT(("ctx_state=%d\n", ctx->ctx_state));
1904 }
1905 }
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918 if (ctx->ctx_smpl_vaddr && current->mm) {
1919 smpl_buf_vaddr = ctx->ctx_smpl_vaddr;
1920 smpl_buf_size = ctx->ctx_smpl_size;
1921 }
1922
1923 UNPROTECT_CTX(ctx, flags);
1924
1925
1926
1927
1928
1929
1930
1931 if (smpl_buf_vaddr) pfm_remove_smpl_mapping(smpl_buf_vaddr, smpl_buf_size);
1932
1933 return 0;
1934}
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950static int
1951pfm_close(struct inode *inode, struct file *filp)
1952{
1953 pfm_context_t *ctx;
1954 struct task_struct *task;
1955 struct pt_regs *regs;
1956 DECLARE_WAITQUEUE(wait, current);
1957 unsigned long flags;
1958 unsigned long smpl_buf_size = 0UL;
1959 void *smpl_buf_addr = NULL;
1960 int free_possible = 1;
1961 int state, is_system;
1962
1963 DPRINT(("pfm_close called private=%p\n", filp->private_data));
1964
1965 if (PFM_IS_FILE(filp) == 0) {
1966 DPRINT(("bad magic\n"));
1967 return -EBADF;
1968 }
1969
1970 ctx = filp->private_data;
1971 if (ctx == NULL) {
1972 printk(KERN_ERR "perfmon: pfm_close: NULL ctx [%d]\n", task_pid_nr(current));
1973 return -EBADF;
1974 }
1975
1976 PROTECT_CTX(ctx, flags);
1977
1978 state = ctx->ctx_state;
1979 is_system = ctx->ctx_fl_system;
1980
1981 task = PFM_CTX_TASK(ctx);
1982 regs = task_pt_regs(task);
1983
1984 DPRINT(("ctx_state=%d is_current=%d\n",
1985 state,
1986 task == current ? 1 : 0));
1987
1988
1989
1990
1991 if (state == PFM_CTX_UNLOADED) goto doit;
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005 if (state == PFM_CTX_MASKED && CTX_OVFL_NOBLOCK(ctx) == 0) {
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021 ctx->ctx_fl_going_zombie = 1;
2022
2023
2024
2025
2026 complete(&ctx->ctx_restart_done);
2027
2028 DPRINT(("waking up ctx_state=%d\n", state));
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038 set_current_state(TASK_INTERRUPTIBLE);
2039 add_wait_queue(&ctx->ctx_zombieq, &wait);
2040
2041 UNPROTECT_CTX(ctx, flags);
2042
2043
2044
2045
2046
2047
2048 schedule();
2049
2050
2051 PROTECT_CTX(ctx, flags);
2052
2053
2054 remove_wait_queue(&ctx->ctx_zombieq, &wait);
2055 set_current_state(TASK_RUNNING);
2056
2057
2058
2059
2060 DPRINT(("after zombie wakeup ctx_state=%d for\n", state));
2061 }
2062 else if (task != current) {
2063#ifdef CONFIG_SMP
2064
2065
2066
2067 ctx->ctx_state = PFM_CTX_ZOMBIE;
2068
2069 DPRINT(("zombie ctx for [%d]\n", task_pid_nr(task)));
2070
2071
2072
2073
2074 free_possible = 0;
2075#else
2076 pfm_context_unload(ctx, NULL, 0, regs);
2077#endif
2078 }
2079
2080doit:
2081
2082 state = ctx->ctx_state;
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098 if (ctx->ctx_smpl_hdr) {
2099 smpl_buf_addr = ctx->ctx_smpl_hdr;
2100 smpl_buf_size = ctx->ctx_smpl_size;
2101
2102 ctx->ctx_smpl_hdr = NULL;
2103 ctx->ctx_fl_is_sampling = 0;
2104 }
2105
2106 DPRINT(("ctx_state=%d free_possible=%d addr=%p size=%lu\n",
2107 state,
2108 free_possible,
2109 smpl_buf_addr,
2110 smpl_buf_size));
2111
2112 if (smpl_buf_addr) pfm_exit_smpl_buffer(ctx->ctx_buf_fmt);
2113
2114
2115
2116
2117 if (state == PFM_CTX_ZOMBIE) {
2118 pfm_unreserve_session(ctx, ctx->ctx_fl_system , ctx->ctx_cpu);
2119 }
2120
2121
2122
2123
2124
2125 filp->private_data = NULL;
2126
2127
2128
2129
2130
2131
2132
2133
2134 UNPROTECT_CTX(ctx, flags);
2135
2136
2137
2138
2139
2140 if (smpl_buf_addr) pfm_rvfree(smpl_buf_addr, smpl_buf_size);
2141
2142
2143
2144
2145 if (free_possible) pfm_context_free(ctx);
2146
2147 return 0;
2148}
2149
2150static const struct file_operations pfm_file_ops = {
2151 .llseek = no_llseek,
2152 .read = pfm_read,
2153 .write = pfm_write,
2154 .poll = pfm_poll,
2155 .unlocked_ioctl = pfm_ioctl,
2156 .fasync = pfm_fasync,
2157 .release = pfm_close,
2158 .flush = pfm_flush
2159};
2160
2161static char *pfmfs_dname(struct dentry *dentry, char *buffer, int buflen)
2162{
2163 return dynamic_dname(dentry, buffer, buflen, "pfm:[%lu]",
2164 d_inode(dentry)->i_ino);
2165}
2166
2167static const struct dentry_operations pfmfs_dentry_operations = {
2168 .d_delete = always_delete_dentry,
2169 .d_dname = pfmfs_dname,
2170};
2171
2172
2173static struct file *
2174pfm_alloc_file(pfm_context_t *ctx)
2175{
2176 struct file *file;
2177 struct inode *inode;
2178 struct path path;
2179 struct qstr this = { .name = "" };
2180
2181
2182
2183
2184 inode = new_inode(pfmfs_mnt->mnt_sb);
2185 if (!inode)
2186 return ERR_PTR(-ENOMEM);
2187
2188 DPRINT(("new inode ino=%ld @%p\n", inode->i_ino, inode));
2189
2190 inode->i_mode = S_IFCHR|S_IRUGO;
2191 inode->i_uid = current_fsuid();
2192 inode->i_gid = current_fsgid();
2193
2194
2195
2196
2197 path.dentry = d_alloc(pfmfs_mnt->mnt_root, &this);
2198 if (!path.dentry) {
2199 iput(inode);
2200 return ERR_PTR(-ENOMEM);
2201 }
2202 path.mnt = mntget(pfmfs_mnt);
2203
2204 d_add(path.dentry, inode);
2205
2206 file = alloc_file(&path, FMODE_READ, &pfm_file_ops);
2207 if (IS_ERR(file)) {
2208 path_put(&path);
2209 return file;
2210 }
2211
2212 file->f_flags = O_RDONLY;
2213 file->private_data = ctx;
2214
2215 return file;
2216}
2217
2218static int
2219pfm_remap_buffer(struct vm_area_struct *vma, unsigned long buf, unsigned long addr, unsigned long size)
2220{
2221 DPRINT(("CPU%d buf=0x%lx addr=0x%lx size=%ld\n", smp_processor_id(), buf, addr, size));
2222
2223 while (size > 0) {
2224 unsigned long pfn = ia64_tpa(buf) >> PAGE_SHIFT;
2225
2226
2227 if (remap_pfn_range(vma, addr, pfn, PAGE_SIZE, PAGE_READONLY))
2228 return -ENOMEM;
2229
2230 addr += PAGE_SIZE;
2231 buf += PAGE_SIZE;
2232 size -= PAGE_SIZE;
2233 }
2234 return 0;
2235}
2236
2237
2238
2239
2240static int
2241pfm_smpl_buffer_alloc(struct task_struct *task, struct file *filp, pfm_context_t *ctx, unsigned long rsize, void **user_vaddr)
2242{
2243 struct mm_struct *mm = task->mm;
2244 struct vm_area_struct *vma = NULL;
2245 unsigned long size;
2246 void *smpl_buf;
2247
2248
2249
2250
2251
2252 size = PAGE_ALIGN(rsize);
2253
2254 DPRINT(("sampling buffer rsize=%lu size=%lu bytes\n", rsize, size));
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264 if (size > task_rlimit(task, RLIMIT_MEMLOCK))
2265 return -ENOMEM;
2266
2267
2268
2269
2270
2271
2272 smpl_buf = pfm_rvmalloc(size);
2273 if (smpl_buf == NULL) {
2274 DPRINT(("Can't allocate sampling buffer\n"));
2275 return -ENOMEM;
2276 }
2277
2278 DPRINT(("smpl_buf @%p\n", smpl_buf));
2279
2280
2281 vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
2282 if (!vma) {
2283 DPRINT(("Cannot allocate vma\n"));
2284 goto error_kmem;
2285 }
2286 INIT_LIST_HEAD(&vma->anon_vma_chain);
2287
2288
2289
2290
2291 vma->vm_mm = mm;
2292 vma->vm_file = get_file(filp);
2293 vma->vm_flags = VM_READ|VM_MAYREAD|VM_DONTEXPAND|VM_DONTDUMP;
2294 vma->vm_page_prot = PAGE_READONLY;
2295
2296
2297
2298
2299
2300
2301 ctx->ctx_smpl_hdr = smpl_buf;
2302 ctx->ctx_smpl_size = size;
2303
2304
2305
2306
2307
2308
2309
2310 down_write(&task->mm->mmap_sem);
2311
2312
2313 vma->vm_start = get_unmapped_area(NULL, 0, size, 0, MAP_PRIVATE|MAP_ANONYMOUS);
2314 if (IS_ERR_VALUE(vma->vm_start)) {
2315 DPRINT(("Cannot find unmapped area for size %ld\n", size));
2316 up_write(&task->mm->mmap_sem);
2317 goto error;
2318 }
2319 vma->vm_end = vma->vm_start + size;
2320 vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT;
2321
2322 DPRINT(("aligned size=%ld, hdr=%p mapped @0x%lx\n", size, ctx->ctx_smpl_hdr, vma->vm_start));
2323
2324
2325 if (pfm_remap_buffer(vma, (unsigned long)smpl_buf, vma->vm_start, size)) {
2326 DPRINT(("Can't remap buffer\n"));
2327 up_write(&task->mm->mmap_sem);
2328 goto error;
2329 }
2330
2331
2332
2333
2334
2335 insert_vm_struct(mm, vma);
2336
2337 vm_stat_account(vma->vm_mm, vma->vm_flags, vma_pages(vma));
2338 up_write(&task->mm->mmap_sem);
2339
2340
2341
2342
2343 ctx->ctx_smpl_vaddr = (void *)vma->vm_start;
2344 *(unsigned long *)user_vaddr = vma->vm_start;
2345
2346 return 0;
2347
2348error:
2349 kmem_cache_free(vm_area_cachep, vma);
2350error_kmem:
2351 pfm_rvfree(smpl_buf, size);
2352
2353 return -ENOMEM;
2354}
2355
2356
2357
2358
2359static int
2360pfm_bad_permissions(struct task_struct *task)
2361{
2362 const struct cred *tcred;
2363 kuid_t uid = current_uid();
2364 kgid_t gid = current_gid();
2365 int ret;
2366
2367 rcu_read_lock();
2368 tcred = __task_cred(task);
2369
2370
2371 DPRINT(("cur: uid=%d gid=%d task: euid=%d suid=%d uid=%d egid=%d sgid=%d\n",
2372 from_kuid(&init_user_ns, uid),
2373 from_kgid(&init_user_ns, gid),
2374 from_kuid(&init_user_ns, tcred->euid),
2375 from_kuid(&init_user_ns, tcred->suid),
2376 from_kuid(&init_user_ns, tcred->uid),
2377 from_kgid(&init_user_ns, tcred->egid),
2378 from_kgid(&init_user_ns, tcred->sgid)));
2379
2380 ret = ((!uid_eq(uid, tcred->euid))
2381 || (!uid_eq(uid, tcred->suid))
2382 || (!uid_eq(uid, tcred->uid))
2383 || (!gid_eq(gid, tcred->egid))
2384 || (!gid_eq(gid, tcred->sgid))
2385 || (!gid_eq(gid, tcred->gid))) && !capable(CAP_SYS_PTRACE);
2386
2387 rcu_read_unlock();
2388 return ret;
2389}
2390
2391static int
2392pfarg_is_sane(struct task_struct *task, pfarg_context_t *pfx)
2393{
2394 int ctx_flags;
2395
2396
2397
2398 ctx_flags = pfx->ctx_flags;
2399
2400 if (ctx_flags & PFM_FL_SYSTEM_WIDE) {
2401
2402
2403
2404
2405 if (ctx_flags & PFM_FL_NOTIFY_BLOCK) {
2406 DPRINT(("cannot use blocking mode when in system wide monitoring\n"));
2407 return -EINVAL;
2408 }
2409 } else {
2410 }
2411
2412
2413 return 0;
2414}
2415
2416static int
2417pfm_setup_buffer_fmt(struct task_struct *task, struct file *filp, pfm_context_t *ctx, unsigned int ctx_flags,
2418 unsigned int cpu, pfarg_context_t *arg)
2419{
2420 pfm_buffer_fmt_t *fmt = NULL;
2421 unsigned long size = 0UL;
2422 void *uaddr = NULL;
2423 void *fmt_arg = NULL;
2424 int ret = 0;
2425#define PFM_CTXARG_BUF_ARG(a) (pfm_buffer_fmt_t *)(a+1)
2426
2427
2428 fmt = pfm_find_buffer_fmt(arg->ctx_smpl_buf_id);
2429 if (fmt == NULL) {
2430 DPRINT(("[%d] cannot find buffer format\n", task_pid_nr(task)));
2431 return -EINVAL;
2432 }
2433
2434
2435
2436
2437 if (fmt->fmt_arg_size) fmt_arg = PFM_CTXARG_BUF_ARG(arg);
2438
2439 ret = pfm_buf_fmt_validate(fmt, task, ctx_flags, cpu, fmt_arg);
2440
2441 DPRINT(("[%d] after validate(0x%x,%d,%p)=%d\n", task_pid_nr(task), ctx_flags, cpu, fmt_arg, ret));
2442
2443 if (ret) goto error;
2444
2445
2446 ctx->ctx_buf_fmt = fmt;
2447 ctx->ctx_fl_is_sampling = 1;
2448
2449
2450
2451
2452 ret = pfm_buf_fmt_getsize(fmt, task, ctx_flags, cpu, fmt_arg, &size);
2453 if (ret) goto error;
2454
2455 if (size) {
2456
2457
2458
2459 ret = pfm_smpl_buffer_alloc(current, filp, ctx, size, &uaddr);
2460 if (ret) goto error;
2461
2462
2463 arg->ctx_smpl_vaddr = uaddr;
2464 }
2465 ret = pfm_buf_fmt_init(fmt, task, ctx->ctx_smpl_hdr, ctx_flags, cpu, fmt_arg);
2466
2467error:
2468 return ret;
2469}
2470
2471static void
2472pfm_reset_pmu_state(pfm_context_t *ctx)
2473{
2474 int i;
2475
2476
2477
2478
2479 for (i=1; PMC_IS_LAST(i) == 0; i++) {
2480 if (PMC_IS_IMPL(i) == 0) continue;
2481 ctx->ctx_pmcs[i] = PMC_DFL_VAL(i);
2482 DPRINT(("pmc[%d]=0x%lx\n", i, ctx->ctx_pmcs[i]));
2483 }
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511 ctx->ctx_all_pmcs[0] = pmu_conf->impl_pmcs[0] & ~0x1;
2512
2513
2514
2515
2516 ctx->ctx_all_pmds[0] = pmu_conf->impl_pmds[0];
2517
2518 DPRINT(("<%d> all_pmcs=0x%lx all_pmds=0x%lx\n", ctx->ctx_fd, ctx->ctx_all_pmcs[0],ctx->ctx_all_pmds[0]));
2519
2520
2521
2522
2523 ctx->ctx_used_ibrs[0] = 0UL;
2524 ctx->ctx_used_dbrs[0] = 0UL;
2525}
2526
2527static int
2528pfm_ctx_getsize(void *arg, size_t *sz)
2529{
2530 pfarg_context_t *req = (pfarg_context_t *)arg;
2531 pfm_buffer_fmt_t *fmt;
2532
2533 *sz = 0;
2534
2535 if (!pfm_uuid_cmp(req->ctx_smpl_buf_id, pfm_null_uuid)) return 0;
2536
2537 fmt = pfm_find_buffer_fmt(req->ctx_smpl_buf_id);
2538 if (fmt == NULL) {
2539 DPRINT(("cannot find buffer format\n"));
2540 return -EINVAL;
2541 }
2542
2543 *sz = fmt->fmt_arg_size;
2544 DPRINT(("arg_size=%lu\n", *sz));
2545
2546 return 0;
2547}
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557static int
2558pfm_task_incompatible(pfm_context_t *ctx, struct task_struct *task)
2559{
2560
2561
2562
2563 if (task->mm == NULL) {
2564 DPRINT(("task [%d] has not memory context (kernel thread)\n", task_pid_nr(task)));
2565 return -EPERM;
2566 }
2567 if (pfm_bad_permissions(task)) {
2568 DPRINT(("no permission to attach to [%d]\n", task_pid_nr(task)));
2569 return -EPERM;
2570 }
2571
2572
2573
2574 if (CTX_OVFL_NOBLOCK(ctx) == 0 && task == current) {
2575 DPRINT(("cannot load a blocking context on self for [%d]\n", task_pid_nr(task)));
2576 return -EINVAL;
2577 }
2578
2579 if (task->exit_state == EXIT_ZOMBIE) {
2580 DPRINT(("cannot attach to zombie task [%d]\n", task_pid_nr(task)));
2581 return -EBUSY;
2582 }
2583
2584
2585
2586
2587 if (task == current) return 0;
2588
2589 if (!task_is_stopped_or_traced(task)) {
2590 DPRINT(("cannot attach to non-stopped task [%d] state=%ld\n", task_pid_nr(task), task->state));
2591 return -EBUSY;
2592 }
2593
2594
2595
2596 wait_task_inactive(task, 0);
2597
2598
2599
2600 return 0;
2601}
2602
2603static int
2604pfm_get_task(pfm_context_t *ctx, pid_t pid, struct task_struct **task)
2605{
2606 struct task_struct *p = current;
2607 int ret;
2608
2609
2610 if (pid < 2) return -EPERM;
2611
2612 if (pid != task_pid_vnr(current)) {
2613
2614 p = find_get_task_by_vpid(pid);
2615 if (!p)
2616 return -ESRCH;
2617 }
2618
2619 ret = pfm_task_incompatible(ctx, p);
2620 if (ret == 0) {
2621 *task = p;
2622 } else if (p != current) {
2623 pfm_put_task(p);
2624 }
2625 return ret;
2626}
2627
2628
2629
2630static int
2631pfm_context_create(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
2632{
2633 pfarg_context_t *req = (pfarg_context_t *)arg;
2634 struct file *filp;
2635 struct path path;
2636 int ctx_flags;
2637 int fd;
2638 int ret;
2639
2640
2641 ret = pfarg_is_sane(current, req);
2642 if (ret < 0)
2643 return ret;
2644
2645 ctx_flags = req->ctx_flags;
2646
2647 ret = -ENOMEM;
2648
2649 fd = get_unused_fd_flags(0);
2650 if (fd < 0)
2651 return fd;
2652
2653 ctx = pfm_context_alloc(ctx_flags);
2654 if (!ctx)
2655 goto error;
2656
2657 filp = pfm_alloc_file(ctx);
2658 if (IS_ERR(filp)) {
2659 ret = PTR_ERR(filp);
2660 goto error_file;
2661 }
2662
2663 req->ctx_fd = ctx->ctx_fd = fd;
2664
2665
2666
2667
2668 if (pfm_uuid_cmp(req->ctx_smpl_buf_id, pfm_null_uuid)) {
2669 ret = pfm_setup_buffer_fmt(current, filp, ctx, ctx_flags, 0, req);
2670 if (ret)
2671 goto buffer_error;
2672 }
2673
2674 DPRINT(("ctx=%p flags=0x%x system=%d notify_block=%d excl_idle=%d no_msg=%d ctx_fd=%d\n",
2675 ctx,
2676 ctx_flags,
2677 ctx->ctx_fl_system,
2678 ctx->ctx_fl_block,
2679 ctx->ctx_fl_excl_idle,
2680 ctx->ctx_fl_no_msg,
2681 ctx->ctx_fd));
2682
2683
2684
2685
2686 pfm_reset_pmu_state(ctx);
2687
2688 fd_install(fd, filp);
2689
2690 return 0;
2691
2692buffer_error:
2693 path = filp->f_path;
2694 put_filp(filp);
2695 path_put(&path);
2696
2697 if (ctx->ctx_buf_fmt) {
2698 pfm_buf_fmt_exit(ctx->ctx_buf_fmt, current, NULL, regs);
2699 }
2700error_file:
2701 pfm_context_free(ctx);
2702
2703error:
2704 put_unused_fd(fd);
2705 return ret;
2706}
2707
2708static inline unsigned long
2709pfm_new_counter_value (pfm_counter_t *reg, int is_long_reset)
2710{
2711 unsigned long val = is_long_reset ? reg->long_reset : reg->short_reset;
2712 unsigned long new_seed, old_seed = reg->seed, mask = reg->mask;
2713 extern unsigned long carta_random32 (unsigned long seed);
2714
2715 if (reg->flags & PFM_REGFL_RANDOM) {
2716 new_seed = carta_random32(old_seed);
2717 val -= (old_seed & mask);
2718 if ((mask >> 32) != 0)
2719
2720 new_seed |= carta_random32(old_seed >> 32) << 32;
2721 reg->seed = new_seed;
2722 }
2723 reg->lval = val;
2724 return val;
2725}
2726
2727static void
2728pfm_reset_regs_masked(pfm_context_t *ctx, unsigned long *ovfl_regs, int is_long_reset)
2729{
2730 unsigned long mask = ovfl_regs[0];
2731 unsigned long reset_others = 0UL;
2732 unsigned long val;
2733 int i;
2734
2735
2736
2737
2738 mask >>= PMU_FIRST_COUNTER;
2739 for(i = PMU_FIRST_COUNTER; mask; i++, mask >>= 1) {
2740
2741 if ((mask & 0x1UL) == 0UL) continue;
2742
2743 ctx->ctx_pmds[i].val = val = pfm_new_counter_value(ctx->ctx_pmds+ i, is_long_reset);
2744 reset_others |= ctx->ctx_pmds[i].reset_pmds[0];
2745
2746 DPRINT_ovfl((" %s reset ctx_pmds[%d]=%lx\n", is_long_reset ? "long" : "short", i, val));
2747 }
2748
2749
2750
2751
2752 for(i = 0; reset_others; i++, reset_others >>= 1) {
2753
2754 if ((reset_others & 0x1) == 0) continue;
2755
2756 ctx->ctx_pmds[i].val = val = pfm_new_counter_value(ctx->ctx_pmds + i, is_long_reset);
2757
2758 DPRINT_ovfl(("%s reset_others pmd[%d]=%lx\n",
2759 is_long_reset ? "long" : "short", i, val));
2760 }
2761}
2762
2763static void
2764pfm_reset_regs(pfm_context_t *ctx, unsigned long *ovfl_regs, int is_long_reset)
2765{
2766 unsigned long mask = ovfl_regs[0];
2767 unsigned long reset_others = 0UL;
2768 unsigned long val;
2769 int i;
2770
2771 DPRINT_ovfl(("ovfl_regs=0x%lx is_long_reset=%d\n", ovfl_regs[0], is_long_reset));
2772
2773 if (ctx->ctx_state == PFM_CTX_MASKED) {
2774 pfm_reset_regs_masked(ctx, ovfl_regs, is_long_reset);
2775 return;
2776 }
2777
2778
2779
2780
2781 mask >>= PMU_FIRST_COUNTER;
2782 for(i = PMU_FIRST_COUNTER; mask; i++, mask >>= 1) {
2783
2784 if ((mask & 0x1UL) == 0UL) continue;
2785
2786 val = pfm_new_counter_value(ctx->ctx_pmds+ i, is_long_reset);
2787 reset_others |= ctx->ctx_pmds[i].reset_pmds[0];
2788
2789 DPRINT_ovfl((" %s reset ctx_pmds[%d]=%lx\n", is_long_reset ? "long" : "short", i, val));
2790
2791 pfm_write_soft_counter(ctx, i, val);
2792 }
2793
2794
2795
2796
2797 for(i = 0; reset_others; i++, reset_others >>= 1) {
2798
2799 if ((reset_others & 0x1) == 0) continue;
2800
2801 val = pfm_new_counter_value(ctx->ctx_pmds + i, is_long_reset);
2802
2803 if (PMD_IS_COUNTING(i)) {
2804 pfm_write_soft_counter(ctx, i, val);
2805 } else {
2806 ia64_set_pmd(i, val);
2807 }
2808 DPRINT_ovfl(("%s reset_others pmd[%d]=%lx\n",
2809 is_long_reset ? "long" : "short", i, val));
2810 }
2811 ia64_srlz_d();
2812}
2813
2814static int
2815pfm_write_pmcs(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
2816{
2817 struct task_struct *task;
2818 pfarg_reg_t *req = (pfarg_reg_t *)arg;
2819 unsigned long value, pmc_pm;
2820 unsigned long smpl_pmds, reset_pmds, impl_pmds;
2821 unsigned int cnum, reg_flags, flags, pmc_type;
2822 int i, can_access_pmu = 0, is_loaded, is_system, expert_mode;
2823 int is_monitor, is_counting, state;
2824 int ret = -EINVAL;
2825 pfm_reg_check_t wr_func;
2826#define PFM_CHECK_PMC_PM(x, y, z) ((x)->ctx_fl_system ^ PMC_PM(y, z))
2827
2828 state = ctx->ctx_state;
2829 is_loaded = state == PFM_CTX_LOADED ? 1 : 0;
2830 is_system = ctx->ctx_fl_system;
2831 task = ctx->ctx_task;
2832 impl_pmds = pmu_conf->impl_pmds[0];
2833
2834 if (state == PFM_CTX_ZOMBIE) return -EINVAL;
2835
2836 if (is_loaded) {
2837
2838
2839
2840
2841
2842 if (is_system && ctx->ctx_cpu != smp_processor_id()) {
2843 DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
2844 return -EBUSY;
2845 }
2846 can_access_pmu = GET_PMU_OWNER() == task || is_system ? 1 : 0;
2847 }
2848 expert_mode = pfm_sysctl.expert_mode;
2849
2850 for (i = 0; i < count; i++, req++) {
2851
2852 cnum = req->reg_num;
2853 reg_flags = req->reg_flags;
2854 value = req->reg_value;
2855 smpl_pmds = req->reg_smpl_pmds[0];
2856 reset_pmds = req->reg_reset_pmds[0];
2857 flags = 0;
2858
2859
2860 if (cnum >= PMU_MAX_PMCS) {
2861 DPRINT(("pmc%u is invalid\n", cnum));
2862 goto error;
2863 }
2864
2865 pmc_type = pmu_conf->pmc_desc[cnum].type;
2866 pmc_pm = (value >> pmu_conf->pmc_desc[cnum].pm_pos) & 0x1;
2867 is_counting = (pmc_type & PFM_REG_COUNTING) == PFM_REG_COUNTING ? 1 : 0;
2868 is_monitor = (pmc_type & PFM_REG_MONITOR) == PFM_REG_MONITOR ? 1 : 0;
2869
2870
2871
2872
2873
2874
2875 if ((pmc_type & PFM_REG_IMPL) == 0 || (pmc_type & PFM_REG_CONTROL) == PFM_REG_CONTROL) {
2876 DPRINT(("pmc%u is unimplemented or no-access pmc_type=%x\n", cnum, pmc_type));
2877 goto error;
2878 }
2879 wr_func = pmu_conf->pmc_desc[cnum].write_check;
2880
2881
2882
2883
2884
2885 if (is_monitor && value != PMC_DFL_VAL(cnum) && is_system ^ pmc_pm) {
2886 DPRINT(("pmc%u pmc_pm=%lu is_system=%d\n",
2887 cnum,
2888 pmc_pm,
2889 is_system));
2890 goto error;
2891 }
2892
2893 if (is_counting) {
2894
2895
2896
2897
2898 value |= 1 << PMU_PMC_OI;
2899
2900 if (reg_flags & PFM_REGFL_OVFL_NOTIFY) {
2901 flags |= PFM_REGFL_OVFL_NOTIFY;
2902 }
2903
2904 if (reg_flags & PFM_REGFL_RANDOM) flags |= PFM_REGFL_RANDOM;
2905
2906
2907 if ((smpl_pmds & impl_pmds) != smpl_pmds) {
2908 DPRINT(("invalid smpl_pmds 0x%lx for pmc%u\n", smpl_pmds, cnum));
2909 goto error;
2910 }
2911
2912
2913 if ((reset_pmds & impl_pmds) != reset_pmds) {
2914 DPRINT(("invalid reset_pmds 0x%lx for pmc%u\n", reset_pmds, cnum));
2915 goto error;
2916 }
2917 } else {
2918 if (reg_flags & (PFM_REGFL_OVFL_NOTIFY|PFM_REGFL_RANDOM)) {
2919 DPRINT(("cannot set ovfl_notify or random on pmc%u\n", cnum));
2920 goto error;
2921 }
2922
2923 }
2924
2925
2926
2927
2928 if (likely(expert_mode == 0 && wr_func)) {
2929 ret = (*wr_func)(task, ctx, cnum, &value, regs);
2930 if (ret) goto error;
2931 ret = -EINVAL;
2932 }
2933
2934
2935
2936
2937 PFM_REG_RETFLAG_SET(req->reg_flags, 0);
2938
2939
2940
2941
2942
2943
2944
2945
2946 if (is_counting) {
2947
2948
2949
2950 ctx->ctx_pmds[cnum].flags = flags;
2951
2952 ctx->ctx_pmds[cnum].reset_pmds[0] = reset_pmds;
2953 ctx->ctx_pmds[cnum].smpl_pmds[0] = smpl_pmds;
2954 ctx->ctx_pmds[cnum].eventid = req->reg_smpl_eventid;
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967 CTX_USED_PMD(ctx, reset_pmds);
2968 CTX_USED_PMD(ctx, smpl_pmds);
2969
2970
2971
2972
2973 if (state == PFM_CTX_MASKED) ctx->ctx_ovfl_regs[0] &= ~1UL << cnum;
2974 }
2975
2976
2977
2978
2979
2980 CTX_USED_PMD(ctx, pmu_conf->pmc_desc[cnum].dep_pmd[0]);
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994 if (is_monitor) CTX_USED_MONITOR(ctx, 1UL << cnum);
2995
2996
2997
2998
2999 ctx->ctx_pmcs[cnum] = value;
3000
3001 if (is_loaded) {
3002
3003
3004
3005 if (is_system == 0) ctx->th_pmcs[cnum] = value;
3006
3007
3008
3009
3010 if (can_access_pmu) {
3011 ia64_set_pmc(cnum, value);
3012 }
3013#ifdef CONFIG_SMP
3014 else {
3015
3016
3017
3018
3019
3020
3021
3022 ctx->ctx_reload_pmcs[0] |= 1UL << cnum;
3023 }
3024#endif
3025 }
3026
3027 DPRINT(("pmc[%u]=0x%lx ld=%d apmu=%d flags=0x%x all_pmcs=0x%lx used_pmds=0x%lx eventid=%ld smpl_pmds=0x%lx reset_pmds=0x%lx reloads_pmcs=0x%lx used_monitors=0x%lx ovfl_regs=0x%lx\n",
3028 cnum,
3029 value,
3030 is_loaded,
3031 can_access_pmu,
3032 flags,
3033 ctx->ctx_all_pmcs[0],
3034 ctx->ctx_used_pmds[0],
3035 ctx->ctx_pmds[cnum].eventid,
3036 smpl_pmds,
3037 reset_pmds,
3038 ctx->ctx_reload_pmcs[0],
3039 ctx->ctx_used_monitors[0],
3040 ctx->ctx_ovfl_regs[0]));
3041 }
3042
3043
3044
3045
3046 if (can_access_pmu) ia64_srlz_d();
3047
3048 return 0;
3049error:
3050 PFM_REG_RETFLAG_SET(req->reg_flags, PFM_REG_RETFL_EINVAL);
3051 return ret;
3052}
3053
3054static int
3055pfm_write_pmds(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
3056{
3057 struct task_struct *task;
3058 pfarg_reg_t *req = (pfarg_reg_t *)arg;
3059 unsigned long value, hw_value, ovfl_mask;
3060 unsigned int cnum;
3061 int i, can_access_pmu = 0, state;
3062 int is_counting, is_loaded, is_system, expert_mode;
3063 int ret = -EINVAL;
3064 pfm_reg_check_t wr_func;
3065
3066
3067 state = ctx->ctx_state;
3068 is_loaded = state == PFM_CTX_LOADED ? 1 : 0;
3069 is_system = ctx->ctx_fl_system;
3070 ovfl_mask = pmu_conf->ovfl_val;
3071 task = ctx->ctx_task;
3072
3073 if (unlikely(state == PFM_CTX_ZOMBIE)) return -EINVAL;
3074
3075
3076
3077
3078
3079 if (likely(is_loaded)) {
3080
3081
3082
3083
3084
3085 if (unlikely(is_system && ctx->ctx_cpu != smp_processor_id())) {
3086 DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
3087 return -EBUSY;
3088 }
3089 can_access_pmu = GET_PMU_OWNER() == task || is_system ? 1 : 0;
3090 }
3091 expert_mode = pfm_sysctl.expert_mode;
3092
3093 for (i = 0; i < count; i++, req++) {
3094
3095 cnum = req->reg_num;
3096 value = req->reg_value;
3097
3098 if (!PMD_IS_IMPL(cnum)) {
3099 DPRINT(("pmd[%u] is unimplemented or invalid\n", cnum));
3100 goto abort_mission;
3101 }
3102 is_counting = PMD_IS_COUNTING(cnum);
3103 wr_func = pmu_conf->pmd_desc[cnum].write_check;
3104
3105
3106
3107
3108 if (unlikely(expert_mode == 0 && wr_func)) {
3109 unsigned long v = value;
3110
3111 ret = (*wr_func)(task, ctx, cnum, &v, regs);
3112 if (ret) goto abort_mission;
3113
3114 value = v;
3115 ret = -EINVAL;
3116 }
3117
3118
3119
3120
3121 PFM_REG_RETFLAG_SET(req->reg_flags, 0);
3122
3123
3124
3125
3126 hw_value = value;
3127
3128
3129
3130
3131 if (is_counting) {
3132
3133
3134
3135 ctx->ctx_pmds[cnum].lval = value;
3136
3137
3138
3139
3140 if (is_loaded) {
3141 hw_value = value & ovfl_mask;
3142 value = value & ~ovfl_mask;
3143 }
3144 }
3145
3146
3147
3148 ctx->ctx_pmds[cnum].long_reset = req->reg_long_reset;
3149 ctx->ctx_pmds[cnum].short_reset = req->reg_short_reset;
3150
3151
3152
3153
3154 ctx->ctx_pmds[cnum].seed = req->reg_random_seed;
3155 ctx->ctx_pmds[cnum].mask = req->reg_random_mask;
3156
3157
3158
3159
3160 ctx->ctx_pmds[cnum].val = value;
3161
3162
3163
3164
3165
3166
3167
3168 CTX_USED_PMD(ctx, PMD_PMD_DEP(cnum));
3169
3170
3171
3172
3173 CTX_USED_PMD(ctx, RDEP(cnum));
3174
3175
3176
3177
3178
3179 if (is_counting && state == PFM_CTX_MASKED) {
3180 ctx->ctx_ovfl_regs[0] &= ~1UL << cnum;
3181 }
3182
3183 if (is_loaded) {
3184
3185
3186
3187 if (is_system == 0) ctx->th_pmds[cnum] = hw_value;
3188
3189
3190
3191
3192 if (can_access_pmu) {
3193 ia64_set_pmd(cnum, hw_value);
3194 } else {
3195#ifdef CONFIG_SMP
3196
3197
3198
3199
3200
3201 ctx->ctx_reload_pmds[0] |= 1UL << cnum;
3202#endif
3203 }
3204 }
3205
3206 DPRINT(("pmd[%u]=0x%lx ld=%d apmu=%d, hw_value=0x%lx ctx_pmd=0x%lx short_reset=0x%lx "
3207 "long_reset=0x%lx notify=%c seed=0x%lx mask=0x%lx used_pmds=0x%lx reset_pmds=0x%lx reload_pmds=0x%lx all_pmds=0x%lx ovfl_regs=0x%lx\n",
3208 cnum,
3209 value,
3210 is_loaded,
3211 can_access_pmu,
3212 hw_value,
3213 ctx->ctx_pmds[cnum].val,
3214 ctx->ctx_pmds[cnum].short_reset,
3215 ctx->ctx_pmds[cnum].long_reset,
3216 PMC_OVFL_NOTIFY(ctx, cnum) ? 'Y':'N',
3217 ctx->ctx_pmds[cnum].seed,
3218 ctx->ctx_pmds[cnum].mask,
3219 ctx->ctx_used_pmds[0],
3220 ctx->ctx_pmds[cnum].reset_pmds[0],
3221 ctx->ctx_reload_pmds[0],
3222 ctx->ctx_all_pmds[0],
3223 ctx->ctx_ovfl_regs[0]));
3224 }
3225
3226
3227
3228
3229 if (can_access_pmu) ia64_srlz_d();
3230
3231 return 0;
3232
3233abort_mission:
3234
3235
3236
3237 PFM_REG_RETFLAG_SET(req->reg_flags, PFM_REG_RETFL_EINVAL);
3238 return ret;
3239}
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250static int
3251pfm_read_pmds(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
3252{
3253 struct task_struct *task;
3254 unsigned long val = 0UL, lval, ovfl_mask, sval;
3255 pfarg_reg_t *req = (pfarg_reg_t *)arg;
3256 unsigned int cnum, reg_flags = 0;
3257 int i, can_access_pmu = 0, state;
3258 int is_loaded, is_system, is_counting, expert_mode;
3259 int ret = -EINVAL;
3260 pfm_reg_check_t rd_func;
3261
3262
3263
3264
3265
3266
3267 state = ctx->ctx_state;
3268 is_loaded = state == PFM_CTX_LOADED ? 1 : 0;
3269 is_system = ctx->ctx_fl_system;
3270 ovfl_mask = pmu_conf->ovfl_val;
3271 task = ctx->ctx_task;
3272
3273 if (state == PFM_CTX_ZOMBIE) return -EINVAL;
3274
3275 if (likely(is_loaded)) {
3276
3277
3278
3279
3280
3281 if (unlikely(is_system && ctx->ctx_cpu != smp_processor_id())) {
3282 DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
3283 return -EBUSY;
3284 }
3285
3286
3287
3288 can_access_pmu = GET_PMU_OWNER() == task || is_system ? 1 : 0;
3289
3290 if (can_access_pmu) ia64_srlz_d();
3291 }
3292 expert_mode = pfm_sysctl.expert_mode;
3293
3294 DPRINT(("ld=%d apmu=%d ctx_state=%d\n",
3295 is_loaded,
3296 can_access_pmu,
3297 state));
3298
3299
3300
3301
3302
3303
3304 for (i = 0; i < count; i++, req++) {
3305
3306 cnum = req->reg_num;
3307 reg_flags = req->reg_flags;
3308
3309 if (unlikely(!PMD_IS_IMPL(cnum))) goto error;
3310
3311
3312
3313
3314
3315
3316
3317
3318 if (unlikely(!CTX_IS_USED_PMD(ctx, cnum))) goto error;
3319
3320 sval = ctx->ctx_pmds[cnum].val;
3321 lval = ctx->ctx_pmds[cnum].lval;
3322 is_counting = PMD_IS_COUNTING(cnum);
3323
3324
3325
3326
3327
3328
3329 if (can_access_pmu){
3330 val = ia64_get_pmd(cnum);
3331 } else {
3332
3333
3334
3335
3336
3337 val = is_loaded ? ctx->th_pmds[cnum] : 0UL;
3338 }
3339 rd_func = pmu_conf->pmd_desc[cnum].read_check;
3340
3341 if (is_counting) {
3342
3343
3344
3345 val &= ovfl_mask;
3346 val += sval;
3347 }
3348
3349
3350
3351
3352 if (unlikely(expert_mode == 0 && rd_func)) {
3353 unsigned long v = val;
3354 ret = (*rd_func)(ctx->ctx_task, ctx, cnum, &v, regs);
3355 if (ret) goto error;
3356 val = v;
3357 ret = -EINVAL;
3358 }
3359
3360 PFM_REG_RETFLAG_SET(reg_flags, 0);
3361
3362 DPRINT(("pmd[%u]=0x%lx\n", cnum, val));
3363
3364
3365
3366
3367
3368
3369 req->reg_value = val;
3370 req->reg_flags = reg_flags;
3371 req->reg_last_reset_val = lval;
3372 }
3373
3374 return 0;
3375
3376error:
3377 PFM_REG_RETFLAG_SET(req->reg_flags, PFM_REG_RETFL_EINVAL);
3378 return ret;
3379}
3380
3381int
3382pfm_mod_write_pmcs(struct task_struct *task, void *req, unsigned int nreq, struct pt_regs *regs)
3383{
3384 pfm_context_t *ctx;
3385
3386 if (req == NULL) return -EINVAL;
3387
3388 ctx = GET_PMU_CTX();
3389
3390 if (ctx == NULL) return -EINVAL;
3391
3392
3393
3394
3395
3396 if (task != current && ctx->ctx_fl_system == 0) return -EBUSY;
3397
3398 return pfm_write_pmcs(ctx, req, nreq, regs);
3399}
3400EXPORT_SYMBOL(pfm_mod_write_pmcs);
3401
3402int
3403pfm_mod_read_pmds(struct task_struct *task, void *req, unsigned int nreq, struct pt_regs *regs)
3404{
3405 pfm_context_t *ctx;
3406
3407 if (req == NULL) return -EINVAL;
3408
3409 ctx = GET_PMU_CTX();
3410
3411 if (ctx == NULL) return -EINVAL;
3412
3413
3414
3415
3416
3417 if (task != current && ctx->ctx_fl_system == 0) return -EBUSY;
3418
3419 return pfm_read_pmds(ctx, req, nreq, regs);
3420}
3421EXPORT_SYMBOL(pfm_mod_read_pmds);
3422
3423
3424
3425
3426
3427int
3428pfm_use_debug_registers(struct task_struct *task)
3429{
3430 pfm_context_t *ctx = task->thread.pfm_context;
3431 unsigned long flags;
3432 int ret = 0;
3433
3434 if (pmu_conf->use_rr_dbregs == 0) return 0;
3435
3436 DPRINT(("called for [%d]\n", task_pid_nr(task)));
3437
3438
3439
3440
3441 if (task->thread.flags & IA64_THREAD_DBG_VALID) return 0;
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451 if (ctx && ctx->ctx_fl_using_dbreg == 1) return -1;
3452
3453 LOCK_PFS(flags);
3454
3455
3456
3457
3458
3459 if (pfm_sessions.pfs_sys_use_dbregs> 0)
3460 ret = -1;
3461 else
3462 pfm_sessions.pfs_ptrace_use_dbregs++;
3463
3464 DPRINT(("ptrace_use_dbregs=%u sys_use_dbregs=%u by [%d] ret = %d\n",
3465 pfm_sessions.pfs_ptrace_use_dbregs,
3466 pfm_sessions.pfs_sys_use_dbregs,
3467 task_pid_nr(task), ret));
3468
3469 UNLOCK_PFS(flags);
3470
3471 return ret;
3472}
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482int
3483pfm_release_debug_registers(struct task_struct *task)
3484{
3485 unsigned long flags;
3486 int ret;
3487
3488 if (pmu_conf->use_rr_dbregs == 0) return 0;
3489
3490 LOCK_PFS(flags);
3491 if (pfm_sessions.pfs_ptrace_use_dbregs == 0) {
3492 printk(KERN_ERR "perfmon: invalid release for [%d] ptrace_use_dbregs=0\n", task_pid_nr(task));
3493 ret = -1;
3494 } else {
3495 pfm_sessions.pfs_ptrace_use_dbregs--;
3496 ret = 0;
3497 }
3498 UNLOCK_PFS(flags);
3499
3500 return ret;
3501}
3502
3503static int
3504pfm_restart(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
3505{
3506 struct task_struct *task;
3507 pfm_buffer_fmt_t *fmt;
3508 pfm_ovfl_ctrl_t rst_ctrl;
3509 int state, is_system;
3510 int ret = 0;
3511
3512 state = ctx->ctx_state;
3513 fmt = ctx->ctx_buf_fmt;
3514 is_system = ctx->ctx_fl_system;
3515 task = PFM_CTX_TASK(ctx);
3516
3517 switch(state) {
3518 case PFM_CTX_MASKED:
3519 break;
3520 case PFM_CTX_LOADED:
3521 if (CTX_HAS_SMPL(ctx) && fmt->fmt_restart_active) break;
3522
3523 case PFM_CTX_UNLOADED:
3524 case PFM_CTX_ZOMBIE:
3525 DPRINT(("invalid state=%d\n", state));
3526 return -EBUSY;
3527 default:
3528 DPRINT(("state=%d, cannot operate (no active_restart handler)\n", state));
3529 return -EINVAL;
3530 }
3531
3532
3533
3534
3535
3536
3537 if (is_system && ctx->ctx_cpu != smp_processor_id()) {
3538 DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
3539 return -EBUSY;
3540 }
3541
3542
3543 if (unlikely(task == NULL)) {
3544 printk(KERN_ERR "perfmon: [%d] pfm_restart no task\n", task_pid_nr(current));
3545 return -EINVAL;
3546 }
3547
3548 if (task == current || is_system) {
3549
3550 fmt = ctx->ctx_buf_fmt;
3551
3552 DPRINT(("restarting self %d ovfl=0x%lx\n",
3553 task_pid_nr(task),
3554 ctx->ctx_ovfl_regs[0]));
3555
3556 if (CTX_HAS_SMPL(ctx)) {
3557
3558 prefetch(ctx->ctx_smpl_hdr);
3559
3560 rst_ctrl.bits.mask_monitoring = 0;
3561 rst_ctrl.bits.reset_ovfl_pmds = 0;
3562
3563 if (state == PFM_CTX_LOADED)
3564 ret = pfm_buf_fmt_restart_active(fmt, task, &rst_ctrl, ctx->ctx_smpl_hdr, regs);
3565 else
3566 ret = pfm_buf_fmt_restart(fmt, task, &rst_ctrl, ctx->ctx_smpl_hdr, regs);
3567 } else {
3568 rst_ctrl.bits.mask_monitoring = 0;
3569 rst_ctrl.bits.reset_ovfl_pmds = 1;
3570 }
3571
3572 if (ret == 0) {
3573 if (rst_ctrl.bits.reset_ovfl_pmds)
3574 pfm_reset_regs(ctx, ctx->ctx_ovfl_regs, PFM_PMD_LONG_RESET);
3575
3576 if (rst_ctrl.bits.mask_monitoring == 0) {
3577 DPRINT(("resuming monitoring for [%d]\n", task_pid_nr(task)));
3578
3579 if (state == PFM_CTX_MASKED) pfm_restore_monitoring(task);
3580 } else {
3581 DPRINT(("keeping monitoring stopped for [%d]\n", task_pid_nr(task)));
3582
3583
3584 }
3585 }
3586
3587
3588
3589 ctx->ctx_ovfl_regs[0] = 0UL;
3590
3591
3592
3593
3594 ctx->ctx_state = PFM_CTX_LOADED;
3595
3596
3597
3598
3599 ctx->ctx_fl_can_restart = 0;
3600
3601 return 0;
3602 }
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612 if (state == PFM_CTX_MASKED) {
3613 if (ctx->ctx_fl_can_restart == 0) return -EINVAL;
3614
3615
3616
3617
3618 ctx->ctx_fl_can_restart = 0;
3619 }
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637 if (CTX_OVFL_NOBLOCK(ctx) == 0 && state == PFM_CTX_MASKED) {
3638 DPRINT(("unblocking [%d]\n", task_pid_nr(task)));
3639 complete(&ctx->ctx_restart_done);
3640 } else {
3641 DPRINT(("[%d] armed exit trap\n", task_pid_nr(task)));
3642
3643 ctx->ctx_fl_trap_reason = PFM_TRAP_REASON_RESET;
3644
3645 PFM_SET_WORK_PENDING(task, 1);
3646
3647 set_notify_resume(task);
3648
3649
3650
3651
3652 }
3653 return 0;
3654}
3655
3656static int
3657pfm_debug(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
3658{
3659 unsigned int m = *(unsigned int *)arg;
3660
3661 pfm_sysctl.debug = m == 0 ? 0 : 1;
3662
3663 printk(KERN_INFO "perfmon debugging %s (timing reset)\n", pfm_sysctl.debug ? "on" : "off");
3664
3665 if (m == 0) {
3666 memset(pfm_stats, 0, sizeof(pfm_stats));
3667 for(m=0; m < NR_CPUS; m++) pfm_stats[m].pfm_ovfl_intr_cycles_min = ~0UL;
3668 }
3669 return 0;
3670}
3671
3672
3673
3674
3675static int
3676pfm_write_ibr_dbr(int mode, pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
3677{
3678 struct thread_struct *thread = NULL;
3679 struct task_struct *task;
3680 pfarg_dbreg_t *req = (pfarg_dbreg_t *)arg;
3681 unsigned long flags;
3682 dbreg_t dbreg;
3683 unsigned int rnum;
3684 int first_time;
3685 int ret = 0, state;
3686 int i, can_access_pmu = 0;
3687 int is_system, is_loaded;
3688
3689 if (pmu_conf->use_rr_dbregs == 0) return -EINVAL;
3690
3691 state = ctx->ctx_state;
3692 is_loaded = state == PFM_CTX_LOADED ? 1 : 0;
3693 is_system = ctx->ctx_fl_system;
3694 task = ctx->ctx_task;
3695
3696 if (state == PFM_CTX_ZOMBIE) return -EINVAL;
3697
3698
3699
3700
3701
3702 if (is_loaded) {
3703 thread = &task->thread;
3704
3705
3706
3707
3708
3709 if (unlikely(is_system && ctx->ctx_cpu != smp_processor_id())) {
3710 DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
3711 return -EBUSY;
3712 }
3713 can_access_pmu = GET_PMU_OWNER() == task || is_system ? 1 : 0;
3714 }
3715
3716
3717
3718
3719
3720
3721
3722
3723 first_time = ctx->ctx_fl_using_dbreg == 0;
3724
3725
3726
3727
3728 if (is_loaded && (thread->flags & IA64_THREAD_DBG_VALID) != 0) {
3729 DPRINT(("debug registers already in use for [%d]\n", task_pid_nr(task)));
3730 return -EBUSY;
3731 }
3732
3733
3734
3735
3736
3737
3738
3739
3740 if (is_loaded) {
3741 LOCK_PFS(flags);
3742
3743 if (first_time && is_system) {
3744 if (pfm_sessions.pfs_ptrace_use_dbregs)
3745 ret = -EBUSY;
3746 else
3747 pfm_sessions.pfs_sys_use_dbregs++;
3748 }
3749 UNLOCK_PFS(flags);
3750 }
3751
3752 if (ret != 0) return ret;
3753
3754
3755
3756
3757
3758 ctx->ctx_fl_using_dbreg = 1;
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769 if (first_time && can_access_pmu) {
3770 DPRINT(("[%d] clearing ibrs, dbrs\n", task_pid_nr(task)));
3771 for (i=0; i < pmu_conf->num_ibrs; i++) {
3772 ia64_set_ibr(i, 0UL);
3773 ia64_dv_serialize_instruction();
3774 }
3775 ia64_srlz_i();
3776 for (i=0; i < pmu_conf->num_dbrs; i++) {
3777 ia64_set_dbr(i, 0UL);
3778 ia64_dv_serialize_data();
3779 }
3780 ia64_srlz_d();
3781 }
3782
3783
3784
3785
3786 for (i = 0; i < count; i++, req++) {
3787
3788 rnum = req->dbreg_num;
3789 dbreg.val = req->dbreg_value;
3790
3791 ret = -EINVAL;
3792
3793 if ((mode == PFM_CODE_RR && rnum >= PFM_NUM_IBRS) || ((mode == PFM_DATA_RR) && rnum >= PFM_NUM_DBRS)) {
3794 DPRINT(("invalid register %u val=0x%lx mode=%d i=%d count=%d\n",
3795 rnum, dbreg.val, mode, i, count));
3796
3797 goto abort_mission;
3798 }
3799
3800
3801
3802
3803 if (rnum & 0x1) {
3804 if (mode == PFM_CODE_RR)
3805 dbreg.ibr.ibr_x = 0;
3806 else
3807 dbreg.dbr.dbr_r = dbreg.dbr.dbr_w = 0;
3808 }
3809
3810 PFM_REG_RETFLAG_SET(req->dbreg_flags, 0);
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822 if (mode == PFM_CODE_RR) {
3823 CTX_USED_IBR(ctx, rnum);
3824
3825 if (can_access_pmu) {
3826 ia64_set_ibr(rnum, dbreg.val);
3827 ia64_dv_serialize_instruction();
3828 }
3829
3830 ctx->ctx_ibrs[rnum] = dbreg.val;
3831
3832 DPRINT(("write ibr%u=0x%lx used_ibrs=0x%x ld=%d apmu=%d\n",
3833 rnum, dbreg.val, ctx->ctx_used_ibrs[0], is_loaded, can_access_pmu));
3834 } else {
3835 CTX_USED_DBR(ctx, rnum);
3836
3837 if (can_access_pmu) {
3838 ia64_set_dbr(rnum, dbreg.val);
3839 ia64_dv_serialize_data();
3840 }
3841 ctx->ctx_dbrs[rnum] = dbreg.val;
3842
3843 DPRINT(("write dbr%u=0x%lx used_dbrs=0x%x ld=%d apmu=%d\n",
3844 rnum, dbreg.val, ctx->ctx_used_dbrs[0], is_loaded, can_access_pmu));
3845 }
3846 }
3847
3848 return 0;
3849
3850abort_mission:
3851
3852
3853
3854 if (first_time) {
3855 LOCK_PFS(flags);
3856 if (ctx->ctx_fl_system) {
3857 pfm_sessions.pfs_sys_use_dbregs--;
3858 }
3859 UNLOCK_PFS(flags);
3860 ctx->ctx_fl_using_dbreg = 0;
3861 }
3862
3863
3864
3865 PFM_REG_RETFLAG_SET(req->dbreg_flags, PFM_REG_RETFL_EINVAL);
3866
3867 return ret;
3868}
3869
3870static int
3871pfm_write_ibrs(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
3872{
3873 return pfm_write_ibr_dbr(PFM_CODE_RR, ctx, arg, count, regs);
3874}
3875
3876static int
3877pfm_write_dbrs(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
3878{
3879 return pfm_write_ibr_dbr(PFM_DATA_RR, ctx, arg, count, regs);
3880}
3881
3882int
3883pfm_mod_write_ibrs(struct task_struct *task, void *req, unsigned int nreq, struct pt_regs *regs)
3884{
3885 pfm_context_t *ctx;
3886
3887 if (req == NULL) return -EINVAL;
3888
3889 ctx = GET_PMU_CTX();
3890
3891 if (ctx == NULL) return -EINVAL;
3892
3893
3894
3895
3896
3897 if (task != current && ctx->ctx_fl_system == 0) return -EBUSY;
3898
3899 return pfm_write_ibrs(ctx, req, nreq, regs);
3900}
3901EXPORT_SYMBOL(pfm_mod_write_ibrs);
3902
3903int
3904pfm_mod_write_dbrs(struct task_struct *task, void *req, unsigned int nreq, struct pt_regs *regs)
3905{
3906 pfm_context_t *ctx;
3907
3908 if (req == NULL) return -EINVAL;
3909
3910 ctx = GET_PMU_CTX();
3911
3912 if (ctx == NULL) return -EINVAL;
3913
3914
3915
3916
3917
3918 if (task != current && ctx->ctx_fl_system == 0) return -EBUSY;
3919
3920 return pfm_write_dbrs(ctx, req, nreq, regs);
3921}
3922EXPORT_SYMBOL(pfm_mod_write_dbrs);
3923
3924
3925static int
3926pfm_get_features(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
3927{
3928 pfarg_features_t *req = (pfarg_features_t *)arg;
3929
3930 req->ft_version = PFM_VERSION;
3931 return 0;
3932}
3933
3934static int
3935pfm_stop(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
3936{
3937 struct pt_regs *tregs;
3938 struct task_struct *task = PFM_CTX_TASK(ctx);
3939 int state, is_system;
3940
3941 state = ctx->ctx_state;
3942 is_system = ctx->ctx_fl_system;
3943
3944
3945
3946
3947 if (state == PFM_CTX_UNLOADED) return -EINVAL;
3948
3949
3950
3951
3952
3953
3954 if (is_system && ctx->ctx_cpu != smp_processor_id()) {
3955 DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
3956 return -EBUSY;
3957 }
3958 DPRINT(("task [%d] ctx_state=%d is_system=%d\n",
3959 task_pid_nr(PFM_CTX_TASK(ctx)),
3960 state,
3961 is_system));
3962
3963
3964
3965
3966
3967 if (is_system) {
3968
3969
3970
3971
3972
3973 ia64_setreg(_IA64_REG_CR_DCR, ia64_getreg(_IA64_REG_CR_DCR) & ~IA64_DCR_PP);
3974 ia64_srlz_i();
3975
3976
3977
3978
3979 PFM_CPUINFO_CLEAR(PFM_CPUINFO_DCR_PP);
3980
3981
3982
3983
3984 pfm_clear_psr_pp();
3985
3986
3987
3988
3989 ia64_psr(regs)->pp = 0;
3990
3991 return 0;
3992 }
3993
3994
3995
3996
3997 if (task == current) {
3998
3999 pfm_clear_psr_up();
4000
4001
4002
4003
4004 ia64_psr(regs)->up = 0;
4005 } else {
4006 tregs = task_pt_regs(task);
4007
4008
4009
4010
4011 ia64_psr(tregs)->up = 0;
4012
4013
4014
4015
4016 ctx->ctx_saved_psr_up = 0;
4017 DPRINT(("task=[%d]\n", task_pid_nr(task)));
4018 }
4019 return 0;
4020}
4021
4022
4023static int
4024pfm_start(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
4025{
4026 struct pt_regs *tregs;
4027 int state, is_system;
4028
4029 state = ctx->ctx_state;
4030 is_system = ctx->ctx_fl_system;
4031
4032 if (state != PFM_CTX_LOADED) return -EINVAL;
4033
4034
4035
4036
4037
4038
4039 if (is_system && ctx->ctx_cpu != smp_processor_id()) {
4040 DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
4041 return -EBUSY;
4042 }
4043
4044
4045
4046
4047
4048
4049 if (is_system) {
4050
4051
4052
4053
4054 ia64_psr(regs)->pp = 1;
4055
4056
4057
4058
4059 PFM_CPUINFO_SET(PFM_CPUINFO_DCR_PP);
4060
4061
4062
4063
4064 pfm_set_psr_pp();
4065
4066
4067 ia64_setreg(_IA64_REG_CR_DCR, ia64_getreg(_IA64_REG_CR_DCR) | IA64_DCR_PP);
4068 ia64_srlz_i();
4069
4070 return 0;
4071 }
4072
4073
4074
4075
4076
4077 if (ctx->ctx_task == current) {
4078
4079
4080 pfm_set_psr_up();
4081
4082
4083
4084
4085 ia64_psr(regs)->up = 1;
4086
4087 } else {
4088 tregs = task_pt_regs(ctx->ctx_task);
4089
4090
4091
4092
4093
4094 ctx->ctx_saved_psr_up = IA64_PSR_UP;
4095
4096
4097
4098
4099 ia64_psr(tregs)->up = 1;
4100 }
4101 return 0;
4102}
4103
4104static int
4105pfm_get_pmc_reset(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
4106{
4107 pfarg_reg_t *req = (pfarg_reg_t *)arg;
4108 unsigned int cnum;
4109 int i;
4110 int ret = -EINVAL;
4111
4112 for (i = 0; i < count; i++, req++) {
4113
4114 cnum = req->reg_num;
4115
4116 if (!PMC_IS_IMPL(cnum)) goto abort_mission;
4117
4118 req->reg_value = PMC_DFL_VAL(cnum);
4119
4120 PFM_REG_RETFLAG_SET(req->reg_flags, 0);
4121
4122 DPRINT(("pmc_reset_val pmc[%u]=0x%lx\n", cnum, req->reg_value));
4123 }
4124 return 0;
4125
4126abort_mission:
4127 PFM_REG_RETFLAG_SET(req->reg_flags, PFM_REG_RETFL_EINVAL);
4128 return ret;
4129}
4130
4131static int
4132pfm_check_task_exist(pfm_context_t *ctx)
4133{
4134 struct task_struct *g, *t;
4135 int ret = -ESRCH;
4136
4137 read_lock(&tasklist_lock);
4138
4139 do_each_thread (g, t) {
4140 if (t->thread.pfm_context == ctx) {
4141 ret = 0;
4142 goto out;
4143 }
4144 } while_each_thread (g, t);
4145out:
4146 read_unlock(&tasklist_lock);
4147
4148 DPRINT(("pfm_check_task_exist: ret=%d ctx=%p\n", ret, ctx));
4149
4150 return ret;
4151}
4152
4153static int
4154pfm_context_load(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
4155{
4156 struct task_struct *task;
4157 struct thread_struct *thread;
4158 struct pfm_context_t *old;
4159 unsigned long flags;
4160#ifndef CONFIG_SMP
4161 struct task_struct *owner_task = NULL;
4162#endif
4163 pfarg_load_t *req = (pfarg_load_t *)arg;
4164 unsigned long *pmcs_source, *pmds_source;
4165 int the_cpu;
4166 int ret = 0;
4167 int state, is_system, set_dbregs = 0;
4168
4169 state = ctx->ctx_state;
4170 is_system = ctx->ctx_fl_system;
4171
4172
4173
4174 if (state != PFM_CTX_UNLOADED) {
4175 DPRINT(("cannot load to [%d], invalid ctx_state=%d\n",
4176 req->load_pid,
4177 ctx->ctx_state));
4178 return -EBUSY;
4179 }
4180
4181 DPRINT(("load_pid [%d] using_dbreg=%d\n", req->load_pid, ctx->ctx_fl_using_dbreg));
4182
4183 if (CTX_OVFL_NOBLOCK(ctx) == 0 && req->load_pid == current->pid) {
4184 DPRINT(("cannot use blocking mode on self\n"));
4185 return -EINVAL;
4186 }
4187
4188 ret = pfm_get_task(ctx, req->load_pid, &task);
4189 if (ret) {
4190 DPRINT(("load_pid [%d] get_task=%d\n", req->load_pid, ret));
4191 return ret;
4192 }
4193
4194 ret = -EINVAL;
4195
4196
4197
4198
4199 if (is_system && task != current) {
4200 DPRINT(("system wide is self monitoring only load_pid=%d\n",
4201 req->load_pid));
4202 goto error;
4203 }
4204
4205 thread = &task->thread;
4206
4207 ret = 0;
4208
4209
4210
4211
4212 if (ctx->ctx_fl_using_dbreg) {
4213 if (thread->flags & IA64_THREAD_DBG_VALID) {
4214 ret = -EBUSY;
4215 DPRINT(("load_pid [%d] task is debugged, cannot load range restrictions\n", req->load_pid));
4216 goto error;
4217 }
4218 LOCK_PFS(flags);
4219
4220 if (is_system) {
4221 if (pfm_sessions.pfs_ptrace_use_dbregs) {
4222 DPRINT(("cannot load [%d] dbregs in use\n",
4223 task_pid_nr(task)));
4224 ret = -EBUSY;
4225 } else {
4226 pfm_sessions.pfs_sys_use_dbregs++;
4227 DPRINT(("load [%d] increased sys_use_dbreg=%u\n", task_pid_nr(task), pfm_sessions.pfs_sys_use_dbregs));
4228 set_dbregs = 1;
4229 }
4230 }
4231
4232 UNLOCK_PFS(flags);
4233
4234 if (ret) goto error;
4235 }
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252 the_cpu = ctx->ctx_cpu = smp_processor_id();
4253
4254 ret = -EBUSY;
4255
4256
4257
4258 ret = pfm_reserve_session(current, is_system, the_cpu);
4259 if (ret) goto error;
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270 DPRINT(("before cmpxchg() old_ctx=%p new_ctx=%p\n",
4271 thread->pfm_context, ctx));
4272
4273 ret = -EBUSY;
4274 old = ia64_cmpxchg(acq, &thread->pfm_context, NULL, ctx, sizeof(pfm_context_t *));
4275 if (old != NULL) {
4276 DPRINT(("load_pid [%d] already has a context\n", req->load_pid));
4277 goto error_unres;
4278 }
4279
4280 pfm_reset_msgq(ctx);
4281
4282 ctx->ctx_state = PFM_CTX_LOADED;
4283
4284
4285
4286
4287 ctx->ctx_task = task;
4288
4289 if (is_system) {
4290
4291
4292
4293 PFM_CPUINFO_SET(PFM_CPUINFO_SYST_WIDE);
4294 PFM_CPUINFO_CLEAR(PFM_CPUINFO_DCR_PP);
4295
4296 if (ctx->ctx_fl_excl_idle) PFM_CPUINFO_SET(PFM_CPUINFO_EXCL_IDLE);
4297 } else {
4298 thread->flags |= IA64_THREAD_PM_VALID;
4299 }
4300
4301
4302
4303
4304 pfm_copy_pmds(task, ctx);
4305 pfm_copy_pmcs(task, ctx);
4306
4307 pmcs_source = ctx->th_pmcs;
4308 pmds_source = ctx->th_pmds;
4309
4310
4311
4312
4313 if (task == current) {
4314
4315 if (is_system == 0) {
4316
4317
4318 ia64_psr(regs)->sp = 0;
4319 DPRINT(("clearing psr.sp for [%d]\n", task_pid_nr(task)));
4320
4321 SET_LAST_CPU(ctx, smp_processor_id());
4322 INC_ACTIVATION();
4323 SET_ACTIVATION(ctx);
4324#ifndef CONFIG_SMP
4325
4326
4327
4328 owner_task = GET_PMU_OWNER();
4329 if (owner_task) pfm_lazy_save_regs(owner_task);
4330#endif
4331 }
4332
4333
4334
4335
4336 pfm_restore_pmds(pmds_source, ctx->ctx_all_pmds[0]);
4337 pfm_restore_pmcs(pmcs_source, ctx->ctx_all_pmcs[0]);
4338
4339 ctx->ctx_reload_pmcs[0] = 0UL;
4340 ctx->ctx_reload_pmds[0] = 0UL;
4341
4342
4343
4344
4345 if (ctx->ctx_fl_using_dbreg) {
4346 pfm_restore_ibrs(ctx->ctx_ibrs, pmu_conf->num_ibrs);
4347 pfm_restore_dbrs(ctx->ctx_dbrs, pmu_conf->num_dbrs);
4348 }
4349
4350
4351
4352 SET_PMU_OWNER(task, ctx);
4353
4354 DPRINT(("context loaded on PMU for [%d]\n", task_pid_nr(task)));
4355 } else {
4356
4357
4358
4359 regs = task_pt_regs(task);
4360
4361
4362 ctx->ctx_last_activation = PFM_INVALID_ACTIVATION;
4363 SET_LAST_CPU(ctx, -1);
4364
4365
4366 ctx->ctx_saved_psr_up = 0UL;
4367 ia64_psr(regs)->up = ia64_psr(regs)->pp = 0;
4368 }
4369
4370 ret = 0;
4371
4372error_unres:
4373 if (ret) pfm_unreserve_session(ctx, ctx->ctx_fl_system, the_cpu);
4374error:
4375
4376
4377
4378 if (ret && set_dbregs) {
4379 LOCK_PFS(flags);
4380 pfm_sessions.pfs_sys_use_dbregs--;
4381 UNLOCK_PFS(flags);
4382 }
4383
4384
4385
4386 if (is_system == 0 && task != current) {
4387 pfm_put_task(task);
4388
4389 if (ret == 0) {
4390 ret = pfm_check_task_exist(ctx);
4391 if (ret) {
4392 ctx->ctx_state = PFM_CTX_UNLOADED;
4393 ctx->ctx_task = NULL;
4394 }
4395 }
4396 }
4397 return ret;
4398}
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408static void pfm_flush_pmds(struct task_struct *, pfm_context_t *ctx);
4409
4410static int
4411pfm_context_unload(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
4412{
4413 struct task_struct *task = PFM_CTX_TASK(ctx);
4414 struct pt_regs *tregs;
4415 int prev_state, is_system;
4416 int ret;
4417
4418 DPRINT(("ctx_state=%d task [%d]\n", ctx->ctx_state, task ? task_pid_nr(task) : -1));
4419
4420 prev_state = ctx->ctx_state;
4421 is_system = ctx->ctx_fl_system;
4422
4423
4424
4425
4426 if (prev_state == PFM_CTX_UNLOADED) {
4427 DPRINT(("ctx_state=%d, nothing to do\n", prev_state));
4428 return 0;
4429 }
4430
4431
4432
4433
4434 ret = pfm_stop(ctx, NULL, 0, regs);
4435 if (ret) return ret;
4436
4437 ctx->ctx_state = PFM_CTX_UNLOADED;
4438
4439
4440
4441
4442
4443
4444 if (is_system) {
4445
4446
4447
4448
4449
4450
4451 PFM_CPUINFO_CLEAR(PFM_CPUINFO_SYST_WIDE);
4452 PFM_CPUINFO_CLEAR(PFM_CPUINFO_EXCL_IDLE);
4453
4454
4455
4456
4457
4458 pfm_flush_pmds(current, ctx);
4459
4460
4461
4462
4463
4464 if (prev_state != PFM_CTX_ZOMBIE)
4465 pfm_unreserve_session(ctx, 1 , ctx->ctx_cpu);
4466
4467
4468
4469
4470 task->thread.pfm_context = NULL;
4471
4472
4473
4474 ctx->ctx_task = NULL;
4475
4476
4477
4478
4479 return 0;
4480 }
4481
4482
4483
4484
4485 tregs = task == current ? regs : task_pt_regs(task);
4486
4487 if (task == current) {
4488
4489
4490
4491 ia64_psr(regs)->sp = 1;
4492
4493 DPRINT(("setting psr.sp for [%d]\n", task_pid_nr(task)));
4494 }
4495
4496
4497
4498
4499 pfm_flush_pmds(task, ctx);
4500
4501
4502
4503
4504
4505
4506
4507 if (prev_state != PFM_CTX_ZOMBIE)
4508 pfm_unreserve_session(ctx, 0 , ctx->ctx_cpu);
4509
4510
4511
4512
4513 ctx->ctx_last_activation = PFM_INVALID_ACTIVATION;
4514 SET_LAST_CPU(ctx, -1);
4515
4516
4517
4518
4519 task->thread.flags &= ~IA64_THREAD_PM_VALID;
4520
4521
4522
4523
4524 task->thread.pfm_context = NULL;
4525 ctx->ctx_task = NULL;
4526
4527 PFM_SET_WORK_PENDING(task, 0);
4528
4529 ctx->ctx_fl_trap_reason = PFM_TRAP_REASON_NONE;
4530 ctx->ctx_fl_can_restart = 0;
4531 ctx->ctx_fl_going_zombie = 0;
4532
4533 DPRINT(("disconnected [%d] from context\n", task_pid_nr(task)));
4534
4535 return 0;
4536}
4537
4538
4539
4540
4541
4542
4543void
4544pfm_exit_thread(struct task_struct *task)
4545{
4546 pfm_context_t *ctx;
4547 unsigned long flags;
4548 struct pt_regs *regs = task_pt_regs(task);
4549 int ret, state;
4550 int free_ok = 0;
4551
4552 ctx = PFM_GET_CTX(task);
4553
4554 PROTECT_CTX(ctx, flags);
4555
4556 DPRINT(("state=%d task [%d]\n", ctx->ctx_state, task_pid_nr(task)));
4557
4558 state = ctx->ctx_state;
4559 switch(state) {
4560 case PFM_CTX_UNLOADED:
4561
4562
4563
4564
4565 printk(KERN_ERR "perfmon: pfm_exit_thread [%d] ctx unloaded\n", task_pid_nr(task));
4566 break;
4567 case PFM_CTX_LOADED:
4568 case PFM_CTX_MASKED:
4569 ret = pfm_context_unload(ctx, NULL, 0, regs);
4570 if (ret) {
4571 printk(KERN_ERR "perfmon: pfm_exit_thread [%d] state=%d unload failed %d\n", task_pid_nr(task), state, ret);
4572 }
4573 DPRINT(("ctx unloaded for current state was %d\n", state));
4574
4575 pfm_end_notify_user(ctx);
4576 break;
4577 case PFM_CTX_ZOMBIE:
4578 ret = pfm_context_unload(ctx, NULL, 0, regs);
4579 if (ret) {
4580 printk(KERN_ERR "perfmon: pfm_exit_thread [%d] state=%d unload failed %d\n", task_pid_nr(task), state, ret);
4581 }
4582 free_ok = 1;
4583 break;
4584 default:
4585 printk(KERN_ERR "perfmon: pfm_exit_thread [%d] unexpected state=%d\n", task_pid_nr(task), state);
4586 break;
4587 }
4588 UNPROTECT_CTX(ctx, flags);
4589
4590 { u64 psr = pfm_get_psr();
4591 BUG_ON(psr & (IA64_PSR_UP|IA64_PSR_PP));
4592 BUG_ON(GET_PMU_OWNER());
4593 BUG_ON(ia64_psr(regs)->up);
4594 BUG_ON(ia64_psr(regs)->pp);
4595 }
4596
4597
4598
4599
4600
4601 if (free_ok) pfm_context_free(ctx);
4602}
4603
4604
4605
4606
4607#define PFM_CMD(name, flags, arg_count, arg_type, getsz) { name, #name, flags, arg_count, sizeof(arg_type), getsz }
4608#define PFM_CMD_S(name, flags) { name, #name, flags, 0, 0, NULL }
4609#define PFM_CMD_PCLRWS (PFM_CMD_FD|PFM_CMD_ARG_RW|PFM_CMD_STOP)
4610#define PFM_CMD_PCLRW (PFM_CMD_FD|PFM_CMD_ARG_RW)
4611#define PFM_CMD_NONE { NULL, "no-cmd", 0, 0, 0, NULL}
4612
4613static pfm_cmd_desc_t pfm_cmd_tab[]={
4614PFM_CMD_NONE,
4615PFM_CMD(pfm_write_pmcs, PFM_CMD_PCLRWS, PFM_CMD_ARG_MANY, pfarg_reg_t, NULL),
4616PFM_CMD(pfm_write_pmds, PFM_CMD_PCLRWS, PFM_CMD_ARG_MANY, pfarg_reg_t, NULL),
4617PFM_CMD(pfm_read_pmds, PFM_CMD_PCLRWS, PFM_CMD_ARG_MANY, pfarg_reg_t, NULL),
4618PFM_CMD_S(pfm_stop, PFM_CMD_PCLRWS),
4619PFM_CMD_S(pfm_start, PFM_CMD_PCLRWS),
4620PFM_CMD_NONE,
4621PFM_CMD_NONE,
4622PFM_CMD(pfm_context_create, PFM_CMD_ARG_RW, 1, pfarg_context_t, pfm_ctx_getsize),
4623PFM_CMD_NONE,
4624PFM_CMD_S(pfm_restart, PFM_CMD_PCLRW),
4625PFM_CMD_NONE,
4626PFM_CMD(pfm_get_features, PFM_CMD_ARG_RW, 1, pfarg_features_t, NULL),
4627PFM_CMD(pfm_debug, 0, 1, unsigned int, NULL),
4628PFM_CMD_NONE,
4629PFM_CMD(pfm_get_pmc_reset, PFM_CMD_ARG_RW, PFM_CMD_ARG_MANY, pfarg_reg_t, NULL),
4630PFM_CMD(pfm_context_load, PFM_CMD_PCLRWS, 1, pfarg_load_t, NULL),
4631PFM_CMD_S(pfm_context_unload, PFM_CMD_PCLRWS),
4632PFM_CMD_NONE,
4633PFM_CMD_NONE,
4634PFM_CMD_NONE,
4635PFM_CMD_NONE,
4636PFM_CMD_NONE,
4637PFM_CMD_NONE,
4638PFM_CMD_NONE,
4639PFM_CMD_NONE,
4640PFM_CMD_NONE,
4641PFM_CMD_NONE,
4642PFM_CMD_NONE,
4643PFM_CMD_NONE,
4644PFM_CMD_NONE,
4645PFM_CMD_NONE,
4646PFM_CMD(pfm_write_ibrs, PFM_CMD_PCLRWS, PFM_CMD_ARG_MANY, pfarg_dbreg_t, NULL),
4647PFM_CMD(pfm_write_dbrs, PFM_CMD_PCLRWS, PFM_CMD_ARG_MANY, pfarg_dbreg_t, NULL)
4648};
4649#define PFM_CMD_COUNT (sizeof(pfm_cmd_tab)/sizeof(pfm_cmd_desc_t))
4650
4651static int
4652pfm_check_task_state(pfm_context_t *ctx, int cmd, unsigned long flags)
4653{
4654 struct task_struct *task;
4655 int state, old_state;
4656
4657recheck:
4658 state = ctx->ctx_state;
4659 task = ctx->ctx_task;
4660
4661 if (task == NULL) {
4662 DPRINT(("context %d no task, state=%d\n", ctx->ctx_fd, state));
4663 return 0;
4664 }
4665
4666 DPRINT(("context %d state=%d [%d] task_state=%ld must_stop=%d\n",
4667 ctx->ctx_fd,
4668 state,
4669 task_pid_nr(task),
4670 task->state, PFM_CMD_STOPPED(cmd)));
4671
4672
4673
4674
4675
4676
4677
4678
4679 if (task == current || ctx->ctx_fl_system) return 0;
4680
4681
4682
4683
4684 switch(state) {
4685 case PFM_CTX_UNLOADED:
4686
4687
4688
4689 return 0;
4690 case PFM_CTX_ZOMBIE:
4691
4692
4693
4694 DPRINT(("cmd %d state zombie cannot operate on context\n", cmd));
4695 return -EINVAL;
4696 case PFM_CTX_MASKED:
4697
4698
4699
4700
4701 if (cmd != PFM_UNLOAD_CONTEXT) return 0;
4702 }
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714 if (PFM_CMD_STOPPED(cmd)) {
4715 if (!task_is_stopped_or_traced(task)) {
4716 DPRINT(("[%d] task not in stopped state\n", task_pid_nr(task)));
4717 return -EBUSY;
4718 }
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733 old_state = state;
4734
4735 UNPROTECT_CTX(ctx, flags);
4736
4737 wait_task_inactive(task, 0);
4738
4739 PROTECT_CTX(ctx, flags);
4740
4741
4742
4743
4744 if (ctx->ctx_state != old_state) {
4745 DPRINT(("old_state=%d new_state=%d\n", old_state, ctx->ctx_state));
4746 goto recheck;
4747 }
4748 }
4749 return 0;
4750}
4751
4752
4753
4754
4755asmlinkage long
4756sys_perfmonctl (int fd, int cmd, void __user *arg, int count)
4757{
4758 struct fd f = {NULL, 0};
4759 pfm_context_t *ctx = NULL;
4760 unsigned long flags = 0UL;
4761 void *args_k = NULL;
4762 long ret;
4763 size_t base_sz, sz, xtra_sz = 0;
4764 int narg, completed_args = 0, call_made = 0, cmd_flags;
4765 int (*func)(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs);
4766 int (*getsize)(void *arg, size_t *sz);
4767#define PFM_MAX_ARGSIZE 4096
4768
4769
4770
4771
4772 if (unlikely(pmu_conf == NULL)) return -ENOSYS;
4773
4774 if (unlikely(cmd < 0 || cmd >= PFM_CMD_COUNT)) {
4775 DPRINT(("invalid cmd=%d\n", cmd));
4776 return -EINVAL;
4777 }
4778
4779 func = pfm_cmd_tab[cmd].cmd_func;
4780 narg = pfm_cmd_tab[cmd].cmd_narg;
4781 base_sz = pfm_cmd_tab[cmd].cmd_argsize;
4782 getsize = pfm_cmd_tab[cmd].cmd_getsize;
4783 cmd_flags = pfm_cmd_tab[cmd].cmd_flags;
4784
4785 if (unlikely(func == NULL)) {
4786 DPRINT(("invalid cmd=%d\n", cmd));
4787 return -EINVAL;
4788 }
4789
4790 DPRINT(("cmd=%s idx=%d narg=0x%x argsz=%lu count=%d\n",
4791 PFM_CMD_NAME(cmd),
4792 cmd,
4793 narg,
4794 base_sz,
4795 count));
4796
4797
4798
4799
4800 if (unlikely((narg == PFM_CMD_ARG_MANY && count <= 0) || (narg > 0 && narg != count)))
4801 return -EINVAL;
4802
4803restart_args:
4804 sz = xtra_sz + base_sz*count;
4805
4806
4807
4808 if (unlikely(sz > PFM_MAX_ARGSIZE)) {
4809 printk(KERN_ERR "perfmon: [%d] argument too big %lu\n", task_pid_nr(current), sz);
4810 return -E2BIG;
4811 }
4812
4813
4814
4815
4816 if (likely(count && args_k == NULL)) {
4817 args_k = kmalloc(PFM_MAX_ARGSIZE, GFP_KERNEL);
4818 if (args_k == NULL) return -ENOMEM;
4819 }
4820
4821 ret = -EFAULT;
4822
4823
4824
4825
4826
4827
4828 if (sz && copy_from_user(args_k, arg, sz)) {
4829 DPRINT(("cannot copy_from_user %lu bytes @%p\n", sz, arg));
4830 goto error_args;
4831 }
4832
4833
4834
4835
4836 if (completed_args == 0 && getsize) {
4837
4838
4839
4840 ret = (*getsize)(args_k, &xtra_sz);
4841 if (ret) goto error_args;
4842
4843 completed_args = 1;
4844
4845 DPRINT(("restart_args sz=%lu xtra_sz=%lu\n", sz, xtra_sz));
4846
4847
4848 if (likely(xtra_sz)) goto restart_args;
4849 }
4850
4851 if (unlikely((cmd_flags & PFM_CMD_FD) == 0)) goto skip_fd;
4852
4853 ret = -EBADF;
4854
4855 f = fdget(fd);
4856 if (unlikely(f.file == NULL)) {
4857 DPRINT(("invalid fd %d\n", fd));
4858 goto error_args;
4859 }
4860 if (unlikely(PFM_IS_FILE(f.file) == 0)) {
4861 DPRINT(("fd %d not related to perfmon\n", fd));
4862 goto error_args;
4863 }
4864
4865 ctx = f.file->private_data;
4866 if (unlikely(ctx == NULL)) {
4867 DPRINT(("no context for fd %d\n", fd));
4868 goto error_args;
4869 }
4870 prefetch(&ctx->ctx_state);
4871
4872 PROTECT_CTX(ctx, flags);
4873
4874
4875
4876
4877 ret = pfm_check_task_state(ctx, cmd, flags);
4878 if (unlikely(ret)) goto abort_locked;
4879
4880skip_fd:
4881 ret = (*func)(ctx, args_k, count, task_pt_regs(current));
4882
4883 call_made = 1;
4884
4885abort_locked:
4886 if (likely(ctx)) {
4887 DPRINT(("context unlocked\n"));
4888 UNPROTECT_CTX(ctx, flags);
4889 }
4890
4891
4892 if (call_made && PFM_CMD_RW_ARG(cmd) && copy_to_user(arg, args_k, base_sz*count)) ret = -EFAULT;
4893
4894error_args:
4895 if (f.file)
4896 fdput(f);
4897
4898 kfree(args_k);
4899
4900 DPRINT(("cmd=%s ret=%ld\n", PFM_CMD_NAME(cmd), ret));
4901
4902 return ret;
4903}
4904
4905static void
4906pfm_resume_after_ovfl(pfm_context_t *ctx, unsigned long ovfl_regs, struct pt_regs *regs)
4907{
4908 pfm_buffer_fmt_t *fmt = ctx->ctx_buf_fmt;
4909 pfm_ovfl_ctrl_t rst_ctrl;
4910 int state;
4911 int ret = 0;
4912
4913 state = ctx->ctx_state;
4914
4915
4916
4917
4918 if (CTX_HAS_SMPL(ctx)) {
4919
4920 rst_ctrl.bits.mask_monitoring = 0;
4921 rst_ctrl.bits.reset_ovfl_pmds = 0;
4922
4923 if (state == PFM_CTX_LOADED)
4924 ret = pfm_buf_fmt_restart_active(fmt, current, &rst_ctrl, ctx->ctx_smpl_hdr, regs);
4925 else
4926 ret = pfm_buf_fmt_restart(fmt, current, &rst_ctrl, ctx->ctx_smpl_hdr, regs);
4927 } else {
4928 rst_ctrl.bits.mask_monitoring = 0;
4929 rst_ctrl.bits.reset_ovfl_pmds = 1;
4930 }
4931
4932 if (ret == 0) {
4933 if (rst_ctrl.bits.reset_ovfl_pmds) {
4934 pfm_reset_regs(ctx, &ovfl_regs, PFM_PMD_LONG_RESET);
4935 }
4936 if (rst_ctrl.bits.mask_monitoring == 0) {
4937 DPRINT(("resuming monitoring\n"));
4938 if (ctx->ctx_state == PFM_CTX_MASKED) pfm_restore_monitoring(current);
4939 } else {
4940 DPRINT(("stopping monitoring\n"));
4941
4942 }
4943 ctx->ctx_state = PFM_CTX_LOADED;
4944 }
4945}
4946
4947
4948
4949
4950
4951static void
4952pfm_context_force_terminate(pfm_context_t *ctx, struct pt_regs *regs)
4953{
4954 int ret;
4955
4956 DPRINT(("entering for [%d]\n", task_pid_nr(current)));
4957
4958 ret = pfm_context_unload(ctx, NULL, 0, regs);
4959 if (ret) {
4960 printk(KERN_ERR "pfm_context_force_terminate: [%d] unloaded failed with %d\n", task_pid_nr(current), ret);
4961 }
4962
4963
4964
4965
4966 wake_up_interruptible(&ctx->ctx_zombieq);
4967
4968
4969
4970
4971
4972
4973}
4974
4975static int pfm_ovfl_notify_user(pfm_context_t *ctx, unsigned long ovfl_pmds);
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986void
4987pfm_handle_work(void)
4988{
4989 pfm_context_t *ctx;
4990 struct pt_regs *regs;
4991 unsigned long flags, dummy_flags;
4992 unsigned long ovfl_regs;
4993 unsigned int reason;
4994 int ret;
4995
4996 ctx = PFM_GET_CTX(current);
4997 if (ctx == NULL) {
4998 printk(KERN_ERR "perfmon: [%d] has no PFM context\n",
4999 task_pid_nr(current));
5000 return;
5001 }
5002
5003 PROTECT_CTX(ctx, flags);
5004
5005 PFM_SET_WORK_PENDING(current, 0);
5006
5007 regs = task_pt_regs(current);
5008
5009
5010
5011
5012 reason = ctx->ctx_fl_trap_reason;
5013 ctx->ctx_fl_trap_reason = PFM_TRAP_REASON_NONE;
5014 ovfl_regs = ctx->ctx_ovfl_regs[0];
5015
5016 DPRINT(("reason=%d state=%d\n", reason, ctx->ctx_state));
5017
5018
5019
5020
5021 if (ctx->ctx_fl_going_zombie || ctx->ctx_state == PFM_CTX_ZOMBIE)
5022 goto do_zombie;
5023
5024
5025 if (reason == PFM_TRAP_REASON_RESET)
5026 goto skip_blocking;
5027
5028
5029
5030
5031
5032 UNPROTECT_CTX(ctx, flags);
5033
5034
5035
5036
5037 local_irq_enable();
5038
5039 DPRINT(("before block sleeping\n"));
5040
5041
5042
5043
5044
5045 ret = wait_for_completion_interruptible(&ctx->ctx_restart_done);
5046
5047 DPRINT(("after block sleeping ret=%d\n", ret));
5048
5049
5050
5051
5052
5053
5054
5055 PROTECT_CTX(ctx, dummy_flags);
5056
5057
5058
5059
5060
5061
5062
5063 ovfl_regs = ctx->ctx_ovfl_regs[0];
5064
5065 if (ctx->ctx_fl_going_zombie) {
5066do_zombie:
5067 DPRINT(("context is zombie, bailing out\n"));
5068 pfm_context_force_terminate(ctx, regs);
5069 goto nothing_to_do;
5070 }
5071
5072
5073
5074 if (ret < 0)
5075 goto nothing_to_do;
5076
5077skip_blocking:
5078 pfm_resume_after_ovfl(ctx, ovfl_regs, regs);
5079 ctx->ctx_ovfl_regs[0] = 0UL;
5080
5081nothing_to_do:
5082
5083
5084
5085 UNPROTECT_CTX(ctx, flags);
5086}
5087
5088static int
5089pfm_notify_user(pfm_context_t *ctx, pfm_msg_t *msg)
5090{
5091 if (ctx->ctx_state == PFM_CTX_ZOMBIE) {
5092 DPRINT(("ignoring overflow notification, owner is zombie\n"));
5093 return 0;
5094 }
5095
5096 DPRINT(("waking up somebody\n"));
5097
5098 if (msg) wake_up_interruptible(&ctx->ctx_msgq_wait);
5099
5100
5101
5102
5103
5104 kill_fasync (&ctx->ctx_async_queue, SIGIO, POLL_IN);
5105
5106 return 0;
5107}
5108
5109static int
5110pfm_ovfl_notify_user(pfm_context_t *ctx, unsigned long ovfl_pmds)
5111{
5112 pfm_msg_t *msg = NULL;
5113
5114 if (ctx->ctx_fl_no_msg == 0) {
5115 msg = pfm_get_new_msg(ctx);
5116 if (msg == NULL) {
5117 printk(KERN_ERR "perfmon: pfm_ovfl_notify_user no more notification msgs\n");
5118 return -1;
5119 }
5120
5121 msg->pfm_ovfl_msg.msg_type = PFM_MSG_OVFL;
5122 msg->pfm_ovfl_msg.msg_ctx_fd = ctx->ctx_fd;
5123 msg->pfm_ovfl_msg.msg_active_set = 0;
5124 msg->pfm_ovfl_msg.msg_ovfl_pmds[0] = ovfl_pmds;
5125 msg->pfm_ovfl_msg.msg_ovfl_pmds[1] = 0UL;
5126 msg->pfm_ovfl_msg.msg_ovfl_pmds[2] = 0UL;
5127 msg->pfm_ovfl_msg.msg_ovfl_pmds[3] = 0UL;
5128 msg->pfm_ovfl_msg.msg_tstamp = 0UL;
5129 }
5130
5131 DPRINT(("ovfl msg: msg=%p no_msg=%d fd=%d ovfl_pmds=0x%lx\n",
5132 msg,
5133 ctx->ctx_fl_no_msg,
5134 ctx->ctx_fd,
5135 ovfl_pmds));
5136
5137 return pfm_notify_user(ctx, msg);
5138}
5139
5140static int
5141pfm_end_notify_user(pfm_context_t *ctx)
5142{
5143 pfm_msg_t *msg;
5144
5145 msg = pfm_get_new_msg(ctx);
5146 if (msg == NULL) {
5147 printk(KERN_ERR "perfmon: pfm_end_notify_user no more notification msgs\n");
5148 return -1;
5149 }
5150
5151 memset(msg, 0, sizeof(*msg));
5152
5153 msg->pfm_end_msg.msg_type = PFM_MSG_END;
5154 msg->pfm_end_msg.msg_ctx_fd = ctx->ctx_fd;
5155 msg->pfm_ovfl_msg.msg_tstamp = 0UL;
5156
5157 DPRINT(("end msg: msg=%p no_msg=%d ctx_fd=%d\n",
5158 msg,
5159 ctx->ctx_fl_no_msg,
5160 ctx->ctx_fd));
5161
5162 return pfm_notify_user(ctx, msg);
5163}
5164
5165
5166
5167
5168
5169static void pfm_overflow_handler(struct task_struct *task, pfm_context_t *ctx,
5170 unsigned long pmc0, struct pt_regs *regs)
5171{
5172 pfm_ovfl_arg_t *ovfl_arg;
5173 unsigned long mask;
5174 unsigned long old_val, ovfl_val, new_val;
5175 unsigned long ovfl_notify = 0UL, ovfl_pmds = 0UL, smpl_pmds = 0UL, reset_pmds;
5176 unsigned long tstamp;
5177 pfm_ovfl_ctrl_t ovfl_ctrl;
5178 unsigned int i, has_smpl;
5179 int must_notify = 0;
5180
5181 if (unlikely(ctx->ctx_state == PFM_CTX_ZOMBIE)) goto stop_monitoring;
5182
5183
5184
5185
5186 if (unlikely((pmc0 & 0x1) == 0)) goto sanity_check;
5187
5188 tstamp = ia64_get_itc();
5189 mask = pmc0 >> PMU_FIRST_COUNTER;
5190 ovfl_val = pmu_conf->ovfl_val;
5191 has_smpl = CTX_HAS_SMPL(ctx);
5192
5193 DPRINT_ovfl(("pmc0=0x%lx pid=%d iip=0x%lx, %s "
5194 "used_pmds=0x%lx\n",
5195 pmc0,
5196 task ? task_pid_nr(task): -1,
5197 (regs ? regs->cr_iip : 0),
5198 CTX_OVFL_NOBLOCK(ctx) ? "nonblocking" : "blocking",
5199 ctx->ctx_used_pmds[0]));
5200
5201
5202
5203
5204
5205
5206 for (i = PMU_FIRST_COUNTER; mask ; i++, mask >>= 1) {
5207
5208
5209 if ((mask & 0x1) == 0) continue;
5210
5211
5212
5213
5214
5215
5216
5217 old_val = new_val = ctx->ctx_pmds[i].val;
5218 new_val += 1 + ovfl_val;
5219 ctx->ctx_pmds[i].val = new_val;
5220
5221
5222
5223
5224 if (likely(old_val > new_val)) {
5225 ovfl_pmds |= 1UL << i;
5226 if (PMC_OVFL_NOTIFY(ctx, i)) ovfl_notify |= 1UL << i;
5227 }
5228
5229 DPRINT_ovfl(("ctx_pmd[%d].val=0x%lx old_val=0x%lx pmd=0x%lx ovfl_pmds=0x%lx ovfl_notify=0x%lx\n",
5230 i,
5231 new_val,
5232 old_val,
5233 ia64_get_pmd(i) & ovfl_val,
5234 ovfl_pmds,
5235 ovfl_notify));
5236 }
5237
5238
5239
5240
5241 if (ovfl_pmds == 0UL) return;
5242
5243
5244
5245
5246 ovfl_ctrl.val = 0;
5247 reset_pmds = 0UL;
5248
5249
5250
5251
5252
5253 if (has_smpl) {
5254 unsigned long start_cycles, end_cycles;
5255 unsigned long pmd_mask;
5256 int j, k, ret = 0;
5257 int this_cpu = smp_processor_id();
5258
5259 pmd_mask = ovfl_pmds >> PMU_FIRST_COUNTER;
5260 ovfl_arg = &ctx->ctx_ovfl_arg;
5261
5262 prefetch(ctx->ctx_smpl_hdr);
5263
5264 for(i=PMU_FIRST_COUNTER; pmd_mask && ret == 0; i++, pmd_mask >>=1) {
5265
5266 mask = 1UL << i;
5267
5268 if ((pmd_mask & 0x1) == 0) continue;
5269
5270 ovfl_arg->ovfl_pmd = (unsigned char )i;
5271 ovfl_arg->ovfl_notify = ovfl_notify & mask ? 1 : 0;
5272 ovfl_arg->active_set = 0;
5273 ovfl_arg->ovfl_ctrl.val = 0;
5274 ovfl_arg->smpl_pmds[0] = smpl_pmds = ctx->ctx_pmds[i].smpl_pmds[0];
5275
5276 ovfl_arg->pmd_value = ctx->ctx_pmds[i].val;
5277 ovfl_arg->pmd_last_reset = ctx->ctx_pmds[i].lval;
5278 ovfl_arg->pmd_eventid = ctx->ctx_pmds[i].eventid;
5279
5280
5281
5282
5283
5284 if (smpl_pmds) {
5285 for(j=0, k=0; smpl_pmds; j++, smpl_pmds >>=1) {
5286 if ((smpl_pmds & 0x1) == 0) continue;
5287 ovfl_arg->smpl_pmds_values[k++] = PMD_IS_COUNTING(j) ? pfm_read_soft_counter(ctx, j) : ia64_get_pmd(j);
5288 DPRINT_ovfl(("smpl_pmd[%d]=pmd%u=0x%lx\n", k-1, j, ovfl_arg->smpl_pmds_values[k-1]));
5289 }
5290 }
5291
5292 pfm_stats[this_cpu].pfm_smpl_handler_calls++;
5293
5294 start_cycles = ia64_get_itc();
5295
5296
5297
5298
5299 ret = (*ctx->ctx_buf_fmt->fmt_handler)(task, ctx->ctx_smpl_hdr, ovfl_arg, regs, tstamp);
5300
5301 end_cycles = ia64_get_itc();
5302
5303
5304
5305
5306
5307 ovfl_ctrl.bits.notify_user |= ovfl_arg->ovfl_ctrl.bits.notify_user;
5308 ovfl_ctrl.bits.block_task |= ovfl_arg->ovfl_ctrl.bits.block_task;
5309 ovfl_ctrl.bits.mask_monitoring |= ovfl_arg->ovfl_ctrl.bits.mask_monitoring;
5310
5311
5312
5313 if (ovfl_arg->ovfl_ctrl.bits.reset_ovfl_pmds) reset_pmds |= mask;
5314
5315 pfm_stats[this_cpu].pfm_smpl_handler_cycles += end_cycles - start_cycles;
5316 }
5317
5318
5319
5320 if (ret && pmd_mask) {
5321 DPRINT(("handler aborts leftover ovfl_pmds=0x%lx\n",
5322 pmd_mask<<PMU_FIRST_COUNTER));
5323 }
5324
5325
5326
5327 ovfl_pmds &= ~reset_pmds;
5328 } else {
5329
5330
5331
5332
5333 ovfl_ctrl.bits.notify_user = ovfl_notify ? 1 : 0;
5334 ovfl_ctrl.bits.block_task = ovfl_notify ? 1 : 0;
5335 ovfl_ctrl.bits.mask_monitoring = ovfl_notify ? 1 : 0;
5336 ovfl_ctrl.bits.reset_ovfl_pmds = ovfl_notify ? 0 : 1;
5337
5338
5339
5340 if (ovfl_notify == 0) reset_pmds = ovfl_pmds;
5341 }
5342
5343 DPRINT_ovfl(("ovfl_pmds=0x%lx reset_pmds=0x%lx\n", ovfl_pmds, reset_pmds));
5344
5345
5346
5347
5348 if (reset_pmds) {
5349 unsigned long bm = reset_pmds;
5350 pfm_reset_regs(ctx, &bm, PFM_PMD_SHORT_RESET);
5351 }
5352
5353 if (ovfl_notify && ovfl_ctrl.bits.notify_user) {
5354
5355
5356
5357 ctx->ctx_ovfl_regs[0] = ovfl_pmds;
5358
5359
5360
5361
5362 if (CTX_OVFL_NOBLOCK(ctx) == 0 && ovfl_ctrl.bits.block_task) {
5363
5364 ctx->ctx_fl_trap_reason = PFM_TRAP_REASON_BLOCK;
5365
5366
5367
5368
5369 PFM_SET_WORK_PENDING(task, 1);
5370
5371
5372
5373
5374
5375 set_notify_resume(task);
5376 }
5377
5378
5379
5380
5381 must_notify = 1;
5382 }
5383
5384 DPRINT_ovfl(("owner [%d] pending=%ld reason=%u ovfl_pmds=0x%lx ovfl_notify=0x%lx masked=%d\n",
5385 GET_PMU_OWNER() ? task_pid_nr(GET_PMU_OWNER()) : -1,
5386 PFM_GET_WORK_PENDING(task),
5387 ctx->ctx_fl_trap_reason,
5388 ovfl_pmds,
5389 ovfl_notify,
5390 ovfl_ctrl.bits.mask_monitoring ? 1 : 0));
5391
5392
5393
5394 if (ovfl_ctrl.bits.mask_monitoring) {
5395 pfm_mask_monitoring(task);
5396 ctx->ctx_state = PFM_CTX_MASKED;
5397 ctx->ctx_fl_can_restart = 1;
5398 }
5399
5400
5401
5402
5403 if (must_notify) pfm_ovfl_notify_user(ctx, ovfl_notify);
5404
5405 return;
5406
5407sanity_check:
5408 printk(KERN_ERR "perfmon: CPU%d overflow handler [%d] pmc0=0x%lx\n",
5409 smp_processor_id(),
5410 task ? task_pid_nr(task) : -1,
5411 pmc0);
5412 return;
5413
5414stop_monitoring:
5415
5416
5417
5418
5419
5420
5421
5422
5423
5424
5425
5426
5427
5428
5429
5430
5431
5432
5433
5434
5435
5436
5437
5438
5439
5440
5441
5442
5443 DPRINT(("ctx is zombie for [%d], converted to spurious\n", task ? task_pid_nr(task): -1));
5444 pfm_clear_psr_up();
5445 ia64_psr(regs)->up = 0;
5446 ia64_psr(regs)->sp = 1;
5447 return;
5448}
5449
5450static int
5451pfm_do_interrupt_handler(void *arg, struct pt_regs *regs)
5452{
5453 struct task_struct *task;
5454 pfm_context_t *ctx;
5455 unsigned long flags;
5456 u64 pmc0;
5457 int this_cpu = smp_processor_id();
5458 int retval = 0;
5459
5460 pfm_stats[this_cpu].pfm_ovfl_intr_count++;
5461
5462
5463
5464
5465 pmc0 = ia64_get_pmc(0);
5466
5467 task = GET_PMU_OWNER();
5468 ctx = GET_PMU_CTX();
5469
5470
5471
5472
5473
5474 if (PMC0_HAS_OVFL(pmc0) && task) {
5475
5476
5477
5478
5479
5480 if (!ctx) goto report_spurious1;
5481
5482 if (ctx->ctx_fl_system == 0 && (task->thread.flags & IA64_THREAD_PM_VALID) == 0)
5483 goto report_spurious2;
5484
5485 PROTECT_CTX_NOPRINT(ctx, flags);
5486
5487 pfm_overflow_handler(task, ctx, pmc0, regs);
5488
5489 UNPROTECT_CTX_NOPRINT(ctx, flags);
5490
5491 } else {
5492 pfm_stats[this_cpu].pfm_spurious_ovfl_intr_count++;
5493 retval = -1;
5494 }
5495
5496
5497
5498 pfm_unfreeze_pmu();
5499
5500 return retval;
5501
5502report_spurious1:
5503 printk(KERN_INFO "perfmon: spurious overflow interrupt on CPU%d: process %d has no PFM context\n",
5504 this_cpu, task_pid_nr(task));
5505 pfm_unfreeze_pmu();
5506 return -1;
5507report_spurious2:
5508 printk(KERN_INFO "perfmon: spurious overflow interrupt on CPU%d: process %d, invalid flag\n",
5509 this_cpu,
5510 task_pid_nr(task));
5511 pfm_unfreeze_pmu();
5512 return -1;
5513}
5514
5515static irqreturn_t
5516pfm_interrupt_handler(int irq, void *arg)
5517{
5518 unsigned long start_cycles, total_cycles;
5519 unsigned long min, max;
5520 int this_cpu;
5521 int ret;
5522 struct pt_regs *regs = get_irq_regs();
5523
5524 this_cpu = get_cpu();
5525 if (likely(!pfm_alt_intr_handler)) {
5526 min = pfm_stats[this_cpu].pfm_ovfl_intr_cycles_min;
5527 max = pfm_stats[this_cpu].pfm_ovfl_intr_cycles_max;
5528
5529 start_cycles = ia64_get_itc();
5530
5531 ret = pfm_do_interrupt_handler(arg, regs);
5532
5533 total_cycles = ia64_get_itc();
5534
5535
5536
5537
5538 if (likely(ret == 0)) {
5539 total_cycles -= start_cycles;
5540
5541 if (total_cycles < min) pfm_stats[this_cpu].pfm_ovfl_intr_cycles_min = total_cycles;
5542 if (total_cycles > max) pfm_stats[this_cpu].pfm_ovfl_intr_cycles_max = total_cycles;
5543
5544 pfm_stats[this_cpu].pfm_ovfl_intr_cycles += total_cycles;
5545 }
5546 }
5547 else {
5548 (*pfm_alt_intr_handler->handler)(irq, arg, regs);
5549 }
5550
5551 put_cpu();
5552 return IRQ_HANDLED;
5553}
5554
5555
5556
5557
5558
5559#define PFM_PROC_SHOW_HEADER ((void *)(long)nr_cpu_ids+1)
5560
5561static void *
5562pfm_proc_start(struct seq_file *m, loff_t *pos)
5563{
5564 if (*pos == 0) {
5565 return PFM_PROC_SHOW_HEADER;
5566 }
5567
5568 while (*pos <= nr_cpu_ids) {
5569 if (cpu_online(*pos - 1)) {
5570 return (void *)*pos;
5571 }
5572 ++*pos;
5573 }
5574 return NULL;
5575}
5576
5577static void *
5578pfm_proc_next(struct seq_file *m, void *v, loff_t *pos)
5579{
5580 ++*pos;
5581 return pfm_proc_start(m, pos);
5582}
5583
5584static void
5585pfm_proc_stop(struct seq_file *m, void *v)
5586{
5587}
5588
5589static void
5590pfm_proc_show_header(struct seq_file *m)
5591{
5592 struct list_head * pos;
5593 pfm_buffer_fmt_t * entry;
5594 unsigned long flags;
5595
5596 seq_printf(m,
5597 "perfmon version : %u.%u\n"
5598 "model : %s\n"
5599 "fastctxsw : %s\n"
5600 "expert mode : %s\n"
5601 "ovfl_mask : 0x%lx\n"
5602 "PMU flags : 0x%x\n",
5603 PFM_VERSION_MAJ, PFM_VERSION_MIN,
5604 pmu_conf->pmu_name,
5605 pfm_sysctl.fastctxsw > 0 ? "Yes": "No",
5606 pfm_sysctl.expert_mode > 0 ? "Yes": "No",
5607 pmu_conf->ovfl_val,
5608 pmu_conf->flags);
5609
5610 LOCK_PFS(flags);
5611
5612 seq_printf(m,
5613 "proc_sessions : %u\n"
5614 "sys_sessions : %u\n"
5615 "sys_use_dbregs : %u\n"
5616 "ptrace_use_dbregs : %u\n",
5617 pfm_sessions.pfs_task_sessions,
5618 pfm_sessions.pfs_sys_sessions,
5619 pfm_sessions.pfs_sys_use_dbregs,
5620 pfm_sessions.pfs_ptrace_use_dbregs);
5621
5622 UNLOCK_PFS(flags);
5623
5624 spin_lock(&pfm_buffer_fmt_lock);
5625
5626 list_for_each(pos, &pfm_buffer_fmt_list) {
5627 entry = list_entry(pos, pfm_buffer_fmt_t, fmt_list);
5628 seq_printf(m, "format : %16phD %s\n",
5629 entry->fmt_uuid, entry->fmt_name);
5630 }
5631 spin_unlock(&pfm_buffer_fmt_lock);
5632
5633}
5634
5635static int
5636pfm_proc_show(struct seq_file *m, void *v)
5637{
5638 unsigned long psr;
5639 unsigned int i;
5640 int cpu;
5641
5642 if (v == PFM_PROC_SHOW_HEADER) {
5643 pfm_proc_show_header(m);
5644 return 0;
5645 }
5646
5647
5648
5649 cpu = (long)v - 1;
5650 seq_printf(m,
5651 "CPU%-2d overflow intrs : %lu\n"
5652 "CPU%-2d overflow cycles : %lu\n"
5653 "CPU%-2d overflow min : %lu\n"
5654 "CPU%-2d overflow max : %lu\n"
5655 "CPU%-2d smpl handler calls : %lu\n"
5656 "CPU%-2d smpl handler cycles : %lu\n"
5657 "CPU%-2d spurious intrs : %lu\n"
5658 "CPU%-2d replay intrs : %lu\n"
5659 "CPU%-2d syst_wide : %d\n"
5660 "CPU%-2d dcr_pp : %d\n"
5661 "CPU%-2d exclude idle : %d\n"
5662 "CPU%-2d owner : %d\n"
5663 "CPU%-2d context : %p\n"
5664 "CPU%-2d activations : %lu\n",
5665 cpu, pfm_stats[cpu].pfm_ovfl_intr_count,
5666 cpu, pfm_stats[cpu].pfm_ovfl_intr_cycles,
5667 cpu, pfm_stats[cpu].pfm_ovfl_intr_cycles_min,
5668 cpu, pfm_stats[cpu].pfm_ovfl_intr_cycles_max,
5669 cpu, pfm_stats[cpu].pfm_smpl_handler_calls,
5670 cpu, pfm_stats[cpu].pfm_smpl_handler_cycles,
5671 cpu, pfm_stats[cpu].pfm_spurious_ovfl_intr_count,
5672 cpu, pfm_stats[cpu].pfm_replay_ovfl_intr_count,
5673 cpu, pfm_get_cpu_data(pfm_syst_info, cpu) & PFM_CPUINFO_SYST_WIDE ? 1 : 0,
5674 cpu, pfm_get_cpu_data(pfm_syst_info, cpu) & PFM_CPUINFO_DCR_PP ? 1 : 0,
5675 cpu, pfm_get_cpu_data(pfm_syst_info, cpu) & PFM_CPUINFO_EXCL_IDLE ? 1 : 0,
5676 cpu, pfm_get_cpu_data(pmu_owner, cpu) ? pfm_get_cpu_data(pmu_owner, cpu)->pid: -1,
5677 cpu, pfm_get_cpu_data(pmu_ctx, cpu),
5678 cpu, pfm_get_cpu_data(pmu_activation_number, cpu));
5679
5680 if (num_online_cpus() == 1 && pfm_sysctl.debug > 0) {
5681
5682 psr = pfm_get_psr();
5683
5684 ia64_srlz_d();
5685
5686 seq_printf(m,
5687 "CPU%-2d psr : 0x%lx\n"
5688 "CPU%-2d pmc0 : 0x%lx\n",
5689 cpu, psr,
5690 cpu, ia64_get_pmc(0));
5691
5692 for (i=0; PMC_IS_LAST(i) == 0; i++) {
5693 if (PMC_IS_COUNTING(i) == 0) continue;
5694 seq_printf(m,
5695 "CPU%-2d pmc%u : 0x%lx\n"
5696 "CPU%-2d pmd%u : 0x%lx\n",
5697 cpu, i, ia64_get_pmc(i),
5698 cpu, i, ia64_get_pmd(i));
5699 }
5700 }
5701 return 0;
5702}
5703
5704const struct seq_operations pfm_seq_ops = {
5705 .start = pfm_proc_start,
5706 .next = pfm_proc_next,
5707 .stop = pfm_proc_stop,
5708 .show = pfm_proc_show
5709};
5710
5711static int
5712pfm_proc_open(struct inode *inode, struct file *file)
5713{
5714 return seq_open(file, &pfm_seq_ops);
5715}
5716
5717
5718
5719
5720
5721
5722
5723
5724void
5725pfm_syst_wide_update_task(struct task_struct *task, unsigned long info, int is_ctxswin)
5726{
5727 struct pt_regs *regs;
5728 unsigned long dcr;
5729 unsigned long dcr_pp;
5730
5731 dcr_pp = info & PFM_CPUINFO_DCR_PP ? 1 : 0;
5732
5733
5734
5735
5736
5737 if ((info & PFM_CPUINFO_EXCL_IDLE) == 0 || task->pid) {
5738 regs = task_pt_regs(task);
5739 ia64_psr(regs)->pp = is_ctxswin ? dcr_pp : 0;
5740 return;
5741 }
5742
5743
5744
5745 if (dcr_pp) {
5746 dcr = ia64_getreg(_IA64_REG_CR_DCR);
5747
5748
5749
5750 if (is_ctxswin) {
5751
5752 ia64_setreg(_IA64_REG_CR_DCR, dcr & ~IA64_DCR_PP);
5753 pfm_clear_psr_pp();
5754 ia64_srlz_i();
5755 return;
5756 }
5757
5758
5759
5760
5761
5762
5763
5764 ia64_setreg(_IA64_REG_CR_DCR, dcr |IA64_DCR_PP);
5765 pfm_set_psr_pp();
5766 ia64_srlz_i();
5767 }
5768}
5769
5770#ifdef CONFIG_SMP
5771
5772static void
5773pfm_force_cleanup(pfm_context_t *ctx, struct pt_regs *regs)
5774{
5775 struct task_struct *task = ctx->ctx_task;
5776
5777 ia64_psr(regs)->up = 0;
5778 ia64_psr(regs)->sp = 1;
5779
5780 if (GET_PMU_OWNER() == task) {
5781 DPRINT(("cleared ownership for [%d]\n",
5782 task_pid_nr(ctx->ctx_task)));
5783 SET_PMU_OWNER(NULL, NULL);
5784 }
5785
5786
5787
5788
5789 PFM_SET_WORK_PENDING(task, 0);
5790
5791 task->thread.pfm_context = NULL;
5792 task->thread.flags &= ~IA64_THREAD_PM_VALID;
5793
5794 DPRINT(("force cleanup for [%d]\n", task_pid_nr(task)));
5795}
5796
5797
5798
5799
5800
5801void
5802pfm_save_regs(struct task_struct *task)
5803{
5804 pfm_context_t *ctx;
5805 unsigned long flags;
5806 u64 psr;
5807
5808
5809 ctx = PFM_GET_CTX(task);
5810 if (ctx == NULL) return;
5811
5812
5813
5814
5815
5816
5817 flags = pfm_protect_ctx_ctxsw(ctx);
5818
5819 if (ctx->ctx_state == PFM_CTX_ZOMBIE) {
5820 struct pt_regs *regs = task_pt_regs(task);
5821
5822 pfm_clear_psr_up();
5823
5824 pfm_force_cleanup(ctx, regs);
5825
5826 BUG_ON(ctx->ctx_smpl_hdr);
5827
5828 pfm_unprotect_ctx_ctxsw(ctx, flags);
5829
5830 pfm_context_free(ctx);
5831 return;
5832 }
5833
5834
5835
5836
5837 ia64_srlz_d();
5838 psr = pfm_get_psr();
5839
5840 BUG_ON(psr & (IA64_PSR_I));
5841
5842
5843
5844
5845
5846
5847
5848
5849 pfm_clear_psr_up();
5850
5851
5852
5853
5854 ctx->ctx_saved_psr_up = psr & IA64_PSR_UP;
5855
5856
5857
5858
5859
5860
5861 SET_PMU_OWNER(NULL, NULL);
5862
5863
5864
5865
5866
5867
5868 pfm_save_pmds(ctx->th_pmds, ctx->ctx_used_pmds[0]);
5869
5870
5871
5872
5873
5874
5875 ctx->th_pmcs[0] = ia64_get_pmc(0);
5876
5877
5878
5879
5880 if (ctx->th_pmcs[0] & ~0x1UL) pfm_unfreeze_pmu();
5881
5882
5883
5884
5885
5886 pfm_unprotect_ctx_ctxsw(ctx, flags);
5887}
5888
5889#else
5890void
5891pfm_save_regs(struct task_struct *task)
5892{
5893 pfm_context_t *ctx;
5894 u64 psr;
5895
5896 ctx = PFM_GET_CTX(task);
5897 if (ctx == NULL) return;
5898
5899
5900
5901
5902 psr = pfm_get_psr();
5903
5904 BUG_ON(psr & (IA64_PSR_I));
5905
5906
5907
5908
5909
5910
5911
5912
5913 pfm_clear_psr_up();
5914
5915
5916
5917
5918 ctx->ctx_saved_psr_up = psr & IA64_PSR_UP;
5919}
5920
5921static void
5922pfm_lazy_save_regs (struct task_struct *task)
5923{
5924 pfm_context_t *ctx;
5925 unsigned long flags;
5926
5927 { u64 psr = pfm_get_psr();
5928 BUG_ON(psr & IA64_PSR_UP);
5929 }
5930
5931 ctx = PFM_GET_CTX(task);
5932
5933
5934
5935
5936
5937
5938
5939
5940
5941
5942 PROTECT_CTX(ctx,flags);
5943
5944
5945
5946
5947
5948
5949
5950
5951 SET_PMU_OWNER(NULL, NULL);
5952
5953
5954
5955
5956 pfm_save_pmds(ctx->th_pmds, ctx->ctx_used_pmds[0]);
5957
5958
5959
5960
5961
5962
5963 ctx->th_pmcs[0] = ia64_get_pmc(0);
5964
5965
5966
5967
5968 if (ctx->th_pmcs[0] & ~0x1UL) pfm_unfreeze_pmu();
5969
5970
5971
5972
5973
5974
5975 UNPROTECT_CTX(ctx,flags);
5976}
5977#endif
5978
5979#ifdef CONFIG_SMP
5980
5981
5982
5983void
5984pfm_load_regs (struct task_struct *task)
5985{
5986 pfm_context_t *ctx;
5987 unsigned long pmc_mask = 0UL, pmd_mask = 0UL;
5988 unsigned long flags;
5989 u64 psr, psr_up;
5990 int need_irq_resend;
5991
5992 ctx = PFM_GET_CTX(task);
5993 if (unlikely(ctx == NULL)) return;
5994
5995 BUG_ON(GET_PMU_OWNER());
5996
5997
5998
5999
6000 if (unlikely((task->thread.flags & IA64_THREAD_PM_VALID) == 0)) return;
6001
6002
6003
6004
6005
6006
6007 flags = pfm_protect_ctx_ctxsw(ctx);
6008 psr = pfm_get_psr();
6009
6010 need_irq_resend = pmu_conf->flags & PFM_PMU_IRQ_RESEND;
6011
6012 BUG_ON(psr & (IA64_PSR_UP|IA64_PSR_PP));
6013 BUG_ON(psr & IA64_PSR_I);
6014
6015 if (unlikely(ctx->ctx_state == PFM_CTX_ZOMBIE)) {
6016 struct pt_regs *regs = task_pt_regs(task);
6017
6018 BUG_ON(ctx->ctx_smpl_hdr);
6019
6020 pfm_force_cleanup(ctx, regs);
6021
6022 pfm_unprotect_ctx_ctxsw(ctx, flags);
6023
6024
6025
6026
6027 pfm_context_free(ctx);
6028
6029 return;
6030 }
6031
6032
6033
6034
6035
6036 if (ctx->ctx_fl_using_dbreg) {
6037 pfm_restore_ibrs(ctx->ctx_ibrs, pmu_conf->num_ibrs);
6038 pfm_restore_dbrs(ctx->ctx_dbrs, pmu_conf->num_dbrs);
6039 }
6040
6041
6042
6043 psr_up = ctx->ctx_saved_psr_up;
6044
6045
6046
6047
6048
6049 if (GET_LAST_CPU(ctx) == smp_processor_id() && ctx->ctx_last_activation == GET_ACTIVATION()) {
6050
6051
6052
6053
6054 pmc_mask = ctx->ctx_reload_pmcs[0];
6055 pmd_mask = ctx->ctx_reload_pmds[0];
6056
6057 } else {
6058
6059
6060
6061
6062
6063
6064 pmd_mask = pfm_sysctl.fastctxsw ? ctx->ctx_used_pmds[0] : ctx->ctx_all_pmds[0];
6065
6066
6067
6068
6069
6070
6071
6072
6073 pmc_mask = ctx->ctx_all_pmcs[0];
6074 }
6075
6076
6077
6078
6079
6080
6081
6082 if (pmd_mask) pfm_restore_pmds(ctx->th_pmds, pmd_mask);
6083 if (pmc_mask) pfm_restore_pmcs(ctx->th_pmcs, pmc_mask);
6084
6085
6086
6087
6088
6089 if (unlikely(PMC0_HAS_OVFL(ctx->th_pmcs[0]))) {
6090
6091
6092
6093
6094 ia64_set_pmc(0, ctx->th_pmcs[0]);
6095 ia64_srlz_d();
6096 ctx->th_pmcs[0] = 0UL;
6097
6098
6099
6100
6101 if (need_irq_resend) ia64_resend_irq(IA64_PERFMON_VECTOR);
6102
6103 pfm_stats[smp_processor_id()].pfm_replay_ovfl_intr_count++;
6104 }
6105
6106
6107
6108
6109 ctx->ctx_reload_pmcs[0] = 0UL;
6110 ctx->ctx_reload_pmds[0] = 0UL;
6111
6112 SET_LAST_CPU(ctx, smp_processor_id());
6113
6114
6115
6116
6117 INC_ACTIVATION();
6118
6119
6120
6121 SET_ACTIVATION(ctx);
6122
6123
6124
6125
6126 SET_PMU_OWNER(task, ctx);
6127
6128
6129
6130
6131
6132
6133
6134 if (likely(psr_up)) pfm_set_psr_up();
6135
6136
6137
6138
6139 pfm_unprotect_ctx_ctxsw(ctx, flags);
6140}
6141#else
6142
6143
6144
6145
6146void
6147pfm_load_regs (struct task_struct *task)
6148{
6149 pfm_context_t *ctx;
6150 struct task_struct *owner;
6151 unsigned long pmd_mask, pmc_mask;
6152 u64 psr, psr_up;
6153 int need_irq_resend;
6154
6155 owner = GET_PMU_OWNER();
6156 ctx = PFM_GET_CTX(task);
6157 psr = pfm_get_psr();
6158
6159 BUG_ON(psr & (IA64_PSR_UP|IA64_PSR_PP));
6160 BUG_ON(psr & IA64_PSR_I);
6161
6162
6163
6164
6165
6166
6167
6168
6169
6170 if (ctx->ctx_fl_using_dbreg) {
6171 pfm_restore_ibrs(ctx->ctx_ibrs, pmu_conf->num_ibrs);
6172 pfm_restore_dbrs(ctx->ctx_dbrs, pmu_conf->num_dbrs);
6173 }
6174
6175
6176
6177
6178 psr_up = ctx->ctx_saved_psr_up;
6179 need_irq_resend = pmu_conf->flags & PFM_PMU_IRQ_RESEND;
6180
6181
6182
6183
6184
6185
6186
6187
6188
6189 if (likely(owner == task)) {
6190 if (likely(psr_up)) pfm_set_psr_up();
6191 return;
6192 }
6193
6194
6195
6196
6197
6198
6199
6200 if (owner) pfm_lazy_save_regs(owner);
6201
6202
6203
6204
6205
6206
6207
6208 pmd_mask = pfm_sysctl.fastctxsw ? ctx->ctx_used_pmds[0] : ctx->ctx_all_pmds[0];
6209
6210
6211
6212
6213
6214
6215
6216
6217 pmc_mask = ctx->ctx_all_pmcs[0];
6218
6219 pfm_restore_pmds(ctx->th_pmds, pmd_mask);
6220 pfm_restore_pmcs(ctx->th_pmcs, pmc_mask);
6221
6222
6223
6224
6225
6226 if (unlikely(PMC0_HAS_OVFL(ctx->th_pmcs[0]))) {
6227
6228
6229
6230
6231 ia64_set_pmc(0, ctx->th_pmcs[0]);
6232 ia64_srlz_d();
6233
6234 ctx->th_pmcs[0] = 0UL;
6235
6236
6237
6238
6239 if (need_irq_resend) ia64_resend_irq(IA64_PERFMON_VECTOR);
6240
6241 pfm_stats[smp_processor_id()].pfm_replay_ovfl_intr_count++;
6242 }
6243
6244
6245
6246
6247 SET_PMU_OWNER(task, ctx);
6248
6249
6250
6251
6252
6253
6254
6255 if (likely(psr_up)) pfm_set_psr_up();
6256}
6257#endif
6258
6259
6260
6261
6262static void
6263pfm_flush_pmds(struct task_struct *task, pfm_context_t *ctx)
6264{
6265 u64 pmc0;
6266 unsigned long mask2, val, pmd_val, ovfl_val;
6267 int i, can_access_pmu = 0;
6268 int is_self;
6269
6270
6271
6272
6273
6274 is_self = ctx->ctx_task == task ? 1 : 0;
6275
6276
6277
6278
6279
6280
6281
6282
6283 can_access_pmu = (GET_PMU_OWNER() == task) || (ctx->ctx_fl_system && ctx->ctx_cpu == smp_processor_id());
6284 if (can_access_pmu) {
6285
6286
6287
6288
6289
6290
6291
6292
6293 SET_PMU_OWNER(NULL, NULL);
6294 DPRINT(("releasing ownership\n"));
6295
6296
6297
6298
6299
6300
6301 ia64_srlz_d();
6302 pmc0 = ia64_get_pmc(0);
6303
6304
6305
6306
6307 pfm_unfreeze_pmu();
6308 } else {
6309 pmc0 = ctx->th_pmcs[0];
6310
6311
6312
6313 ctx->th_pmcs[0] = 0;
6314 }
6315 ovfl_val = pmu_conf->ovfl_val;
6316
6317
6318
6319
6320
6321
6322 mask2 = ctx->ctx_used_pmds[0];
6323
6324 DPRINT(("is_self=%d ovfl_val=0x%lx mask2=0x%lx\n", is_self, ovfl_val, mask2));
6325
6326 for (i = 0; mask2; i++, mask2>>=1) {
6327
6328
6329 if ((mask2 & 0x1) == 0) continue;
6330
6331
6332
6333
6334 val = pmd_val = can_access_pmu ? ia64_get_pmd(i) : ctx->th_pmds[i];
6335
6336 if (PMD_IS_COUNTING(i)) {
6337 DPRINT(("[%d] pmd[%d] ctx_pmd=0x%lx hw_pmd=0x%lx\n",
6338 task_pid_nr(task),
6339 i,
6340 ctx->ctx_pmds[i].val,
6341 val & ovfl_val));
6342
6343
6344
6345
6346 val = ctx->ctx_pmds[i].val + (val & ovfl_val);
6347
6348
6349
6350
6351
6352
6353 pmd_val = 0UL;
6354
6355
6356
6357
6358 if (pmc0 & (1UL << i)) {
6359 val += 1 + ovfl_val;
6360 DPRINT(("[%d] pmd[%d] overflowed\n", task_pid_nr(task), i));
6361 }
6362 }
6363
6364 DPRINT(("[%d] ctx_pmd[%d]=0x%lx pmd_val=0x%lx\n", task_pid_nr(task), i, val, pmd_val));
6365
6366 if (is_self) ctx->th_pmds[i] = pmd_val;
6367
6368 ctx->ctx_pmds[i].val = val;
6369 }
6370}
6371
6372static struct irqaction perfmon_irqaction = {
6373 .handler = pfm_interrupt_handler,
6374 .name = "perfmon"
6375};
6376
6377static void
6378pfm_alt_save_pmu_state(void *data)
6379{
6380 struct pt_regs *regs;
6381
6382 regs = task_pt_regs(current);
6383
6384 DPRINT(("called\n"));
6385
6386
6387
6388
6389
6390 pfm_clear_psr_up();
6391 pfm_clear_psr_pp();
6392 ia64_psr(regs)->pp = 0;
6393
6394
6395
6396
6397
6398 pfm_freeze_pmu();
6399
6400 ia64_srlz_d();
6401}
6402
6403void
6404pfm_alt_restore_pmu_state(void *data)
6405{
6406 struct pt_regs *regs;
6407
6408 regs = task_pt_regs(current);
6409
6410 DPRINT(("called\n"));
6411
6412
6413
6414
6415
6416 pfm_clear_psr_up();
6417 pfm_clear_psr_pp();
6418 ia64_psr(regs)->pp = 0;
6419
6420
6421
6422
6423 pfm_unfreeze_pmu();
6424
6425 ia64_srlz_d();
6426}
6427
6428int
6429pfm_install_alt_pmu_interrupt(pfm_intr_handler_desc_t *hdl)
6430{
6431 int ret, i;
6432 int reserve_cpu;
6433
6434
6435 if (hdl == NULL || hdl->handler == NULL) return -EINVAL;
6436
6437
6438 if (pfm_alt_intr_handler) return -EBUSY;
6439
6440
6441 if (!spin_trylock(&pfm_alt_install_check)) {
6442 return -EBUSY;
6443 }
6444
6445
6446 for_each_online_cpu(reserve_cpu) {
6447 ret = pfm_reserve_session(NULL, 1, reserve_cpu);
6448 if (ret) goto cleanup_reserve;
6449 }
6450
6451
6452 ret = on_each_cpu(pfm_alt_save_pmu_state, NULL, 1);
6453 if (ret) {
6454 DPRINT(("on_each_cpu() failed: %d\n", ret));
6455 goto cleanup_reserve;
6456 }
6457
6458
6459 pfm_alt_intr_handler = hdl;
6460
6461 spin_unlock(&pfm_alt_install_check);
6462
6463 return 0;
6464
6465cleanup_reserve:
6466 for_each_online_cpu(i) {
6467
6468 if (i >= reserve_cpu) break;
6469
6470 pfm_unreserve_session(NULL, 1, i);
6471 }
6472
6473 spin_unlock(&pfm_alt_install_check);
6474
6475 return ret;
6476}
6477EXPORT_SYMBOL_GPL(pfm_install_alt_pmu_interrupt);
6478
6479int
6480pfm_remove_alt_pmu_interrupt(pfm_intr_handler_desc_t *hdl)
6481{
6482 int i;
6483 int ret;
6484
6485 if (hdl == NULL) return -EINVAL;
6486
6487
6488 if (pfm_alt_intr_handler != hdl) return -EINVAL;
6489
6490
6491 if (!spin_trylock(&pfm_alt_install_check)) {
6492 return -EBUSY;
6493 }
6494
6495 pfm_alt_intr_handler = NULL;
6496
6497 ret = on_each_cpu(pfm_alt_restore_pmu_state, NULL, 1);
6498 if (ret) {
6499 DPRINT(("on_each_cpu() failed: %d\n", ret));
6500 }
6501
6502 for_each_online_cpu(i) {
6503 pfm_unreserve_session(NULL, 1, i);
6504 }
6505
6506 spin_unlock(&pfm_alt_install_check);
6507
6508 return 0;
6509}
6510EXPORT_SYMBOL_GPL(pfm_remove_alt_pmu_interrupt);
6511
6512
6513
6514
6515static int init_pfm_fs(void);
6516
6517static int __init
6518pfm_probe_pmu(void)
6519{
6520 pmu_config_t **p;
6521 int family;
6522
6523 family = local_cpu_data->family;
6524 p = pmu_confs;
6525
6526 while(*p) {
6527 if ((*p)->probe) {
6528 if ((*p)->probe() == 0) goto found;
6529 } else if ((*p)->pmu_family == family || (*p)->pmu_family == 0xff) {
6530 goto found;
6531 }
6532 p++;
6533 }
6534 return -1;
6535found:
6536 pmu_conf = *p;
6537 return 0;
6538}
6539
6540static const struct file_operations pfm_proc_fops = {
6541 .open = pfm_proc_open,
6542 .read = seq_read,
6543 .llseek = seq_lseek,
6544 .release = seq_release,
6545};
6546
6547int __init
6548pfm_init(void)
6549{
6550 unsigned int n, n_counters, i;
6551
6552 printk("perfmon: version %u.%u IRQ %u\n",
6553 PFM_VERSION_MAJ,
6554 PFM_VERSION_MIN,
6555 IA64_PERFMON_VECTOR);
6556
6557 if (pfm_probe_pmu()) {
6558 printk(KERN_INFO "perfmon: disabled, there is no support for processor family %d\n",
6559 local_cpu_data->family);
6560 return -ENODEV;
6561 }
6562
6563
6564
6565
6566
6567 n = 0;
6568 for (i=0; PMC_IS_LAST(i) == 0; i++) {
6569 if (PMC_IS_IMPL(i) == 0) continue;
6570 pmu_conf->impl_pmcs[i>>6] |= 1UL << (i&63);
6571 n++;
6572 }
6573 pmu_conf->num_pmcs = n;
6574
6575 n = 0; n_counters = 0;
6576 for (i=0; PMD_IS_LAST(i) == 0; i++) {
6577 if (PMD_IS_IMPL(i) == 0) continue;
6578 pmu_conf->impl_pmds[i>>6] |= 1UL << (i&63);
6579 n++;
6580 if (PMD_IS_COUNTING(i)) n_counters++;
6581 }
6582 pmu_conf->num_pmds = n;
6583 pmu_conf->num_counters = n_counters;
6584
6585
6586
6587
6588 if (pmu_conf->use_rr_dbregs) {
6589 if (pmu_conf->num_ibrs > IA64_NUM_DBG_REGS) {
6590 printk(KERN_INFO "perfmon: unsupported number of code debug registers (%u)\n", pmu_conf->num_ibrs);
6591 pmu_conf = NULL;
6592 return -1;
6593 }
6594 if (pmu_conf->num_dbrs > IA64_NUM_DBG_REGS) {
6595 printk(KERN_INFO "perfmon: unsupported number of data debug registers (%u)\n", pmu_conf->num_ibrs);
6596 pmu_conf = NULL;
6597 return -1;
6598 }
6599 }
6600
6601 printk("perfmon: %s PMU detected, %u PMCs, %u PMDs, %u counters (%lu bits)\n",
6602 pmu_conf->pmu_name,
6603 pmu_conf->num_pmcs,
6604 pmu_conf->num_pmds,
6605 pmu_conf->num_counters,
6606 ffz(pmu_conf->ovfl_val));
6607
6608
6609 if (pmu_conf->num_pmds >= PFM_NUM_PMD_REGS || pmu_conf->num_pmcs >= PFM_NUM_PMC_REGS) {
6610 printk(KERN_ERR "perfmon: not enough pmc/pmd, perfmon disabled\n");
6611 pmu_conf = NULL;
6612 return -1;
6613 }
6614
6615
6616
6617
6618 perfmon_dir = proc_create("perfmon", S_IRUGO, NULL, &pfm_proc_fops);
6619 if (perfmon_dir == NULL) {
6620 printk(KERN_ERR "perfmon: cannot create /proc entry, perfmon disabled\n");
6621 pmu_conf = NULL;
6622 return -1;
6623 }
6624
6625
6626
6627
6628 pfm_sysctl_header = register_sysctl_table(pfm_sysctl_root);
6629
6630
6631
6632
6633 spin_lock_init(&pfm_sessions.pfs_lock);
6634 spin_lock_init(&pfm_buffer_fmt_lock);
6635
6636 init_pfm_fs();
6637
6638 for(i=0; i < NR_CPUS; i++) pfm_stats[i].pfm_ovfl_intr_cycles_min = ~0UL;
6639
6640 return 0;
6641}
6642
6643__initcall(pfm_init);
6644
6645
6646
6647
6648void
6649pfm_init_percpu (void)
6650{
6651 static int first_time=1;
6652
6653
6654
6655
6656 pfm_clear_psr_pp();
6657 pfm_clear_psr_up();
6658
6659
6660
6661
6662 pfm_unfreeze_pmu();
6663
6664 if (first_time) {
6665 register_percpu_irq(IA64_PERFMON_VECTOR, &perfmon_irqaction);
6666 first_time=0;
6667 }
6668
6669 ia64_setreg(_IA64_REG_CR_PMV, IA64_PERFMON_VECTOR);
6670 ia64_srlz_d();
6671}
6672
6673
6674
6675
6676void
6677dump_pmu_state(const char *from)
6678{
6679 struct task_struct *task;
6680 struct pt_regs *regs;
6681 pfm_context_t *ctx;
6682 unsigned long psr, dcr, info, flags;
6683 int i, this_cpu;
6684
6685 local_irq_save(flags);
6686
6687 this_cpu = smp_processor_id();
6688 regs = task_pt_regs(current);
6689 info = PFM_CPUINFO_GET();
6690 dcr = ia64_getreg(_IA64_REG_CR_DCR);
6691
6692 if (info == 0 && ia64_psr(regs)->pp == 0 && (dcr & IA64_DCR_PP) == 0) {
6693 local_irq_restore(flags);
6694 return;
6695 }
6696
6697 printk("CPU%d from %s() current [%d] iip=0x%lx %s\n",
6698 this_cpu,
6699 from,
6700 task_pid_nr(current),
6701 regs->cr_iip,
6702 current->comm);
6703
6704 task = GET_PMU_OWNER();
6705 ctx = GET_PMU_CTX();
6706
6707 printk("->CPU%d owner [%d] ctx=%p\n", this_cpu, task ? task_pid_nr(task) : -1, ctx);
6708
6709 psr = pfm_get_psr();
6710
6711 printk("->CPU%d pmc0=0x%lx psr.pp=%d psr.up=%d dcr.pp=%d syst_info=0x%lx user_psr.up=%d user_psr.pp=%d\n",
6712 this_cpu,
6713 ia64_get_pmc(0),
6714 psr & IA64_PSR_PP ? 1 : 0,
6715 psr & IA64_PSR_UP ? 1 : 0,
6716 dcr & IA64_DCR_PP ? 1 : 0,
6717 info,
6718 ia64_psr(regs)->up,
6719 ia64_psr(regs)->pp);
6720
6721 ia64_psr(regs)->up = 0;
6722 ia64_psr(regs)->pp = 0;
6723
6724 for (i=1; PMC_IS_LAST(i) == 0; i++) {
6725 if (PMC_IS_IMPL(i) == 0) continue;
6726 printk("->CPU%d pmc[%d]=0x%lx thread_pmc[%d]=0x%lx\n", this_cpu, i, ia64_get_pmc(i), i, ctx->th_pmcs[i]);
6727 }
6728
6729 for (i=1; PMD_IS_LAST(i) == 0; i++) {
6730 if (PMD_IS_IMPL(i) == 0) continue;
6731 printk("->CPU%d pmd[%d]=0x%lx thread_pmd[%d]=0x%lx\n", this_cpu, i, ia64_get_pmd(i), i, ctx->th_pmds[i]);
6732 }
6733
6734 if (ctx) {
6735 printk("->CPU%d ctx_state=%d vaddr=%p addr=%p fd=%d ctx_task=[%d] saved_psr_up=0x%lx\n",
6736 this_cpu,
6737 ctx->ctx_state,
6738 ctx->ctx_smpl_vaddr,
6739 ctx->ctx_smpl_hdr,
6740 ctx->ctx_msgq_head,
6741 ctx->ctx_msgq_tail,
6742 ctx->ctx_saved_psr_up);
6743 }
6744 local_irq_restore(flags);
6745}
6746
6747
6748
6749
6750void
6751pfm_inherit(struct task_struct *task, struct pt_regs *regs)
6752{
6753 struct thread_struct *thread;
6754
6755 DPRINT(("perfmon: pfm_inherit clearing state for [%d]\n", task_pid_nr(task)));
6756
6757 thread = &task->thread;
6758
6759
6760
6761
6762 thread->pfm_context = NULL;
6763
6764 PFM_SET_WORK_PENDING(task, 0);
6765
6766
6767
6768
6769}
6770#else
6771asmlinkage long
6772sys_perfmonctl (int fd, int cmd, void *arg, int count)
6773{
6774 return -ENOSYS;
6775}
6776#endif
6777