1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43enum qlock_stats {
44 qstat_pv_hash_hops,
45 qstat_pv_kick_unlock,
46 qstat_pv_kick_wake,
47 qstat_pv_latency_kick,
48 qstat_pv_latency_wake,
49 qstat_pv_lock_slowpath,
50 qstat_pv_lock_stealing,
51 qstat_pv_spurious_wakeup,
52 qstat_pv_wait_again,
53 qstat_pv_wait_early,
54 qstat_pv_wait_head,
55 qstat_pv_wait_node,
56 qstat_num,
57 qstat_reset_cnts = qstat_num,
58};
59
60#ifdef CONFIG_QUEUED_LOCK_STAT
61
62
63
64#include <linux/debugfs.h>
65#include <linux/sched.h>
66#include <linux/fs.h>
67
68static const char * const qstat_names[qstat_num + 1] = {
69 [qstat_pv_hash_hops] = "pv_hash_hops",
70 [qstat_pv_kick_unlock] = "pv_kick_unlock",
71 [qstat_pv_kick_wake] = "pv_kick_wake",
72 [qstat_pv_spurious_wakeup] = "pv_spurious_wakeup",
73 [qstat_pv_latency_kick] = "pv_latency_kick",
74 [qstat_pv_latency_wake] = "pv_latency_wake",
75 [qstat_pv_lock_slowpath] = "pv_lock_slowpath",
76 [qstat_pv_lock_stealing] = "pv_lock_stealing",
77 [qstat_pv_wait_again] = "pv_wait_again",
78 [qstat_pv_wait_early] = "pv_wait_early",
79 [qstat_pv_wait_head] = "pv_wait_head",
80 [qstat_pv_wait_node] = "pv_wait_node",
81 [qstat_reset_cnts] = "reset_counters",
82};
83
84
85
86
87static DEFINE_PER_CPU(unsigned long, qstats[qstat_num]);
88static DEFINE_PER_CPU(u64, pv_kick_time);
89
90
91
92
93
94
95
96
97
98
99
100
101static ssize_t qstat_read(struct file *file, char __user *user_buf,
102 size_t count, loff_t *ppos)
103{
104 char buf[64];
105 int cpu, counter, len;
106 u64 stat = 0, kicks = 0;
107
108
109
110
111 if (!file->f_inode) {
112 WARN_ON_ONCE(1);
113 return -EBADF;
114 }
115 counter = (long)(file->f_inode->i_private);
116
117 if (counter >= qstat_num)
118 return -EBADF;
119
120 for_each_possible_cpu(cpu) {
121 stat += per_cpu(qstats[counter], cpu);
122
123
124
125 switch (counter) {
126
127 case qstat_pv_latency_kick:
128 case qstat_pv_hash_hops:
129 kicks += per_cpu(qstats[qstat_pv_kick_unlock], cpu);
130 break;
131
132 case qstat_pv_latency_wake:
133 kicks += per_cpu(qstats[qstat_pv_kick_wake], cpu);
134 break;
135 }
136 }
137
138 if (counter == qstat_pv_hash_hops) {
139 u64 frac = 0;
140
141 if (kicks) {
142 frac = 100ULL * do_div(stat, kicks);
143 frac = DIV_ROUND_CLOSEST_ULL(frac, kicks);
144 }
145
146
147
148
149 len = snprintf(buf, sizeof(buf) - 1, "%llu.%02llu\n", stat, frac);
150 } else {
151
152
153
154 if ((counter == qstat_pv_latency_kick) ||
155 (counter == qstat_pv_latency_wake)) {
156 stat = 0;
157 if (kicks)
158 stat = DIV_ROUND_CLOSEST_ULL(stat, kicks);
159 }
160 len = snprintf(buf, sizeof(buf) - 1, "%llu\n", stat);
161 }
162
163 return simple_read_from_buffer(user_buf, count, ppos, buf, len);
164}
165
166
167
168
169
170
171
172
173static ssize_t qstat_write(struct file *file, const char __user *user_buf,
174 size_t count, loff_t *ppos)
175{
176 int cpu;
177
178
179
180
181 if (!file->f_inode) {
182 WARN_ON_ONCE(1);
183 return -EBADF;
184 }
185 if ((long)(file->f_inode->i_private) != qstat_reset_cnts)
186 return count;
187
188 for_each_possible_cpu(cpu) {
189 int i;
190 unsigned long *ptr = per_cpu_ptr(qstats, cpu);
191
192 for (i = 0 ; i < qstat_num; i++)
193 WRITE_ONCE(ptr[i], 0);
194 for (i = 0 ; i < qstat_num; i++)
195 WRITE_ONCE(ptr[i], 0);
196 }
197 return count;
198}
199
200
201
202
203static const struct file_operations fops_qstat = {
204 .read = qstat_read,
205 .write = qstat_write,
206 .llseek = default_llseek,
207};
208
209
210
211
212static int __init init_qspinlock_stat(void)
213{
214 struct dentry *d_qstat = debugfs_create_dir("qlockstat", NULL);
215 int i;
216
217 if (!d_qstat) {
218 pr_warn("Could not create 'qlockstat' debugfs directory\n");
219 return 0;
220 }
221
222
223
224
225
226
227
228
229 for (i = 0; i < qstat_num; i++)
230 debugfs_create_file(qstat_names[i], 0400, d_qstat,
231 (void *)(long)i, &fops_qstat);
232
233 debugfs_create_file(qstat_names[qstat_reset_cnts], 0200, d_qstat,
234 (void *)(long)qstat_reset_cnts, &fops_qstat);
235 return 0;
236}
237fs_initcall(init_qspinlock_stat);
238
239
240
241
242static inline void qstat_inc(enum qlock_stats stat, bool cond)
243{
244 if (cond)
245 this_cpu_inc(qstats[stat]);
246}
247
248
249
250
251static inline void qstat_hop(int hopcnt)
252{
253 this_cpu_add(qstats[qstat_pv_hash_hops], hopcnt);
254}
255
256
257
258
259static inline void __pv_kick(int cpu)
260{
261 u64 start = sched_clock();
262
263 per_cpu(pv_kick_time, cpu) = start;
264 pv_kick(cpu);
265 this_cpu_add(qstats[qstat_pv_latency_kick], sched_clock() - start);
266}
267
268
269
270
271static inline void __pv_wait(u8 *ptr, u8 val)
272{
273 u64 *pkick_time = this_cpu_ptr(&pv_kick_time);
274
275 *pkick_time = 0;
276 pv_wait(ptr, val);
277 if (*pkick_time) {
278 this_cpu_add(qstats[qstat_pv_latency_wake],
279 sched_clock() - *pkick_time);
280 qstat_inc(qstat_pv_kick_wake, true);
281 }
282}
283
284#define pv_kick(c) __pv_kick(c)
285#define pv_wait(p, v) __pv_wait(p, v)
286
287#else
288
289static inline void qstat_inc(enum qlock_stats stat, bool cond) { }
290static inline void qstat_hop(int hopcnt) { }
291
292#endif
293