1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44enum qlock_stats {
45 qstat_pv_hash_hops,
46 qstat_pv_kick_unlock,
47 qstat_pv_kick_wake,
48 qstat_pv_latency_kick,
49 qstat_pv_latency_wake,
50 qstat_pv_lock_stealing,
51 qstat_pv_spurious_wakeup,
52 qstat_pv_wait_again,
53 qstat_pv_wait_early,
54 qstat_pv_wait_head,
55 qstat_pv_wait_node,
56 qstat_lock_pending,
57 qstat_lock_slowpath,
58 qstat_num,
59 qstat_reset_cnts = qstat_num,
60};
61
62#ifdef CONFIG_QUEUED_LOCK_STAT
63
64
65
66#include <linux/debugfs.h>
67#include <linux/sched.h>
68#include <linux/sched/clock.h>
69#include <linux/fs.h>
70
71static const char * const qstat_names[qstat_num + 1] = {
72 [qstat_pv_hash_hops] = "pv_hash_hops",
73 [qstat_pv_kick_unlock] = "pv_kick_unlock",
74 [qstat_pv_kick_wake] = "pv_kick_wake",
75 [qstat_pv_spurious_wakeup] = "pv_spurious_wakeup",
76 [qstat_pv_latency_kick] = "pv_latency_kick",
77 [qstat_pv_latency_wake] = "pv_latency_wake",
78 [qstat_pv_lock_stealing] = "pv_lock_stealing",
79 [qstat_pv_wait_again] = "pv_wait_again",
80 [qstat_pv_wait_early] = "pv_wait_early",
81 [qstat_pv_wait_head] = "pv_wait_head",
82 [qstat_pv_wait_node] = "pv_wait_node",
83 [qstat_lock_pending] = "lock_pending",
84 [qstat_lock_slowpath] = "lock_slowpath",
85 [qstat_reset_cnts] = "reset_counters",
86};
87
88
89
90
91static DEFINE_PER_CPU(unsigned long, qstats[qstat_num]);
92static DEFINE_PER_CPU(u64, pv_kick_time);
93
94
95
96
97
98
99
100
101
102
103
104
105static ssize_t qstat_read(struct file *file, char __user *user_buf,
106 size_t count, loff_t *ppos)
107{
108 char buf[64];
109 int cpu, counter, len;
110 u64 stat = 0, kicks = 0;
111
112
113
114
115 counter = (long)file_inode(file)->i_private;
116
117 if (counter >= qstat_num)
118 return -EBADF;
119
120 for_each_possible_cpu(cpu) {
121 stat += per_cpu(qstats[counter], cpu);
122
123
124
125 switch (counter) {
126
127 case qstat_pv_latency_kick:
128 case qstat_pv_hash_hops:
129 kicks += per_cpu(qstats[qstat_pv_kick_unlock], cpu);
130 break;
131
132 case qstat_pv_latency_wake:
133 kicks += per_cpu(qstats[qstat_pv_kick_wake], cpu);
134 break;
135 }
136 }
137
138 if (counter == qstat_pv_hash_hops) {
139 u64 frac = 0;
140
141 if (kicks) {
142 frac = 100ULL * do_div(stat, kicks);
143 frac = DIV_ROUND_CLOSEST_ULL(frac, kicks);
144 }
145
146
147
148
149 len = snprintf(buf, sizeof(buf) - 1, "%llu.%02llu\n", stat, frac);
150 } else {
151
152
153
154 if ((counter == qstat_pv_latency_kick) ||
155 (counter == qstat_pv_latency_wake)) {
156 if (kicks)
157 stat = DIV_ROUND_CLOSEST_ULL(stat, kicks);
158 }
159 len = snprintf(buf, sizeof(buf) - 1, "%llu\n", stat);
160 }
161
162 return simple_read_from_buffer(user_buf, count, ppos, buf, len);
163}
164
165
166
167
168
169
170
171
172static ssize_t qstat_write(struct file *file, const char __user *user_buf,
173 size_t count, loff_t *ppos)
174{
175 int cpu;
176
177
178
179
180 if ((long)file_inode(file)->i_private != qstat_reset_cnts)
181 return count;
182
183 for_each_possible_cpu(cpu) {
184 int i;
185 unsigned long *ptr = per_cpu_ptr(qstats, cpu);
186
187 for (i = 0 ; i < qstat_num; i++)
188 WRITE_ONCE(ptr[i], 0);
189 }
190 return count;
191}
192
193
194
195
196static const struct file_operations fops_qstat = {
197 .read = qstat_read,
198 .write = qstat_write,
199 .llseek = default_llseek,
200};
201
202
203
204
205static int __init init_qspinlock_stat(void)
206{
207 struct dentry *d_qstat = debugfs_create_dir("qlockstat", NULL);
208 int i;
209
210 if (!d_qstat)
211 goto out;
212
213
214
215
216
217
218
219
220 for (i = 0; i < qstat_num; i++)
221 if (!debugfs_create_file(qstat_names[i], 0400, d_qstat,
222 (void *)(long)i, &fops_qstat))
223 goto fail_undo;
224
225 if (!debugfs_create_file(qstat_names[qstat_reset_cnts], 0200, d_qstat,
226 (void *)(long)qstat_reset_cnts, &fops_qstat))
227 goto fail_undo;
228
229 return 0;
230fail_undo:
231 debugfs_remove_recursive(d_qstat);
232out:
233 pr_warn("Could not create 'qlockstat' debugfs entries\n");
234 return -ENOMEM;
235}
236fs_initcall(init_qspinlock_stat);
237
238
239
240
241static inline void qstat_inc(enum qlock_stats stat, bool cond)
242{
243 if (cond)
244 this_cpu_inc(qstats[stat]);
245}
246
247
248
249
250static inline void qstat_hop(int hopcnt)
251{
252 this_cpu_add(qstats[qstat_pv_hash_hops], hopcnt);
253}
254
255
256
257
258static inline void __pv_kick(int cpu)
259{
260 u64 start = sched_clock();
261
262 per_cpu(pv_kick_time, cpu) = start;
263 pv_kick(cpu);
264 this_cpu_add(qstats[qstat_pv_latency_kick], sched_clock() - start);
265}
266
267
268
269
270static inline void __pv_wait(u8 *ptr, u8 val)
271{
272 u64 *pkick_time = this_cpu_ptr(&pv_kick_time);
273
274 *pkick_time = 0;
275 pv_wait(ptr, val);
276 if (*pkick_time) {
277 this_cpu_add(qstats[qstat_pv_latency_wake],
278 sched_clock() - *pkick_time);
279 qstat_inc(qstat_pv_kick_wake, true);
280 }
281}
282
283#define pv_kick(c) __pv_kick(c)
284#define pv_wait(p, v) __pv_wait(p, v)
285
286#else
287
288static inline void qstat_inc(enum qlock_stats stat, bool cond) { }
289static inline void qstat_hop(int hopcnt) { }
290
291#endif
292