1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43enum qlock_stats {
44 qstat_pv_hash_hops,
45 qstat_pv_kick_unlock,
46 qstat_pv_kick_wake,
47 qstat_pv_latency_kick,
48 qstat_pv_latency_wake,
49 qstat_pv_lock_slowpath,
50 qstat_pv_lock_stealing,
51 qstat_pv_spurious_wakeup,
52 qstat_pv_wait_again,
53 qstat_pv_wait_early,
54 qstat_pv_wait_head,
55 qstat_pv_wait_node,
56 qstat_num,
57 qstat_reset_cnts = qstat_num,
58};
59
60#ifdef CONFIG_QUEUED_LOCK_STAT
61
62
63
64#include <linux/debugfs.h>
65#include <linux/sched.h>
66#include <linux/sched/clock.h>
67#include <linux/fs.h>
68
69static const char * const qstat_names[qstat_num + 1] = {
70 [qstat_pv_hash_hops] = "pv_hash_hops",
71 [qstat_pv_kick_unlock] = "pv_kick_unlock",
72 [qstat_pv_kick_wake] = "pv_kick_wake",
73 [qstat_pv_spurious_wakeup] = "pv_spurious_wakeup",
74 [qstat_pv_latency_kick] = "pv_latency_kick",
75 [qstat_pv_latency_wake] = "pv_latency_wake",
76 [qstat_pv_lock_slowpath] = "pv_lock_slowpath",
77 [qstat_pv_lock_stealing] = "pv_lock_stealing",
78 [qstat_pv_wait_again] = "pv_wait_again",
79 [qstat_pv_wait_early] = "pv_wait_early",
80 [qstat_pv_wait_head] = "pv_wait_head",
81 [qstat_pv_wait_node] = "pv_wait_node",
82 [qstat_reset_cnts] = "reset_counters",
83};
84
85
86
87
88static DEFINE_PER_CPU(unsigned long, qstats[qstat_num]);
89static DEFINE_PER_CPU(u64, pv_kick_time);
90
91
92
93
94
95
96
97
98
99
100
101
102static ssize_t qstat_read(struct file *file, char __user *user_buf,
103 size_t count, loff_t *ppos)
104{
105 char buf[64];
106 int cpu, counter, len;
107 u64 stat = 0, kicks = 0;
108
109
110
111
112 counter = (long)file_inode(file)->i_private;
113
114 if (counter >= qstat_num)
115 return -EBADF;
116
117 for_each_possible_cpu(cpu) {
118 stat += per_cpu(qstats[counter], cpu);
119
120
121
122 switch (counter) {
123
124 case qstat_pv_latency_kick:
125 case qstat_pv_hash_hops:
126 kicks += per_cpu(qstats[qstat_pv_kick_unlock], cpu);
127 break;
128
129 case qstat_pv_latency_wake:
130 kicks += per_cpu(qstats[qstat_pv_kick_wake], cpu);
131 break;
132 }
133 }
134
135 if (counter == qstat_pv_hash_hops) {
136 u64 frac = 0;
137
138 if (kicks) {
139 frac = 100ULL * do_div(stat, kicks);
140 frac = DIV_ROUND_CLOSEST_ULL(frac, kicks);
141 }
142
143
144
145
146 len = snprintf(buf, sizeof(buf) - 1, "%llu.%02llu\n", stat, frac);
147 } else {
148
149
150
151 if ((counter == qstat_pv_latency_kick) ||
152 (counter == qstat_pv_latency_wake)) {
153 if (kicks)
154 stat = DIV_ROUND_CLOSEST_ULL(stat, kicks);
155 }
156 len = snprintf(buf, sizeof(buf) - 1, "%llu\n", stat);
157 }
158
159 return simple_read_from_buffer(user_buf, count, ppos, buf, len);
160}
161
162
163
164
165
166
167
168
169static ssize_t qstat_write(struct file *file, const char __user *user_buf,
170 size_t count, loff_t *ppos)
171{
172 int cpu;
173
174
175
176
177 if ((long)file_inode(file)->i_private != qstat_reset_cnts)
178 return count;
179
180 for_each_possible_cpu(cpu) {
181 int i;
182 unsigned long *ptr = per_cpu_ptr(qstats, cpu);
183
184 for (i = 0 ; i < qstat_num; i++)
185 WRITE_ONCE(ptr[i], 0);
186 }
187 return count;
188}
189
190
191
192
193static const struct file_operations fops_qstat = {
194 .read = qstat_read,
195 .write = qstat_write,
196 .llseek = default_llseek,
197};
198
199
200
201
202static int __init init_qspinlock_stat(void)
203{
204 struct dentry *d_qstat = debugfs_create_dir("qlockstat", NULL);
205 int i;
206
207 if (!d_qstat)
208 goto out;
209
210
211
212
213
214
215
216
217 for (i = 0; i < qstat_num; i++)
218 if (!debugfs_create_file(qstat_names[i], 0400, d_qstat,
219 (void *)(long)i, &fops_qstat))
220 goto fail_undo;
221
222 if (!debugfs_create_file(qstat_names[qstat_reset_cnts], 0200, d_qstat,
223 (void *)(long)qstat_reset_cnts, &fops_qstat))
224 goto fail_undo;
225
226 return 0;
227fail_undo:
228 debugfs_remove_recursive(d_qstat);
229out:
230 pr_warn("Could not create 'qlockstat' debugfs entries\n");
231 return -ENOMEM;
232}
233fs_initcall(init_qspinlock_stat);
234
235
236
237
238static inline void qstat_inc(enum qlock_stats stat, bool cond)
239{
240 if (cond)
241 this_cpu_inc(qstats[stat]);
242}
243
244
245
246
247static inline void qstat_hop(int hopcnt)
248{
249 this_cpu_add(qstats[qstat_pv_hash_hops], hopcnt);
250}
251
252
253
254
255static inline void __pv_kick(int cpu)
256{
257 u64 start = sched_clock();
258
259 per_cpu(pv_kick_time, cpu) = start;
260 pv_kick(cpu);
261 this_cpu_add(qstats[qstat_pv_latency_kick], sched_clock() - start);
262}
263
264
265
266
267static inline void __pv_wait(u8 *ptr, u8 val)
268{
269 u64 *pkick_time = this_cpu_ptr(&pv_kick_time);
270
271 *pkick_time = 0;
272 pv_wait(ptr, val);
273 if (*pkick_time) {
274 this_cpu_add(qstats[qstat_pv_latency_wake],
275 sched_clock() - *pkick_time);
276 qstat_inc(qstat_pv_kick_wake, true);
277 }
278}
279
280#define pv_kick(c) __pv_kick(c)
281#define pv_wait(p, v) __pv_wait(p, v)
282
283#else
284
285static inline void qstat_inc(enum qlock_stats stat, bool cond) { }
286static inline void qstat_hop(int hopcnt) { }
287
288#endif
289