1
2
3
4
5
6#include <linux/atomic.h>
7#include <linux/bug.h>
8#include <linux/interrupt.h>
9#include <linux/jiffies.h>
10#include <linux/kernel.h>
11#include <linux/list.h>
12#include <linux/lockdep.h>
13#include <linux/module.h>
14#include <linux/of.h>
15#include <linux/platform_device.h>
16#include <linux/slab.h>
17#include <linux/spinlock.h>
18#include <linux/types.h>
19#include <linux/wait.h>
20
21#include <soc/qcom/rpmh.h>
22
23#include "rpmh-internal.h"
24
25#define RPMH_TIMEOUT_MS msecs_to_jiffies(10000)
26
27#define DEFINE_RPMH_MSG_ONSTACK(device, s, q, name) \
28 struct rpmh_request name = { \
29 .msg = { \
30 .state = s, \
31 .cmds = name.cmd, \
32 .num_cmds = 0, \
33 .wait_for_compl = true, \
34 }, \
35 .cmd = { { 0 } }, \
36 .completion = q, \
37 .dev = device, \
38 .needs_free = false, \
39 }
40
41#define ctrlr_to_drv(ctrlr) container_of(ctrlr, struct rsc_drv, client)
42
43
44
45
46
47
48
49
50
51struct cache_req {
52 u32 addr;
53 u32 sleep_val;
54 u32 wake_val;
55 struct list_head list;
56};
57
58
59
60
61
62
63
64
65
66struct batch_cache_req {
67 struct list_head list;
68 int count;
69 struct rpmh_request rpm_msgs[];
70};
71
72static struct rpmh_ctrlr *get_rpmh_ctrlr(const struct device *dev)
73{
74 struct rsc_drv *drv = dev_get_drvdata(dev->parent);
75
76 return &drv->client;
77}
78
79void rpmh_tx_done(const struct tcs_request *msg, int r)
80{
81 struct rpmh_request *rpm_msg = container_of(msg, struct rpmh_request,
82 msg);
83 struct completion *compl = rpm_msg->completion;
84 bool free = rpm_msg->needs_free;
85
86 rpm_msg->err = r;
87
88 if (r)
89 dev_err(rpm_msg->dev, "RPMH TX fail in msg addr=%#x, err=%d\n",
90 rpm_msg->msg.cmds[0].addr, r);
91
92 if (!compl)
93 goto exit;
94
95
96 complete(compl);
97
98exit:
99 if (free)
100 kfree(rpm_msg);
101}
102
103static struct cache_req *__find_req(struct rpmh_ctrlr *ctrlr, u32 addr)
104{
105 struct cache_req *p, *req = NULL;
106
107 list_for_each_entry(p, &ctrlr->cache, list) {
108 if (p->addr == addr) {
109 req = p;
110 break;
111 }
112 }
113
114 return req;
115}
116
117static struct cache_req *cache_rpm_request(struct rpmh_ctrlr *ctrlr,
118 enum rpmh_state state,
119 struct tcs_cmd *cmd)
120{
121 struct cache_req *req;
122 unsigned long flags;
123 u32 old_sleep_val, old_wake_val;
124
125 spin_lock_irqsave(&ctrlr->cache_lock, flags);
126 req = __find_req(ctrlr, cmd->addr);
127 if (req)
128 goto existing;
129
130 req = kzalloc(sizeof(*req), GFP_ATOMIC);
131 if (!req) {
132 req = ERR_PTR(-ENOMEM);
133 goto unlock;
134 }
135
136 req->addr = cmd->addr;
137 req->sleep_val = req->wake_val = UINT_MAX;
138 list_add_tail(&req->list, &ctrlr->cache);
139
140existing:
141 old_sleep_val = req->sleep_val;
142 old_wake_val = req->wake_val;
143
144 switch (state) {
145 case RPMH_ACTIVE_ONLY_STATE:
146 case RPMH_WAKE_ONLY_STATE:
147 req->wake_val = cmd->data;
148 break;
149 case RPMH_SLEEP_STATE:
150 req->sleep_val = cmd->data;
151 break;
152 }
153
154 ctrlr->dirty |= (req->sleep_val != old_sleep_val ||
155 req->wake_val != old_wake_val) &&
156 req->sleep_val != UINT_MAX &&
157 req->wake_val != UINT_MAX;
158
159unlock:
160 spin_unlock_irqrestore(&ctrlr->cache_lock, flags);
161
162 return req;
163}
164
165
166
167
168
169
170
171
172
173
174
175
176static int __rpmh_write(const struct device *dev, enum rpmh_state state,
177 struct rpmh_request *rpm_msg)
178{
179 struct rpmh_ctrlr *ctrlr = get_rpmh_ctrlr(dev);
180 int ret = -EINVAL;
181 struct cache_req *req;
182 int i;
183
184 rpm_msg->msg.state = state;
185
186
187 for (i = 0; i < rpm_msg->msg.num_cmds; i++) {
188 req = cache_rpm_request(ctrlr, state, &rpm_msg->msg.cmds[i]);
189 if (IS_ERR(req))
190 return PTR_ERR(req);
191 }
192
193 rpm_msg->msg.state = state;
194
195 if (state == RPMH_ACTIVE_ONLY_STATE) {
196 WARN_ON(irqs_disabled());
197 ret = rpmh_rsc_send_data(ctrlr_to_drv(ctrlr), &rpm_msg->msg);
198 } else {
199
200 ret = 0;
201 rpmh_tx_done(&rpm_msg->msg, ret);
202 }
203
204 return ret;
205}
206
207static int __fill_rpmh_msg(struct rpmh_request *req, enum rpmh_state state,
208 const struct tcs_cmd *cmd, u32 n)
209{
210 if (!cmd || !n || n > MAX_RPMH_PAYLOAD)
211 return -EINVAL;
212
213 memcpy(req->cmd, cmd, n * sizeof(*cmd));
214
215 req->msg.state = state;
216 req->msg.cmds = req->cmd;
217 req->msg.num_cmds = n;
218
219 return 0;
220}
221
222
223
224
225
226
227
228
229
230
231
232
233int rpmh_write_async(const struct device *dev, enum rpmh_state state,
234 const struct tcs_cmd *cmd, u32 n)
235{
236 struct rpmh_request *rpm_msg;
237 int ret;
238
239 rpm_msg = kzalloc(sizeof(*rpm_msg), GFP_ATOMIC);
240 if (!rpm_msg)
241 return -ENOMEM;
242 rpm_msg->needs_free = true;
243
244 ret = __fill_rpmh_msg(rpm_msg, state, cmd, n);
245 if (ret) {
246 kfree(rpm_msg);
247 return ret;
248 }
249
250 return __rpmh_write(dev, state, rpm_msg);
251}
252EXPORT_SYMBOL(rpmh_write_async);
253
254
255
256
257
258
259
260
261
262
263
264int rpmh_write(const struct device *dev, enum rpmh_state state,
265 const struct tcs_cmd *cmd, u32 n)
266{
267 DECLARE_COMPLETION_ONSTACK(compl);
268 DEFINE_RPMH_MSG_ONSTACK(dev, state, &compl, rpm_msg);
269 int ret;
270
271 if (!cmd || !n || n > MAX_RPMH_PAYLOAD)
272 return -EINVAL;
273
274 memcpy(rpm_msg.cmd, cmd, n * sizeof(*cmd));
275 rpm_msg.msg.num_cmds = n;
276
277 ret = __rpmh_write(dev, state, &rpm_msg);
278 if (ret)
279 return ret;
280
281 ret = wait_for_completion_timeout(&compl, RPMH_TIMEOUT_MS);
282 WARN_ON(!ret);
283 return (ret > 0) ? 0 : -ETIMEDOUT;
284}
285EXPORT_SYMBOL(rpmh_write);
286
287static void cache_batch(struct rpmh_ctrlr *ctrlr, struct batch_cache_req *req)
288{
289 unsigned long flags;
290
291 spin_lock_irqsave(&ctrlr->cache_lock, flags);
292 list_add_tail(&req->list, &ctrlr->batch_cache);
293 ctrlr->dirty = true;
294 spin_unlock_irqrestore(&ctrlr->cache_lock, flags);
295}
296
297static int flush_batch(struct rpmh_ctrlr *ctrlr)
298{
299 struct batch_cache_req *req;
300 const struct rpmh_request *rpm_msg;
301 int ret = 0;
302 int i;
303
304
305 list_for_each_entry(req, &ctrlr->batch_cache, list) {
306 for (i = 0; i < req->count; i++) {
307 rpm_msg = req->rpm_msgs + i;
308 ret = rpmh_rsc_write_ctrl_data(ctrlr_to_drv(ctrlr),
309 &rpm_msg->msg);
310 if (ret)
311 break;
312 }
313 }
314
315 return ret;
316}
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335int rpmh_write_batch(const struct device *dev, enum rpmh_state state,
336 const struct tcs_cmd *cmd, u32 *n)
337{
338 struct batch_cache_req *req;
339 struct rpmh_request *rpm_msgs;
340 struct completion *compls;
341 struct rpmh_ctrlr *ctrlr = get_rpmh_ctrlr(dev);
342 unsigned long time_left;
343 int count = 0;
344 int ret, i;
345 void *ptr;
346
347 if (!cmd || !n)
348 return -EINVAL;
349
350 while (n[count] > 0)
351 count++;
352 if (!count)
353 return -EINVAL;
354
355 ptr = kzalloc(sizeof(*req) +
356 count * (sizeof(req->rpm_msgs[0]) + sizeof(*compls)),
357 GFP_ATOMIC);
358 if (!ptr)
359 return -ENOMEM;
360
361 req = ptr;
362 compls = ptr + sizeof(*req) + count * sizeof(*rpm_msgs);
363
364 req->count = count;
365 rpm_msgs = req->rpm_msgs;
366
367 for (i = 0; i < count; i++) {
368 __fill_rpmh_msg(rpm_msgs + i, state, cmd, n[i]);
369 cmd += n[i];
370 }
371
372 if (state != RPMH_ACTIVE_ONLY_STATE) {
373 cache_batch(ctrlr, req);
374 return 0;
375 }
376
377 for (i = 0; i < count; i++) {
378 struct completion *compl = &compls[i];
379
380 init_completion(compl);
381 rpm_msgs[i].completion = compl;
382 ret = rpmh_rsc_send_data(ctrlr_to_drv(ctrlr), &rpm_msgs[i].msg);
383 if (ret) {
384 pr_err("Error(%d) sending RPMH message addr=%#x\n",
385 ret, rpm_msgs[i].msg.cmds[0].addr);
386 break;
387 }
388 }
389
390 time_left = RPMH_TIMEOUT_MS;
391 while (i--) {
392 time_left = wait_for_completion_timeout(&compls[i], time_left);
393 if (!time_left) {
394
395
396
397
398
399 WARN_ON(1);
400 ret = -ETIMEDOUT;
401 goto exit;
402 }
403 }
404
405exit:
406 kfree(ptr);
407
408 return ret;
409}
410EXPORT_SYMBOL(rpmh_write_batch);
411
412static int is_req_valid(struct cache_req *req)
413{
414 return (req->sleep_val != UINT_MAX &&
415 req->wake_val != UINT_MAX &&
416 req->sleep_val != req->wake_val);
417}
418
419static int send_single(struct rpmh_ctrlr *ctrlr, enum rpmh_state state,
420 u32 addr, u32 data)
421{
422 DEFINE_RPMH_MSG_ONSTACK(NULL, state, NULL, rpm_msg);
423
424
425 rpm_msg.msg.wait_for_compl = (state == RPMH_WAKE_ONLY_STATE);
426 rpm_msg.cmd[0].addr = addr;
427 rpm_msg.cmd[0].data = data;
428 rpm_msg.msg.num_cmds = 1;
429
430 return rpmh_rsc_write_ctrl_data(ctrlr_to_drv(ctrlr), &rpm_msg.msg);
431}
432
433
434
435
436
437
438
439
440
441
442int rpmh_flush(struct rpmh_ctrlr *ctrlr)
443{
444 struct cache_req *p;
445 int ret = 0;
446
447 lockdep_assert_irqs_disabled();
448
449
450
451
452
453
454 if (!spin_trylock(&ctrlr->cache_lock))
455 return -EBUSY;
456
457 if (!ctrlr->dirty) {
458 pr_debug("Skipping flush, TCS has latest data.\n");
459 goto exit;
460 }
461
462
463 rpmh_rsc_invalidate(ctrlr_to_drv(ctrlr));
464
465
466 ret = flush_batch(ctrlr);
467 if (ret)
468 goto exit;
469
470 list_for_each_entry(p, &ctrlr->cache, list) {
471 if (!is_req_valid(p)) {
472 pr_debug("%s: skipping RPMH req: a:%#x s:%#x w:%#x",
473 __func__, p->addr, p->sleep_val, p->wake_val);
474 continue;
475 }
476 ret = send_single(ctrlr, RPMH_SLEEP_STATE, p->addr,
477 p->sleep_val);
478 if (ret)
479 goto exit;
480 ret = send_single(ctrlr, RPMH_WAKE_ONLY_STATE, p->addr,
481 p->wake_val);
482 if (ret)
483 goto exit;
484 }
485
486 ctrlr->dirty = false;
487
488exit:
489 spin_unlock(&ctrlr->cache_lock);
490 return ret;
491}
492
493
494
495
496
497
498
499
500void rpmh_invalidate(const struct device *dev)
501{
502 struct rpmh_ctrlr *ctrlr = get_rpmh_ctrlr(dev);
503 struct batch_cache_req *req, *tmp;
504 unsigned long flags;
505
506 spin_lock_irqsave(&ctrlr->cache_lock, flags);
507 list_for_each_entry_safe(req, tmp, &ctrlr->batch_cache, list)
508 kfree(req);
509 INIT_LIST_HEAD(&ctrlr->batch_cache);
510 ctrlr->dirty = true;
511 spin_unlock_irqrestore(&ctrlr->cache_lock, flags);
512}
513EXPORT_SYMBOL(rpmh_invalidate);
514