1
2
3
4
5
6
7
8
9
10
11#include "sun4i-ss.h"
12#include <asm/unaligned.h>
13#include <linux/scatterlist.h>
14
15
16#define SS_TIMEOUT 100
17
18int sun4i_hash_crainit(struct crypto_tfm *tfm)
19{
20 struct sun4i_tfm_ctx *op = crypto_tfm_ctx(tfm);
21 struct ahash_alg *alg = __crypto_ahash_alg(tfm->__crt_alg);
22 struct sun4i_ss_alg_template *algt;
23 int err;
24
25 memset(op, 0, sizeof(struct sun4i_tfm_ctx));
26
27 algt = container_of(alg, struct sun4i_ss_alg_template, alg.hash);
28 op->ss = algt->ss;
29
30 err = pm_runtime_get_sync(op->ss->dev);
31 if (err < 0)
32 return err;
33
34 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
35 sizeof(struct sun4i_req_ctx));
36 return 0;
37}
38
39void sun4i_hash_craexit(struct crypto_tfm *tfm)
40{
41 struct sun4i_tfm_ctx *op = crypto_tfm_ctx(tfm);
42
43 pm_runtime_put(op->ss->dev);
44}
45
46
47int sun4i_hash_init(struct ahash_request *areq)
48{
49 struct sun4i_req_ctx *op = ahash_request_ctx(areq);
50 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
51 struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg);
52 struct sun4i_ss_alg_template *algt;
53
54 memset(op, 0, sizeof(struct sun4i_req_ctx));
55
56 algt = container_of(alg, struct sun4i_ss_alg_template, alg.hash);
57 op->mode = algt->mode;
58
59 return 0;
60}
61
62int sun4i_hash_export_md5(struct ahash_request *areq, void *out)
63{
64 struct sun4i_req_ctx *op = ahash_request_ctx(areq);
65 struct md5_state *octx = out;
66 int i;
67
68 octx->byte_count = op->byte_count + op->len;
69
70 memcpy(octx->block, op->buf, op->len);
71
72 if (op->byte_count) {
73 for (i = 0; i < 4; i++)
74 octx->hash[i] = op->hash[i];
75 } else {
76 octx->hash[0] = SHA1_H0;
77 octx->hash[1] = SHA1_H1;
78 octx->hash[2] = SHA1_H2;
79 octx->hash[3] = SHA1_H3;
80 }
81
82 return 0;
83}
84
85int sun4i_hash_import_md5(struct ahash_request *areq, const void *in)
86{
87 struct sun4i_req_ctx *op = ahash_request_ctx(areq);
88 const struct md5_state *ictx = in;
89 int i;
90
91 sun4i_hash_init(areq);
92
93 op->byte_count = ictx->byte_count & ~0x3F;
94 op->len = ictx->byte_count & 0x3F;
95
96 memcpy(op->buf, ictx->block, op->len);
97
98 for (i = 0; i < 4; i++)
99 op->hash[i] = ictx->hash[i];
100
101 return 0;
102}
103
104int sun4i_hash_export_sha1(struct ahash_request *areq, void *out)
105{
106 struct sun4i_req_ctx *op = ahash_request_ctx(areq);
107 struct sha1_state *octx = out;
108 int i;
109
110 octx->count = op->byte_count + op->len;
111
112 memcpy(octx->buffer, op->buf, op->len);
113
114 if (op->byte_count) {
115 for (i = 0; i < 5; i++)
116 octx->state[i] = op->hash[i];
117 } else {
118 octx->state[0] = SHA1_H0;
119 octx->state[1] = SHA1_H1;
120 octx->state[2] = SHA1_H2;
121 octx->state[3] = SHA1_H3;
122 octx->state[4] = SHA1_H4;
123 }
124
125 return 0;
126}
127
128int sun4i_hash_import_sha1(struct ahash_request *areq, const void *in)
129{
130 struct sun4i_req_ctx *op = ahash_request_ctx(areq);
131 const struct sha1_state *ictx = in;
132 int i;
133
134 sun4i_hash_init(areq);
135
136 op->byte_count = ictx->count & ~0x3F;
137 op->len = ictx->count & 0x3F;
138
139 memcpy(op->buf, ictx->buffer, op->len);
140
141 for (i = 0; i < 5; i++)
142 op->hash[i] = ictx->state[i];
143
144 return 0;
145}
146
147#define SS_HASH_UPDATE 1
148#define SS_HASH_FINAL 2
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177static int sun4i_hash(struct ahash_request *areq)
178{
179
180
181
182
183
184
185
186
187
188
189 unsigned int i = 0, end, fill, min_fill, nwait, nbw = 0, j = 0, todo;
190 unsigned int in_i = 0;
191 u32 spaces, rx_cnt = SS_RX_DEFAULT, bf[32] = {0}, v, ivmode = 0;
192 struct sun4i_req_ctx *op = ahash_request_ctx(areq);
193 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
194 struct sun4i_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
195 struct sun4i_ss_ctx *ss = tfmctx->ss;
196 struct scatterlist *in_sg = areq->src;
197 struct sg_mapping_iter mi;
198 int in_r, err = 0;
199 size_t copied = 0;
200 u32 wb = 0;
201
202 dev_dbg(ss->dev, "%s %s bc=%llu len=%u mode=%x wl=%u h0=%0x",
203 __func__, crypto_tfm_alg_name(areq->base.tfm),
204 op->byte_count, areq->nbytes, op->mode,
205 op->len, op->hash[0]);
206
207 if (unlikely(!areq->nbytes) && !(op->flags & SS_HASH_FINAL))
208 return 0;
209
210
211 if (unlikely(areq->nbytes > UINT_MAX - op->len)) {
212 dev_err(ss->dev, "Cannot process too large request\n");
213 return -EINVAL;
214 }
215
216 if (op->len + areq->nbytes < 64 && !(op->flags & SS_HASH_FINAL)) {
217
218 copied = sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
219 op->buf + op->len, areq->nbytes, 0);
220 op->len += copied;
221 return 0;
222 }
223
224 spin_lock_bh(&ss->slock);
225
226
227
228
229
230 if (op->byte_count) {
231 ivmode = SS_IV_ARBITRARY;
232 for (i = 0; i < crypto_ahash_digestsize(tfm) / 4; i++)
233 writel(op->hash[i], ss->base + SS_IV0 + i * 4);
234 }
235
236 writel(op->mode | SS_ENABLED | ivmode, ss->base + SS_CTL);
237
238 if (!(op->flags & SS_HASH_UPDATE))
239 goto hash_final;
240
241
242 if (!(op->flags & SS_HASH_FINAL)) {
243 end = ((areq->nbytes + op->len) / 64) * 64 - op->len;
244
245 if (end > areq->nbytes || areq->nbytes - end > 63) {
246 dev_err(ss->dev, "ERROR: Bound error %u %u\n",
247 end, areq->nbytes);
248 err = -EINVAL;
249 goto release_ss;
250 }
251 } else {
252
253 if (areq->nbytes < 4)
254 end = 0;
255 else
256 end = ((areq->nbytes + op->len) / 4) * 4 - op->len;
257 }
258
259
260 i = 1;
261 while (in_sg && i == 1) {
262 if (in_sg->length % 4)
263 i = 0;
264 in_sg = sg_next(in_sg);
265 }
266 if (i == 1 && !op->len && areq->nbytes)
267 dev_dbg(ss->dev, "We can DMA\n");
268
269 i = 0;
270 sg_miter_start(&mi, areq->src, sg_nents(areq->src),
271 SG_MITER_FROM_SG | SG_MITER_ATOMIC);
272 sg_miter_next(&mi);
273 in_i = 0;
274
275 do {
276
277
278
279
280
281 if (op->len || (mi.length - in_i) < 4) {
282
283
284
285
286
287 while (op->len < 64 && i < end) {
288
289 in_r = min(end - i, 64 - op->len);
290 in_r = min_t(size_t, mi.length - in_i, in_r);
291 memcpy(op->buf + op->len, mi.addr + in_i, in_r);
292 op->len += in_r;
293 i += in_r;
294 in_i += in_r;
295 if (in_i == mi.length) {
296 sg_miter_next(&mi);
297 in_i = 0;
298 }
299 }
300 if (op->len > 3 && !(op->len % 4)) {
301
302 writesl(ss->base + SS_RXFIFO, op->buf,
303 op->len / 4);
304 op->byte_count += op->len;
305 op->len = 0;
306 }
307 }
308 if (mi.length - in_i > 3 && i < end) {
309
310 in_r = min_t(size_t, mi.length - in_i, areq->nbytes - i);
311 in_r = min_t(size_t, ((mi.length - in_i) / 4) * 4, in_r);
312
313 todo = min3((u32)(end - i) / 4, rx_cnt, (u32)in_r / 4);
314 writesl(ss->base + SS_RXFIFO, mi.addr + in_i, todo);
315 op->byte_count += todo * 4;
316 i += todo * 4;
317 in_i += todo * 4;
318 rx_cnt -= todo;
319 if (!rx_cnt) {
320 spaces = readl(ss->base + SS_FCSR);
321 rx_cnt = SS_RXFIFO_SPACES(spaces);
322 }
323 if (in_i == mi.length) {
324 sg_miter_next(&mi);
325 in_i = 0;
326 }
327 }
328 } while (i < end);
329
330
331
332
333
334 if ((areq->nbytes - i) < 64) {
335 while (i < areq->nbytes && in_i < mi.length && op->len < 64) {
336
337 in_r = min(areq->nbytes - i, 64 - op->len);
338 in_r = min_t(size_t, mi.length - in_i, in_r);
339 memcpy(op->buf + op->len, mi.addr + in_i, in_r);
340 op->len += in_r;
341 i += in_r;
342 in_i += in_r;
343 if (in_i == mi.length) {
344 sg_miter_next(&mi);
345 in_i = 0;
346 }
347 }
348 }
349
350 sg_miter_stop(&mi);
351
352
353
354
355
356
357 if (op->flags & SS_HASH_FINAL)
358 goto hash_final;
359
360 writel(op->mode | SS_ENABLED | SS_DATA_END, ss->base + SS_CTL);
361 i = 0;
362 do {
363 v = readl(ss->base + SS_CTL);
364 i++;
365 } while (i < SS_TIMEOUT && (v & SS_DATA_END));
366 if (unlikely(i >= SS_TIMEOUT)) {
367 dev_err_ratelimited(ss->dev,
368 "ERROR: hash end timeout %d>%d ctl=%x len=%u\n",
369 i, SS_TIMEOUT, v, areq->nbytes);
370 err = -EIO;
371 goto release_ss;
372 }
373
374
375
376
377
378
379
380
381 ndelay(1);
382
383 for (i = 0; i < crypto_ahash_digestsize(tfm) / 4; i++)
384 op->hash[i] = readl(ss->base + SS_MD0 + i * 4);
385
386 goto release_ss;
387
388
389
390
391
392
393
394
395
396
397
398
399
400hash_final:
401
402
403 if (op->len) {
404 nwait = op->len / 4;
405 if (nwait) {
406 writesl(ss->base + SS_RXFIFO, op->buf, nwait);
407 op->byte_count += 4 * nwait;
408 }
409
410 nbw = op->len - 4 * nwait;
411 if (nbw) {
412 wb = le32_to_cpup((__le32 *)(op->buf + nwait * 4));
413 wb &= GENMASK((nbw * 8) - 1, 0);
414
415 op->byte_count += nbw;
416 }
417 }
418
419
420 wb |= ((1 << 7) << (nbw * 8));
421 ((__le32 *)bf)[j++] = cpu_to_le32(wb);
422
423
424
425
426
427
428
429 fill = 64 - (op->byte_count % 64);
430 min_fill = 2 * sizeof(u32) + (nbw ? 0 : sizeof(u32));
431
432
433 if (fill < min_fill)
434 fill += 64;
435
436 j += (fill - min_fill) / sizeof(u32);
437
438
439 if (op->mode == SS_OP_SHA1) {
440 __be64 *bits = (__be64 *)&bf[j];
441 *bits = cpu_to_be64(op->byte_count << 3);
442 j += 2;
443 } else {
444 __le64 *bits = (__le64 *)&bf[j];
445 *bits = cpu_to_le64(op->byte_count << 3);
446 j += 2;
447 }
448 writesl(ss->base + SS_RXFIFO, bf, j);
449
450
451 writel(op->mode | SS_ENABLED | SS_DATA_END, ss->base + SS_CTL);
452
453
454
455
456
457
458 i = 0;
459 do {
460 v = readl(ss->base + SS_CTL);
461 i++;
462 } while (i < SS_TIMEOUT && (v & SS_DATA_END));
463 if (unlikely(i >= SS_TIMEOUT)) {
464 dev_err_ratelimited(ss->dev,
465 "ERROR: hash end timeout %d>%d ctl=%x len=%u\n",
466 i, SS_TIMEOUT, v, areq->nbytes);
467 err = -EIO;
468 goto release_ss;
469 }
470
471
472
473
474
475
476
477
478 ndelay(1);
479
480
481 if (op->mode == SS_OP_SHA1) {
482 for (i = 0; i < 5; i++) {
483 v = readl(ss->base + SS_MD0 + i * 4);
484 if (ss->variant->sha1_in_be)
485 put_unaligned_le32(v, areq->result + i * 4);
486 else
487 put_unaligned_be32(v, areq->result + i * 4);
488 }
489 } else {
490 for (i = 0; i < 4; i++) {
491 v = readl(ss->base + SS_MD0 + i * 4);
492 put_unaligned_le32(v, areq->result + i * 4);
493 }
494 }
495
496release_ss:
497 writel(0, ss->base + SS_CTL);
498 spin_unlock_bh(&ss->slock);
499 return err;
500}
501
502int sun4i_hash_final(struct ahash_request *areq)
503{
504 struct sun4i_req_ctx *op = ahash_request_ctx(areq);
505
506 op->flags = SS_HASH_FINAL;
507 return sun4i_hash(areq);
508}
509
510int sun4i_hash_update(struct ahash_request *areq)
511{
512 struct sun4i_req_ctx *op = ahash_request_ctx(areq);
513
514 op->flags = SS_HASH_UPDATE;
515 return sun4i_hash(areq);
516}
517
518
519int sun4i_hash_finup(struct ahash_request *areq)
520{
521 struct sun4i_req_ctx *op = ahash_request_ctx(areq);
522
523 op->flags = SS_HASH_UPDATE | SS_HASH_FINAL;
524 return sun4i_hash(areq);
525}
526
527
528int sun4i_hash_digest(struct ahash_request *areq)
529{
530 int err;
531 struct sun4i_req_ctx *op = ahash_request_ctx(areq);
532
533 err = sun4i_hash_init(areq);
534 if (err)
535 return err;
536
537 op->flags = SS_HASH_UPDATE | SS_HASH_FINAL;
538 return sun4i_hash(areq);
539}
540