1
2
3
4
5
6
7
8
9
10
11#include "sun4i-ss.h"
12#include <linux/scatterlist.h>
13
14
15#define SS_TIMEOUT 100
16
17int sun4i_hash_crainit(struct crypto_tfm *tfm)
18{
19 struct sun4i_tfm_ctx *op = crypto_tfm_ctx(tfm);
20 struct ahash_alg *alg = __crypto_ahash_alg(tfm->__crt_alg);
21 struct sun4i_ss_alg_template *algt;
22
23 memset(op, 0, sizeof(struct sun4i_tfm_ctx));
24
25 algt = container_of(alg, struct sun4i_ss_alg_template, alg.hash);
26 op->ss = algt->ss;
27
28 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
29 sizeof(struct sun4i_req_ctx));
30 return 0;
31}
32
33
34int sun4i_hash_init(struct ahash_request *areq)
35{
36 struct sun4i_req_ctx *op = ahash_request_ctx(areq);
37 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
38 struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg);
39 struct sun4i_ss_alg_template *algt;
40
41 memset(op, 0, sizeof(struct sun4i_req_ctx));
42
43 algt = container_of(alg, struct sun4i_ss_alg_template, alg.hash);
44 op->mode = algt->mode;
45
46 return 0;
47}
48
49int sun4i_hash_export_md5(struct ahash_request *areq, void *out)
50{
51 struct sun4i_req_ctx *op = ahash_request_ctx(areq);
52 struct md5_state *octx = out;
53 int i;
54
55 octx->byte_count = op->byte_count + op->len;
56
57 memcpy(octx->block, op->buf, op->len);
58
59 if (op->byte_count) {
60 for (i = 0; i < 4; i++)
61 octx->hash[i] = op->hash[i];
62 } else {
63 octx->hash[0] = SHA1_H0;
64 octx->hash[1] = SHA1_H1;
65 octx->hash[2] = SHA1_H2;
66 octx->hash[3] = SHA1_H3;
67 }
68
69 return 0;
70}
71
72int sun4i_hash_import_md5(struct ahash_request *areq, const void *in)
73{
74 struct sun4i_req_ctx *op = ahash_request_ctx(areq);
75 const struct md5_state *ictx = in;
76 int i;
77
78 sun4i_hash_init(areq);
79
80 op->byte_count = ictx->byte_count & ~0x3F;
81 op->len = ictx->byte_count & 0x3F;
82
83 memcpy(op->buf, ictx->block, op->len);
84
85 for (i = 0; i < 4; i++)
86 op->hash[i] = ictx->hash[i];
87
88 return 0;
89}
90
91int sun4i_hash_export_sha1(struct ahash_request *areq, void *out)
92{
93 struct sun4i_req_ctx *op = ahash_request_ctx(areq);
94 struct sha1_state *octx = out;
95 int i;
96
97 octx->count = op->byte_count + op->len;
98
99 memcpy(octx->buffer, op->buf, op->len);
100
101 if (op->byte_count) {
102 for (i = 0; i < 5; i++)
103 octx->state[i] = op->hash[i];
104 } else {
105 octx->state[0] = SHA1_H0;
106 octx->state[1] = SHA1_H1;
107 octx->state[2] = SHA1_H2;
108 octx->state[3] = SHA1_H3;
109 octx->state[4] = SHA1_H4;
110 }
111
112 return 0;
113}
114
115int sun4i_hash_import_sha1(struct ahash_request *areq, const void *in)
116{
117 struct sun4i_req_ctx *op = ahash_request_ctx(areq);
118 const struct sha1_state *ictx = in;
119 int i;
120
121 sun4i_hash_init(areq);
122
123 op->byte_count = ictx->count & ~0x3F;
124 op->len = ictx->count & 0x3F;
125
126 memcpy(op->buf, ictx->buffer, op->len);
127
128 for (i = 0; i < 5; i++)
129 op->hash[i] = ictx->state[i];
130
131 return 0;
132}
133
134#define SS_HASH_UPDATE 1
135#define SS_HASH_FINAL 2
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164static int sun4i_hash(struct ahash_request *areq)
165{
166
167
168
169
170
171
172
173
174
175
176 unsigned int i = 0, end, fill, min_fill, nwait, nbw = 0, j = 0, todo;
177 unsigned int in_i = 0;
178 u32 spaces, rx_cnt = SS_RX_DEFAULT, bf[32] = {0}, wb = 0, v, ivmode = 0;
179 struct sun4i_req_ctx *op = ahash_request_ctx(areq);
180 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
181 struct sun4i_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
182 struct sun4i_ss_ctx *ss = tfmctx->ss;
183 struct scatterlist *in_sg = areq->src;
184 struct sg_mapping_iter mi;
185 int in_r, err = 0;
186 size_t copied = 0;
187
188 dev_dbg(ss->dev, "%s %s bc=%llu len=%u mode=%x wl=%u h0=%0x",
189 __func__, crypto_tfm_alg_name(areq->base.tfm),
190 op->byte_count, areq->nbytes, op->mode,
191 op->len, op->hash[0]);
192
193 if (unlikely(!areq->nbytes) && !(op->flags & SS_HASH_FINAL))
194 return 0;
195
196
197 if (unlikely(areq->nbytes > UINT_MAX - op->len)) {
198 dev_err(ss->dev, "Cannot process too large request\n");
199 return -EINVAL;
200 }
201
202 if (op->len + areq->nbytes < 64 && !(op->flags & SS_HASH_FINAL)) {
203
204 copied = sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
205 op->buf + op->len, areq->nbytes, 0);
206 op->len += copied;
207 return 0;
208 }
209
210 spin_lock_bh(&ss->slock);
211
212
213
214
215
216 if (op->byte_count) {
217 ivmode = SS_IV_ARBITRARY;
218 for (i = 0; i < 5; i++)
219 writel(op->hash[i], ss->base + SS_IV0 + i * 4);
220 }
221
222 writel(op->mode | SS_ENABLED | ivmode, ss->base + SS_CTL);
223
224 if (!(op->flags & SS_HASH_UPDATE))
225 goto hash_final;
226
227
228 if (!(op->flags & SS_HASH_FINAL)) {
229 end = ((areq->nbytes + op->len) / 64) * 64 - op->len;
230
231 if (end > areq->nbytes || areq->nbytes - end > 63) {
232 dev_err(ss->dev, "ERROR: Bound error %u %u\n",
233 end, areq->nbytes);
234 err = -EINVAL;
235 goto release_ss;
236 }
237 } else {
238
239 if (areq->nbytes < 4)
240 end = 0;
241 else
242 end = ((areq->nbytes + op->len) / 4) * 4 - op->len;
243 }
244
245
246 i = 1;
247 while (in_sg && i == 1) {
248 if (in_sg->length % 4)
249 i = 0;
250 in_sg = sg_next(in_sg);
251 }
252 if (i == 1 && !op->len && areq->nbytes)
253 dev_dbg(ss->dev, "We can DMA\n");
254
255 i = 0;
256 sg_miter_start(&mi, areq->src, sg_nents(areq->src),
257 SG_MITER_FROM_SG | SG_MITER_ATOMIC);
258 sg_miter_next(&mi);
259 in_i = 0;
260
261 do {
262
263
264
265
266
267 if (op->len || (mi.length - in_i) < 4) {
268
269
270
271
272
273 while (op->len < 64 && i < end) {
274
275 in_r = min3(mi.length - in_i, end - i,
276 64 - op->len);
277 memcpy(op->buf + op->len, mi.addr + in_i, in_r);
278 op->len += in_r;
279 i += in_r;
280 in_i += in_r;
281 if (in_i == mi.length) {
282 sg_miter_next(&mi);
283 in_i = 0;
284 }
285 }
286 if (op->len > 3 && !(op->len % 4)) {
287
288 writesl(ss->base + SS_RXFIFO, op->buf,
289 op->len / 4);
290 op->byte_count += op->len;
291 op->len = 0;
292 }
293 }
294 if (mi.length - in_i > 3 && i < end) {
295
296 in_r = min3(mi.length - in_i, areq->nbytes - i,
297 ((mi.length - in_i) / 4) * 4);
298
299 todo = min3((u32)(end - i) / 4, rx_cnt, (u32)in_r / 4);
300 writesl(ss->base + SS_RXFIFO, mi.addr + in_i, todo);
301 op->byte_count += todo * 4;
302 i += todo * 4;
303 in_i += todo * 4;
304 rx_cnt -= todo;
305 if (!rx_cnt) {
306 spaces = readl(ss->base + SS_FCSR);
307 rx_cnt = SS_RXFIFO_SPACES(spaces);
308 }
309 if (in_i == mi.length) {
310 sg_miter_next(&mi);
311 in_i = 0;
312 }
313 }
314 } while (i < end);
315
316
317
318
319
320 if ((areq->nbytes - i) < 64) {
321 while (i < areq->nbytes && in_i < mi.length && op->len < 64) {
322
323 in_r = min3(mi.length - in_i, areq->nbytes - i,
324 64 - op->len);
325 memcpy(op->buf + op->len, mi.addr + in_i, in_r);
326 op->len += in_r;
327 i += in_r;
328 in_i += in_r;
329 if (in_i == mi.length) {
330 sg_miter_next(&mi);
331 in_i = 0;
332 }
333 }
334 }
335
336 sg_miter_stop(&mi);
337
338
339
340
341
342
343 if (op->flags & SS_HASH_FINAL)
344 goto hash_final;
345
346 writel(op->mode | SS_ENABLED | SS_DATA_END, ss->base + SS_CTL);
347 i = 0;
348 do {
349 v = readl(ss->base + SS_CTL);
350 i++;
351 } while (i < SS_TIMEOUT && (v & SS_DATA_END));
352 if (unlikely(i >= SS_TIMEOUT)) {
353 dev_err_ratelimited(ss->dev,
354 "ERROR: hash end timeout %d>%d ctl=%x len=%u\n",
355 i, SS_TIMEOUT, v, areq->nbytes);
356 err = -EIO;
357 goto release_ss;
358 }
359
360
361
362
363
364
365
366
367 ndelay(1);
368
369 for (i = 0; i < crypto_ahash_digestsize(tfm) / 4; i++)
370 op->hash[i] = readl(ss->base + SS_MD0 + i * 4);
371
372 goto release_ss;
373
374
375
376
377
378
379
380
381
382
383
384
385
386hash_final:
387
388
389 if (op->len) {
390 nwait = op->len / 4;
391 if (nwait) {
392 writesl(ss->base + SS_RXFIFO, op->buf, nwait);
393 op->byte_count += 4 * nwait;
394 }
395
396 nbw = op->len - 4 * nwait;
397 if (nbw) {
398 wb = *(u32 *)(op->buf + nwait * 4);
399 wb &= GENMASK((nbw * 8) - 1, 0);
400
401 op->byte_count += nbw;
402 }
403 }
404
405
406 wb |= ((1 << 7) << (nbw * 8));
407 bf[j++] = wb;
408
409
410
411
412
413
414
415 fill = 64 - (op->byte_count % 64);
416 min_fill = 2 * sizeof(u32) + (nbw ? 0 : sizeof(u32));
417
418
419 if (fill < min_fill)
420 fill += 64;
421
422 j += (fill - min_fill) / sizeof(u32);
423
424
425 if (op->mode == SS_OP_SHA1) {
426 __be64 bits = cpu_to_be64(op->byte_count << 3);
427 bf[j++] = lower_32_bits(bits);
428 bf[j++] = upper_32_bits(bits);
429 } else {
430 __le64 bits = op->byte_count << 3;
431 bf[j++] = lower_32_bits(bits);
432 bf[j++] = upper_32_bits(bits);
433 }
434 writesl(ss->base + SS_RXFIFO, bf, j);
435
436
437 writel(op->mode | SS_ENABLED | SS_DATA_END, ss->base + SS_CTL);
438
439
440
441
442
443
444 i = 0;
445 do {
446 v = readl(ss->base + SS_CTL);
447 i++;
448 } while (i < SS_TIMEOUT && (v & SS_DATA_END));
449 if (unlikely(i >= SS_TIMEOUT)) {
450 dev_err_ratelimited(ss->dev,
451 "ERROR: hash end timeout %d>%d ctl=%x len=%u\n",
452 i, SS_TIMEOUT, v, areq->nbytes);
453 err = -EIO;
454 goto release_ss;
455 }
456
457
458
459
460
461
462
463
464 ndelay(1);
465
466
467 if (op->mode == SS_OP_SHA1) {
468 for (i = 0; i < 5; i++) {
469 v = cpu_to_be32(readl(ss->base + SS_MD0 + i * 4));
470 memcpy(areq->result + i * 4, &v, 4);
471 }
472 } else {
473 for (i = 0; i < 4; i++) {
474 v = readl(ss->base + SS_MD0 + i * 4);
475 memcpy(areq->result + i * 4, &v, 4);
476 }
477 }
478
479release_ss:
480 writel(0, ss->base + SS_CTL);
481 spin_unlock_bh(&ss->slock);
482 return err;
483}
484
485int sun4i_hash_final(struct ahash_request *areq)
486{
487 struct sun4i_req_ctx *op = ahash_request_ctx(areq);
488
489 op->flags = SS_HASH_FINAL;
490 return sun4i_hash(areq);
491}
492
493int sun4i_hash_update(struct ahash_request *areq)
494{
495 struct sun4i_req_ctx *op = ahash_request_ctx(areq);
496
497 op->flags = SS_HASH_UPDATE;
498 return sun4i_hash(areq);
499}
500
501
502int sun4i_hash_finup(struct ahash_request *areq)
503{
504 struct sun4i_req_ctx *op = ahash_request_ctx(areq);
505
506 op->flags = SS_HASH_UPDATE | SS_HASH_FINAL;
507 return sun4i_hash(areq);
508}
509
510
511int sun4i_hash_digest(struct ahash_request *areq)
512{
513 int err;
514 struct sun4i_req_ctx *op = ahash_request_ctx(areq);
515
516 err = sun4i_hash_init(areq);
517 if (err)
518 return err;
519
520 op->flags = SS_HASH_UPDATE | SS_HASH_FINAL;
521 return sun4i_hash(areq);
522}
523