1
2
3
4
5
6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7
8#include <linux/crypto.h>
9#include <linux/init.h>
10#include <linux/module.h>
11#include <linux/mm.h>
12#include <linux/types.h>
13#include <crypto/algapi.h>
14#include <crypto/des.h>
15
16#include <asm/fpumacro.h>
17#include <asm/pstate.h>
18#include <asm/elf.h>
19
20#include "opcodes.h"
21
22struct des_sparc64_ctx {
23 u64 encrypt_expkey[DES_EXPKEY_WORDS / 2];
24 u64 decrypt_expkey[DES_EXPKEY_WORDS / 2];
25};
26
27struct des3_ede_sparc64_ctx {
28 u64 encrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2];
29 u64 decrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2];
30};
31
32static void encrypt_to_decrypt(u64 *d, const u64 *e)
33{
34 const u64 *s = e + (DES_EXPKEY_WORDS / 2) - 1;
35 int i;
36
37 for (i = 0; i < DES_EXPKEY_WORDS / 2; i++)
38 *d++ = *s--;
39}
40
41extern void des_sparc64_key_expand(const u32 *input_key, u64 *key);
42
43static int des_set_key(struct crypto_tfm *tfm, const u8 *key,
44 unsigned int keylen)
45{
46 struct des_sparc64_ctx *dctx = crypto_tfm_ctx(tfm);
47 u32 *flags = &tfm->crt_flags;
48 u32 tmp[DES_EXPKEY_WORDS];
49 int ret;
50
51
52
53
54
55 ret = des_ekey(tmp, key);
56 if (unlikely(ret == 0) && (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
57 *flags |= CRYPTO_TFM_RES_WEAK_KEY;
58 return -EINVAL;
59 }
60
61 des_sparc64_key_expand((const u32 *) key, &dctx->encrypt_expkey[0]);
62 encrypt_to_decrypt(&dctx->decrypt_expkey[0], &dctx->encrypt_expkey[0]);
63
64 return 0;
65}
66
67extern void des_sparc64_crypt(const u64 *key, const u64 *input,
68 u64 *output);
69
70static void des_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
71{
72 struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
73 const u64 *K = ctx->encrypt_expkey;
74
75 des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
76}
77
78static void des_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
79{
80 struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
81 const u64 *K = ctx->decrypt_expkey;
82
83 des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
84}
85
86extern void des_sparc64_load_keys(const u64 *key);
87
88extern void des_sparc64_ecb_crypt(const u64 *input, u64 *output,
89 unsigned int len);
90
91#define DES_BLOCK_MASK (~(DES_BLOCK_SIZE - 1))
92
93static int __ecb_crypt(struct blkcipher_desc *desc,
94 struct scatterlist *dst, struct scatterlist *src,
95 unsigned int nbytes, bool encrypt)
96{
97 struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
98 struct blkcipher_walk walk;
99 int err;
100
101 blkcipher_walk_init(&walk, dst, src, nbytes);
102 err = blkcipher_walk_virt(desc, &walk);
103 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
104
105 if (encrypt)
106 des_sparc64_load_keys(&ctx->encrypt_expkey[0]);
107 else
108 des_sparc64_load_keys(&ctx->decrypt_expkey[0]);
109 while ((nbytes = walk.nbytes)) {
110 unsigned int block_len = nbytes & DES_BLOCK_MASK;
111
112 if (likely(block_len)) {
113 des_sparc64_ecb_crypt((const u64 *)walk.src.virt.addr,
114 (u64 *) walk.dst.virt.addr,
115 block_len);
116 }
117 nbytes &= DES_BLOCK_SIZE - 1;
118 err = blkcipher_walk_done(desc, &walk, nbytes);
119 }
120 fprs_write(0);
121 return err;
122}
123
124static int ecb_encrypt(struct blkcipher_desc *desc,
125 struct scatterlist *dst, struct scatterlist *src,
126 unsigned int nbytes)
127{
128 return __ecb_crypt(desc, dst, src, nbytes, true);
129}
130
131static int ecb_decrypt(struct blkcipher_desc *desc,
132 struct scatterlist *dst, struct scatterlist *src,
133 unsigned int nbytes)
134{
135 return __ecb_crypt(desc, dst, src, nbytes, false);
136}
137
138extern void des_sparc64_cbc_encrypt(const u64 *input, u64 *output,
139 unsigned int len, u64 *iv);
140
141static int cbc_encrypt(struct blkcipher_desc *desc,
142 struct scatterlist *dst, struct scatterlist *src,
143 unsigned int nbytes)
144{
145 struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
146 struct blkcipher_walk walk;
147 int err;
148
149 blkcipher_walk_init(&walk, dst, src, nbytes);
150 err = blkcipher_walk_virt(desc, &walk);
151 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
152
153 des_sparc64_load_keys(&ctx->encrypt_expkey[0]);
154 while ((nbytes = walk.nbytes)) {
155 unsigned int block_len = nbytes & DES_BLOCK_MASK;
156
157 if (likely(block_len)) {
158 des_sparc64_cbc_encrypt((const u64 *)walk.src.virt.addr,
159 (u64 *) walk.dst.virt.addr,
160 block_len, (u64 *) walk.iv);
161 }
162 nbytes &= DES_BLOCK_SIZE - 1;
163 err = blkcipher_walk_done(desc, &walk, nbytes);
164 }
165 fprs_write(0);
166 return err;
167}
168
169extern void des_sparc64_cbc_decrypt(const u64 *input, u64 *output,
170 unsigned int len, u64 *iv);
171
172static int cbc_decrypt(struct blkcipher_desc *desc,
173 struct scatterlist *dst, struct scatterlist *src,
174 unsigned int nbytes)
175{
176 struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
177 struct blkcipher_walk walk;
178 int err;
179
180 blkcipher_walk_init(&walk, dst, src, nbytes);
181 err = blkcipher_walk_virt(desc, &walk);
182 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
183
184 des_sparc64_load_keys(&ctx->decrypt_expkey[0]);
185 while ((nbytes = walk.nbytes)) {
186 unsigned int block_len = nbytes & DES_BLOCK_MASK;
187
188 if (likely(block_len)) {
189 des_sparc64_cbc_decrypt((const u64 *)walk.src.virt.addr,
190 (u64 *) walk.dst.virt.addr,
191 block_len, (u64 *) walk.iv);
192 }
193 nbytes &= DES_BLOCK_SIZE - 1;
194 err = blkcipher_walk_done(desc, &walk, nbytes);
195 }
196 fprs_write(0);
197 return err;
198}
199
200static int des3_ede_set_key(struct crypto_tfm *tfm, const u8 *key,
201 unsigned int keylen)
202{
203 struct des3_ede_sparc64_ctx *dctx = crypto_tfm_ctx(tfm);
204 const u32 *K = (const u32 *)key;
205 u32 *flags = &tfm->crt_flags;
206 u64 k1[DES_EXPKEY_WORDS / 2];
207 u64 k2[DES_EXPKEY_WORDS / 2];
208 u64 k3[DES_EXPKEY_WORDS / 2];
209
210 if (unlikely(!((K[0] ^ K[2]) | (K[1] ^ K[3])) ||
211 !((K[2] ^ K[4]) | (K[3] ^ K[5]))) &&
212 (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
213 *flags |= CRYPTO_TFM_RES_WEAK_KEY;
214 return -EINVAL;
215 }
216
217 des_sparc64_key_expand((const u32 *)key, k1);
218 key += DES_KEY_SIZE;
219 des_sparc64_key_expand((const u32 *)key, k2);
220 key += DES_KEY_SIZE;
221 des_sparc64_key_expand((const u32 *)key, k3);
222
223 memcpy(&dctx->encrypt_expkey[0], &k1[0], sizeof(k1));
224 encrypt_to_decrypt(&dctx->encrypt_expkey[DES_EXPKEY_WORDS / 2], &k2[0]);
225 memcpy(&dctx->encrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2],
226 &k3[0], sizeof(k3));
227
228 encrypt_to_decrypt(&dctx->decrypt_expkey[0], &k3[0]);
229 memcpy(&dctx->decrypt_expkey[DES_EXPKEY_WORDS / 2],
230 &k2[0], sizeof(k2));
231 encrypt_to_decrypt(&dctx->decrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2],
232 &k1[0]);
233
234 return 0;
235}
236
237extern void des3_ede_sparc64_crypt(const u64 *key, const u64 *input,
238 u64 *output);
239
240static void des3_ede_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
241{
242 struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
243 const u64 *K = ctx->encrypt_expkey;
244
245 des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
246}
247
248static void des3_ede_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
249{
250 struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
251 const u64 *K = ctx->decrypt_expkey;
252
253 des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
254}
255
256extern void des3_ede_sparc64_load_keys(const u64 *key);
257
258extern void des3_ede_sparc64_ecb_crypt(const u64 *expkey, const u64 *input,
259 u64 *output, unsigned int len);
260
261static int __ecb3_crypt(struct blkcipher_desc *desc,
262 struct scatterlist *dst, struct scatterlist *src,
263 unsigned int nbytes, bool encrypt)
264{
265 struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
266 struct blkcipher_walk walk;
267 const u64 *K;
268 int err;
269
270 blkcipher_walk_init(&walk, dst, src, nbytes);
271 err = blkcipher_walk_virt(desc, &walk);
272 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
273
274 if (encrypt)
275 K = &ctx->encrypt_expkey[0];
276 else
277 K = &ctx->decrypt_expkey[0];
278 des3_ede_sparc64_load_keys(K);
279 while ((nbytes = walk.nbytes)) {
280 unsigned int block_len = nbytes & DES_BLOCK_MASK;
281
282 if (likely(block_len)) {
283 const u64 *src64 = (const u64 *)walk.src.virt.addr;
284 des3_ede_sparc64_ecb_crypt(K, src64,
285 (u64 *) walk.dst.virt.addr,
286 block_len);
287 }
288 nbytes &= DES_BLOCK_SIZE - 1;
289 err = blkcipher_walk_done(desc, &walk, nbytes);
290 }
291 fprs_write(0);
292 return err;
293}
294
295static int ecb3_encrypt(struct blkcipher_desc *desc,
296 struct scatterlist *dst, struct scatterlist *src,
297 unsigned int nbytes)
298{
299 return __ecb3_crypt(desc, dst, src, nbytes, true);
300}
301
302static int ecb3_decrypt(struct blkcipher_desc *desc,
303 struct scatterlist *dst, struct scatterlist *src,
304 unsigned int nbytes)
305{
306 return __ecb3_crypt(desc, dst, src, nbytes, false);
307}
308
309extern void des3_ede_sparc64_cbc_encrypt(const u64 *expkey, const u64 *input,
310 u64 *output, unsigned int len,
311 u64 *iv);
312
313static int cbc3_encrypt(struct blkcipher_desc *desc,
314 struct scatterlist *dst, struct scatterlist *src,
315 unsigned int nbytes)
316{
317 struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
318 struct blkcipher_walk walk;
319 const u64 *K;
320 int err;
321
322 blkcipher_walk_init(&walk, dst, src, nbytes);
323 err = blkcipher_walk_virt(desc, &walk);
324 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
325
326 K = &ctx->encrypt_expkey[0];
327 des3_ede_sparc64_load_keys(K);
328 while ((nbytes = walk.nbytes)) {
329 unsigned int block_len = nbytes & DES_BLOCK_MASK;
330
331 if (likely(block_len)) {
332 const u64 *src64 = (const u64 *)walk.src.virt.addr;
333 des3_ede_sparc64_cbc_encrypt(K, src64,
334 (u64 *) walk.dst.virt.addr,
335 block_len,
336 (u64 *) walk.iv);
337 }
338 nbytes &= DES_BLOCK_SIZE - 1;
339 err = blkcipher_walk_done(desc, &walk, nbytes);
340 }
341 fprs_write(0);
342 return err;
343}
344
345extern void des3_ede_sparc64_cbc_decrypt(const u64 *expkey, const u64 *input,
346 u64 *output, unsigned int len,
347 u64 *iv);
348
349static int cbc3_decrypt(struct blkcipher_desc *desc,
350 struct scatterlist *dst, struct scatterlist *src,
351 unsigned int nbytes)
352{
353 struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
354 struct blkcipher_walk walk;
355 const u64 *K;
356 int err;
357
358 blkcipher_walk_init(&walk, dst, src, nbytes);
359 err = blkcipher_walk_virt(desc, &walk);
360 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
361
362 K = &ctx->decrypt_expkey[0];
363 des3_ede_sparc64_load_keys(K);
364 while ((nbytes = walk.nbytes)) {
365 unsigned int block_len = nbytes & DES_BLOCK_MASK;
366
367 if (likely(block_len)) {
368 const u64 *src64 = (const u64 *)walk.src.virt.addr;
369 des3_ede_sparc64_cbc_decrypt(K, src64,
370 (u64 *) walk.dst.virt.addr,
371 block_len,
372 (u64 *) walk.iv);
373 }
374 nbytes &= DES_BLOCK_SIZE - 1;
375 err = blkcipher_walk_done(desc, &walk, nbytes);
376 }
377 fprs_write(0);
378 return err;
379}
380
381static struct crypto_alg algs[] = { {
382 .cra_name = "des",
383 .cra_driver_name = "des-sparc64",
384 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
385 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
386 .cra_blocksize = DES_BLOCK_SIZE,
387 .cra_ctxsize = sizeof(struct des_sparc64_ctx),
388 .cra_alignmask = 7,
389 .cra_module = THIS_MODULE,
390 .cra_u = {
391 .cipher = {
392 .cia_min_keysize = DES_KEY_SIZE,
393 .cia_max_keysize = DES_KEY_SIZE,
394 .cia_setkey = des_set_key,
395 .cia_encrypt = des_encrypt,
396 .cia_decrypt = des_decrypt
397 }
398 }
399}, {
400 .cra_name = "ecb(des)",
401 .cra_driver_name = "ecb-des-sparc64",
402 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
403 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
404 .cra_blocksize = DES_BLOCK_SIZE,
405 .cra_ctxsize = sizeof(struct des_sparc64_ctx),
406 .cra_alignmask = 7,
407 .cra_type = &crypto_blkcipher_type,
408 .cra_module = THIS_MODULE,
409 .cra_u = {
410 .blkcipher = {
411 .min_keysize = DES_KEY_SIZE,
412 .max_keysize = DES_KEY_SIZE,
413 .setkey = des_set_key,
414 .encrypt = ecb_encrypt,
415 .decrypt = ecb_decrypt,
416 },
417 },
418}, {
419 .cra_name = "cbc(des)",
420 .cra_driver_name = "cbc-des-sparc64",
421 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
422 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
423 .cra_blocksize = DES_BLOCK_SIZE,
424 .cra_ctxsize = sizeof(struct des_sparc64_ctx),
425 .cra_alignmask = 7,
426 .cra_type = &crypto_blkcipher_type,
427 .cra_module = THIS_MODULE,
428 .cra_u = {
429 .blkcipher = {
430 .min_keysize = DES_KEY_SIZE,
431 .max_keysize = DES_KEY_SIZE,
432 .setkey = des_set_key,
433 .encrypt = cbc_encrypt,
434 .decrypt = cbc_decrypt,
435 },
436 },
437}, {
438 .cra_name = "des3_ede",
439 .cra_driver_name = "des3_ede-sparc64",
440 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
441 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
442 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
443 .cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx),
444 .cra_alignmask = 7,
445 .cra_module = THIS_MODULE,
446 .cra_u = {
447 .cipher = {
448 .cia_min_keysize = DES3_EDE_KEY_SIZE,
449 .cia_max_keysize = DES3_EDE_KEY_SIZE,
450 .cia_setkey = des3_ede_set_key,
451 .cia_encrypt = des3_ede_encrypt,
452 .cia_decrypt = des3_ede_decrypt
453 }
454 }
455}, {
456 .cra_name = "ecb(des3_ede)",
457 .cra_driver_name = "ecb-des3_ede-sparc64",
458 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
459 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
460 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
461 .cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx),
462 .cra_alignmask = 7,
463 .cra_type = &crypto_blkcipher_type,
464 .cra_module = THIS_MODULE,
465 .cra_u = {
466 .blkcipher = {
467 .min_keysize = DES3_EDE_KEY_SIZE,
468 .max_keysize = DES3_EDE_KEY_SIZE,
469 .setkey = des3_ede_set_key,
470 .encrypt = ecb3_encrypt,
471 .decrypt = ecb3_decrypt,
472 },
473 },
474}, {
475 .cra_name = "cbc(des3_ede)",
476 .cra_driver_name = "cbc-des3_ede-sparc64",
477 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
478 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
479 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
480 .cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx),
481 .cra_alignmask = 7,
482 .cra_type = &crypto_blkcipher_type,
483 .cra_module = THIS_MODULE,
484 .cra_u = {
485 .blkcipher = {
486 .min_keysize = DES3_EDE_KEY_SIZE,
487 .max_keysize = DES3_EDE_KEY_SIZE,
488 .setkey = des3_ede_set_key,
489 .encrypt = cbc3_encrypt,
490 .decrypt = cbc3_decrypt,
491 },
492 },
493} };
494
495static bool __init sparc64_has_des_opcode(void)
496{
497 unsigned long cfr;
498
499 if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
500 return false;
501
502 __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
503 if (!(cfr & CFR_DES))
504 return false;
505
506 return true;
507}
508
509static int __init des_sparc64_mod_init(void)
510{
511 int i;
512
513 for (i = 0; i < ARRAY_SIZE(algs); i++)
514 INIT_LIST_HEAD(&algs[i].cra_list);
515
516 if (sparc64_has_des_opcode()) {
517 pr_info("Using sparc64 des opcodes optimized DES implementation\n");
518 return crypto_register_algs(algs, ARRAY_SIZE(algs));
519 }
520 pr_info("sparc64 des opcodes not available.\n");
521 return -ENODEV;
522}
523
524static void __exit des_sparc64_mod_fini(void)
525{
526 crypto_unregister_algs(algs, ARRAY_SIZE(algs));
527}
528
529module_init(des_sparc64_mod_init);
530module_exit(des_sparc64_mod_fini);
531
532MODULE_LICENSE("GPL");
533MODULE_DESCRIPTION("DES & Triple DES EDE Cipher Algorithms, sparc64 des opcode accelerated");
534
535MODULE_ALIAS("des");
536
537#include "crop_devid.c"
538
