2 * linux/arch/arm64/crypto/aes-glue.c - wrapper code for ARMv8 AES
4 * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
12 #include <asm/hwcap.h>
13 #include <crypto/aes.h>
14 #include <crypto/internal/hash.h>
15 #include <crypto/internal/simd.h>
16 #include <crypto/internal/skcipher.h>
17 #include <linux/module.h>
18 #include <linux/cpufeature.h>
19 #include <crypto/xts.h>
21 #include "aes-ce-setkey.h"
23 #ifdef USE_V8_CRYPTO_EXTENSIONS
26 #define aes_setkey ce_aes_setkey
27 #define aes_expandkey ce_aes_expandkey
28 #define aes_ecb_encrypt ce_aes_ecb_encrypt
29 #define aes_ecb_decrypt ce_aes_ecb_decrypt
30 #define aes_cbc_encrypt ce_aes_cbc_encrypt
31 #define aes_cbc_decrypt ce_aes_cbc_decrypt
32 #define aes_ctr_encrypt ce_aes_ctr_encrypt
33 #define aes_xts_encrypt ce_aes_xts_encrypt
34 #define aes_xts_decrypt ce_aes_xts_decrypt
35 #define aes_mac_update ce_aes_mac_update
36 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions");
40 #define aes_setkey crypto_aes_set_key
41 #define aes_expandkey crypto_aes_expand_key
42 #define aes_ecb_encrypt neon_aes_ecb_encrypt
43 #define aes_ecb_decrypt neon_aes_ecb_decrypt
44 #define aes_cbc_encrypt neon_aes_cbc_encrypt
45 #define aes_cbc_decrypt neon_aes_cbc_decrypt
46 #define aes_ctr_encrypt neon_aes_ctr_encrypt
47 #define aes_xts_encrypt neon_aes_xts_encrypt
48 #define aes_xts_decrypt neon_aes_xts_decrypt
49 #define aes_mac_update neon_aes_mac_update
50 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 NEON");
51 MODULE_ALIAS_CRYPTO("ecb(aes)");
52 MODULE_ALIAS_CRYPTO("cbc(aes)");
53 MODULE_ALIAS_CRYPTO("ctr(aes)");
54 MODULE_ALIAS_CRYPTO("xts(aes)");
55 MODULE_ALIAS_CRYPTO("cmac(aes)");
56 MODULE_ALIAS_CRYPTO("xcbc(aes)");
57 MODULE_ALIAS_CRYPTO("cbcmac(aes)");
60 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
61 MODULE_LICENSE("GPL v2");
63 /* defined in aes-modes.S */
64 asmlinkage void aes_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[],
65 int rounds, int blocks, int first);
66 asmlinkage void aes_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[],
67 int rounds, int blocks, int first);
69 asmlinkage void aes_cbc_encrypt(u8 out[], u8 const in[], u8 const rk[],
70 int rounds, int blocks, u8 iv[], int first);
71 asmlinkage void aes_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[],
72 int rounds, int blocks, u8 iv[], int first);
74 asmlinkage void aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[],
75 int rounds, int blocks, u8 ctr[], int first);
77 asmlinkage void aes_xts_encrypt(u8 out[], u8 const in[], u8 const rk1[],
78 int rounds, int blocks, u8 const rk2[], u8 iv[],
80 asmlinkage void aes_xts_decrypt(u8 out[], u8 const in[], u8 const rk1[],
81 int rounds, int blocks, u8 const rk2[], u8 iv[],
84 asmlinkage void aes_mac_update(u8 const in[], u32 const rk[], int rounds,
85 int blocks, u8 dg[], int enc_before,
88 struct crypto_aes_xts_ctx {
89 struct crypto_aes_ctx key1;
90 struct crypto_aes_ctx __aligned(8) key2;
94 struct crypto_aes_ctx key;
95 u8 __aligned(8) consts[];
100 u8 dg[AES_BLOCK_SIZE];
103 static int skcipher_aes_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
104 unsigned int key_len)
106 return aes_setkey(crypto_skcipher_tfm(tfm), in_key, key_len);
109 static int xts_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
110 unsigned int key_len)
112 struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
115 ret = xts_verify_key(tfm, in_key, key_len);
119 ret = aes_expandkey(&ctx->key1, in_key, key_len / 2);
121 ret = aes_expandkey(&ctx->key2, &in_key[key_len / 2],
126 crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
130 static int ecb_encrypt(struct skcipher_request *req)
132 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
133 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
134 int err, first, rounds = 6 + ctx->key_length / 4;
135 struct skcipher_walk walk;
138 err = skcipher_walk_virt(&walk, req, true);
141 for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
142 aes_ecb_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
143 (u8 *)ctx->key_enc, rounds, blocks, first);
144 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
150 static int ecb_decrypt(struct skcipher_request *req)
152 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
153 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
154 int err, first, rounds = 6 + ctx->key_length / 4;
155 struct skcipher_walk walk;
158 err = skcipher_walk_virt(&walk, req, true);
161 for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
162 aes_ecb_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
163 (u8 *)ctx->key_dec, rounds, blocks, first);
164 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
170 static int cbc_encrypt(struct skcipher_request *req)
172 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
173 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
174 int err, first, rounds = 6 + ctx->key_length / 4;
175 struct skcipher_walk walk;
178 err = skcipher_walk_virt(&walk, req, true);
181 for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
182 aes_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
183 (u8 *)ctx->key_enc, rounds, blocks, walk.iv,
185 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
191 static int cbc_decrypt(struct skcipher_request *req)
193 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
194 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
195 int err, first, rounds = 6 + ctx->key_length / 4;
196 struct skcipher_walk walk;
199 err = skcipher_walk_virt(&walk, req, true);
202 for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
203 aes_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
204 (u8 *)ctx->key_dec, rounds, blocks, walk.iv,
206 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
212 static int ctr_encrypt(struct skcipher_request *req)
214 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
215 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
216 int err, first, rounds = 6 + ctx->key_length / 4;
217 struct skcipher_walk walk;
220 err = skcipher_walk_virt(&walk, req, true);
224 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
225 aes_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
226 (u8 *)ctx->key_enc, rounds, blocks, walk.iv,
228 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
232 u8 __aligned(8) tail[AES_BLOCK_SIZE];
233 unsigned int nbytes = walk.nbytes;
234 u8 *tdst = walk.dst.virt.addr;
235 u8 *tsrc = walk.src.virt.addr;
238 * Tell aes_ctr_encrypt() to process a tail block.
242 aes_ctr_encrypt(tail, NULL, (u8 *)ctx->key_enc, rounds,
243 blocks, walk.iv, first);
244 crypto_xor_cpy(tdst, tsrc, tail, nbytes);
245 err = skcipher_walk_done(&walk, 0);
252 static int xts_encrypt(struct skcipher_request *req)
254 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
255 struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
256 int err, first, rounds = 6 + ctx->key1.key_length / 4;
257 struct skcipher_walk walk;
260 err = skcipher_walk_virt(&walk, req, true);
263 for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
264 aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
265 (u8 *)ctx->key1.key_enc, rounds, blocks,
266 (u8 *)ctx->key2.key_enc, walk.iv, first);
267 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
274 static int xts_decrypt(struct skcipher_request *req)
276 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
277 struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
278 int err, first, rounds = 6 + ctx->key1.key_length / 4;
279 struct skcipher_walk walk;
282 err = skcipher_walk_virt(&walk, req, true);
285 for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
286 aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
287 (u8 *)ctx->key1.key_dec, rounds, blocks,
288 (u8 *)ctx->key2.key_enc, walk.iv, first);
289 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
296 static struct skcipher_alg aes_algs[] = { {
298 .cra_name = "__ecb(aes)",
299 .cra_driver_name = "__ecb-aes-" MODE,
300 .cra_priority = PRIO,
301 .cra_flags = CRYPTO_ALG_INTERNAL,
302 .cra_blocksize = AES_BLOCK_SIZE,
303 .cra_ctxsize = sizeof(struct crypto_aes_ctx),
304 .cra_module = THIS_MODULE,
306 .min_keysize = AES_MIN_KEY_SIZE,
307 .max_keysize = AES_MAX_KEY_SIZE,
308 .setkey = skcipher_aes_setkey,
309 .encrypt = ecb_encrypt,
310 .decrypt = ecb_decrypt,
313 .cra_name = "__cbc(aes)",
314 .cra_driver_name = "__cbc-aes-" MODE,
315 .cra_priority = PRIO,
316 .cra_flags = CRYPTO_ALG_INTERNAL,
317 .cra_blocksize = AES_BLOCK_SIZE,
318 .cra_ctxsize = sizeof(struct crypto_aes_ctx),
319 .cra_module = THIS_MODULE,
321 .min_keysize = AES_MIN_KEY_SIZE,
322 .max_keysize = AES_MAX_KEY_SIZE,
323 .ivsize = AES_BLOCK_SIZE,
324 .setkey = skcipher_aes_setkey,
325 .encrypt = cbc_encrypt,
326 .decrypt = cbc_decrypt,
329 .cra_name = "__ctr(aes)",
330 .cra_driver_name = "__ctr-aes-" MODE,
331 .cra_priority = PRIO,
332 .cra_flags = CRYPTO_ALG_INTERNAL,
334 .cra_ctxsize = sizeof(struct crypto_aes_ctx),
335 .cra_module = THIS_MODULE,
337 .min_keysize = AES_MIN_KEY_SIZE,
338 .max_keysize = AES_MAX_KEY_SIZE,
339 .ivsize = AES_BLOCK_SIZE,
340 .chunksize = AES_BLOCK_SIZE,
341 .setkey = skcipher_aes_setkey,
342 .encrypt = ctr_encrypt,
343 .decrypt = ctr_encrypt,
346 .cra_name = "ctr(aes)",
347 .cra_driver_name = "ctr-aes-" MODE,
348 .cra_priority = PRIO - 1,
350 .cra_ctxsize = sizeof(struct crypto_aes_ctx),
351 .cra_module = THIS_MODULE,
353 .min_keysize = AES_MIN_KEY_SIZE,
354 .max_keysize = AES_MAX_KEY_SIZE,
355 .ivsize = AES_BLOCK_SIZE,
356 .chunksize = AES_BLOCK_SIZE,
357 .setkey = skcipher_aes_setkey,
358 .encrypt = ctr_encrypt,
359 .decrypt = ctr_encrypt,
362 .cra_name = "__xts(aes)",
363 .cra_driver_name = "__xts-aes-" MODE,
364 .cra_priority = PRIO,
365 .cra_flags = CRYPTO_ALG_INTERNAL,
366 .cra_blocksize = AES_BLOCK_SIZE,
367 .cra_ctxsize = sizeof(struct crypto_aes_xts_ctx),
368 .cra_module = THIS_MODULE,
370 .min_keysize = 2 * AES_MIN_KEY_SIZE,
371 .max_keysize = 2 * AES_MAX_KEY_SIZE,
372 .ivsize = AES_BLOCK_SIZE,
373 .setkey = xts_set_key,
374 .encrypt = xts_encrypt,
375 .decrypt = xts_decrypt,
378 static int cbcmac_setkey(struct crypto_shash *tfm, const u8 *in_key,
379 unsigned int key_len)
381 struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
384 err = aes_expandkey(&ctx->key, in_key, key_len);
386 crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
391 static void cmac_gf128_mul_by_x(be128 *y, const be128 *x)
393 u64 a = be64_to_cpu(x->a);
394 u64 b = be64_to_cpu(x->b);
396 y->a = cpu_to_be64((a << 1) | (b >> 63));
397 y->b = cpu_to_be64((b << 1) ^ ((a >> 63) ? 0x87 : 0));
400 static int cmac_setkey(struct crypto_shash *tfm, const u8 *in_key,
401 unsigned int key_len)
403 struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
404 be128 *consts = (be128 *)ctx->consts;
405 u8 *rk = (u8 *)ctx->key.key_enc;
406 int rounds = 6 + key_len / 4;
409 err = cbcmac_setkey(tfm, in_key, key_len);
413 /* encrypt the zero vector */
415 aes_ecb_encrypt(ctx->consts, (u8[AES_BLOCK_SIZE]){}, rk, rounds, 1, 1);
418 cmac_gf128_mul_by_x(consts, consts);
419 cmac_gf128_mul_by_x(consts + 1, consts);
424 static int xcbc_setkey(struct crypto_shash *tfm, const u8 *in_key,
425 unsigned int key_len)
427 static u8 const ks[3][AES_BLOCK_SIZE] = {
428 { [0 ... AES_BLOCK_SIZE - 1] = 0x1 },
429 { [0 ... AES_BLOCK_SIZE - 1] = 0x2 },
430 { [0 ... AES_BLOCK_SIZE - 1] = 0x3 },
433 struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
434 u8 *rk = (u8 *)ctx->key.key_enc;
435 int rounds = 6 + key_len / 4;
436 u8 key[AES_BLOCK_SIZE];
439 err = cbcmac_setkey(tfm, in_key, key_len);
444 aes_ecb_encrypt(key, ks[0], rk, rounds, 1, 1);
445 aes_ecb_encrypt(ctx->consts, ks[1], rk, rounds, 2, 0);
448 return cbcmac_setkey(tfm, key, sizeof(key));
451 static int mac_init(struct shash_desc *desc)
453 struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
455 memset(ctx->dg, 0, AES_BLOCK_SIZE);
461 static int mac_update(struct shash_desc *desc, const u8 *p, unsigned int len)
463 struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
464 struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
465 int rounds = 6 + tctx->key.key_length / 4;
470 if ((ctx->len % AES_BLOCK_SIZE) == 0 &&
471 (ctx->len + len) > AES_BLOCK_SIZE) {
473 int blocks = len / AES_BLOCK_SIZE;
475 len %= AES_BLOCK_SIZE;
478 aes_mac_update(p, tctx->key.key_enc, rounds, blocks,
479 ctx->dg, (ctx->len != 0), (len != 0));
482 p += blocks * AES_BLOCK_SIZE;
485 ctx->len = AES_BLOCK_SIZE;
491 l = min(len, AES_BLOCK_SIZE - ctx->len);
493 if (l <= AES_BLOCK_SIZE) {
494 crypto_xor(ctx->dg + ctx->len, p, l);
504 static int cbcmac_final(struct shash_desc *desc, u8 *out)
506 struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
507 struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
508 int rounds = 6 + tctx->key.key_length / 4;
511 aes_mac_update(NULL, tctx->key.key_enc, rounds, 0, ctx->dg, 1, 0);
514 memcpy(out, ctx->dg, AES_BLOCK_SIZE);
519 static int cmac_final(struct shash_desc *desc, u8 *out)
521 struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
522 struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
523 int rounds = 6 + tctx->key.key_length / 4;
524 u8 *consts = tctx->consts;
526 if (ctx->len != AES_BLOCK_SIZE) {
527 ctx->dg[ctx->len] ^= 0x80;
528 consts += AES_BLOCK_SIZE;
532 aes_mac_update(consts, tctx->key.key_enc, rounds, 1, ctx->dg, 0, 1);
535 memcpy(out, ctx->dg, AES_BLOCK_SIZE);
540 static struct shash_alg mac_algs[] = { {
541 .base.cra_name = "cmac(aes)",
542 .base.cra_driver_name = "cmac-aes-" MODE,
543 .base.cra_priority = PRIO,
544 .base.cra_flags = CRYPTO_ALG_TYPE_SHASH,
545 .base.cra_blocksize = AES_BLOCK_SIZE,
546 .base.cra_ctxsize = sizeof(struct mac_tfm_ctx) +
548 .base.cra_module = THIS_MODULE,
550 .digestsize = AES_BLOCK_SIZE,
552 .update = mac_update,
554 .setkey = cmac_setkey,
555 .descsize = sizeof(struct mac_desc_ctx),
557 .base.cra_name = "xcbc(aes)",
558 .base.cra_driver_name = "xcbc-aes-" MODE,
559 .base.cra_priority = PRIO,
560 .base.cra_flags = CRYPTO_ALG_TYPE_SHASH,
561 .base.cra_blocksize = AES_BLOCK_SIZE,
562 .base.cra_ctxsize = sizeof(struct mac_tfm_ctx) +
564 .base.cra_module = THIS_MODULE,
566 .digestsize = AES_BLOCK_SIZE,
568 .update = mac_update,
570 .setkey = xcbc_setkey,
571 .descsize = sizeof(struct mac_desc_ctx),
573 .base.cra_name = "cbcmac(aes)",
574 .base.cra_driver_name = "cbcmac-aes-" MODE,
575 .base.cra_priority = PRIO,
576 .base.cra_flags = CRYPTO_ALG_TYPE_SHASH,
577 .base.cra_blocksize = 1,
578 .base.cra_ctxsize = sizeof(struct mac_tfm_ctx),
579 .base.cra_module = THIS_MODULE,
581 .digestsize = AES_BLOCK_SIZE,
583 .update = mac_update,
584 .final = cbcmac_final,
585 .setkey = cbcmac_setkey,
586 .descsize = sizeof(struct mac_desc_ctx),
589 static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)];
591 static void aes_exit(void)
595 for (i = 0; i < ARRAY_SIZE(aes_simd_algs); i++)
596 if (aes_simd_algs[i])
597 simd_skcipher_free(aes_simd_algs[i]);
599 crypto_unregister_shashes(mac_algs, ARRAY_SIZE(mac_algs));
600 crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
603 static int __init aes_init(void)
605 struct simd_skcipher_alg *simd;
606 const char *basename;
612 err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
616 err = crypto_register_shashes(mac_algs, ARRAY_SIZE(mac_algs));
618 goto unregister_ciphers;
620 for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
621 if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL))
624 algname = aes_algs[i].base.cra_name + 2;
625 drvname = aes_algs[i].base.cra_driver_name + 2;
626 basename = aes_algs[i].base.cra_driver_name;
627 simd = simd_skcipher_create_compat(algname, drvname, basename);
630 goto unregister_simds;
632 aes_simd_algs[i] = simd;
640 crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
644 #ifdef USE_V8_CRYPTO_EXTENSIONS
645 module_cpu_feature_match(AES, aes_init);
647 module_init(aes_init);
648 EXPORT_SYMBOL(neon_aes_ecb_encrypt);
649 EXPORT_SYMBOL(neon_aes_cbc_encrypt);
651 module_exit(aes_exit);