1 // SPDX-License-Identifier: GPL-2.0+
3 * Freescale FSL CAAM support for crypto API over QI backend.
6 * Copyright 2013-2016 Freescale Semiconductor, Inc.
7 * Copyright 2016-2019 NXP
14 #include "desc_constr.h"
20 #include "caamalg_desc.h"
25 #define CAAM_CRA_PRIORITY 2000
26 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */
27 #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \
28 SHA512_DIGEST_SIZE * 2)
30 #define DESC_MAX_USED_BYTES (DESC_QI_AEAD_GIVENC_LEN + \
32 #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ)
34 struct caam_alg_entry {
42 struct caam_aead_alg {
44 struct caam_alg_entry caam;
48 struct caam_skcipher_alg {
49 struct skcipher_alg skcipher;
50 struct caam_alg_entry caam;
59 u32 sh_desc_enc[DESC_MAX_USED_LEN];
60 u32 sh_desc_dec[DESC_MAX_USED_LEN];
61 u8 key[CAAM_MAX_KEY_SIZE];
63 enum dma_data_direction dir;
66 unsigned int authsize;
68 spinlock_t lock; /* Protects multiple init of driver context */
69 struct caam_drv_ctx *drv_ctx[NUM_OP];
72 static int aead_set_sh_desc(struct crypto_aead *aead)
74 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
76 struct caam_ctx *ctx = crypto_aead_ctx(aead);
77 unsigned int ivsize = crypto_aead_ivsize(aead);
80 unsigned int data_len[2];
82 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
83 OP_ALG_AAI_CTR_MOD128);
84 const bool is_rfc3686 = alg->caam.rfc3686;
85 struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent);
87 if (!ctx->cdata.keylen || !ctx->authsize)
91 * AES-CTR needs to load IV in CONTEXT1 reg
92 * at an offset of 128bits (16bytes)
93 * CONTEXT1[255:128] = IV
100 * CONTEXT1[255:128] = {NONCE, IV, COUNTER}
103 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
104 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad +
105 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE);
109 * In case |user key| > |derived key|, using DKP<imm,imm> would result
110 * in invalid opcodes (last bytes of user key) in the resulting
111 * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key
112 * addresses are needed.
114 ctx->adata.key_virt = ctx->key;
115 ctx->adata.key_dma = ctx->key_dma;
117 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
118 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
120 data_len[0] = ctx->adata.keylen_pad;
121 data_len[1] = ctx->cdata.keylen;
126 /* aead_encrypt shared descriptor */
127 if (desc_inline_query(DESC_QI_AEAD_ENC_LEN +
128 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
129 DESC_JOB_IO_LEN, data_len, &inl_mask,
130 ARRAY_SIZE(data_len)) < 0)
133 ctx->adata.key_inline = !!(inl_mask & 1);
134 ctx->cdata.key_inline = !!(inl_mask & 2);
136 cnstr_shdsc_aead_encap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata,
137 ivsize, ctx->authsize, is_rfc3686, nonce,
138 ctx1_iv_off, true, ctrlpriv->era);
141 /* aead_decrypt shared descriptor */
142 if (desc_inline_query(DESC_QI_AEAD_DEC_LEN +
143 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
144 DESC_JOB_IO_LEN, data_len, &inl_mask,
145 ARRAY_SIZE(data_len)) < 0)
148 ctx->adata.key_inline = !!(inl_mask & 1);
149 ctx->cdata.key_inline = !!(inl_mask & 2);
151 cnstr_shdsc_aead_decap(ctx->sh_desc_dec, &ctx->cdata, &ctx->adata,
152 ivsize, ctx->authsize, alg->caam.geniv,
153 is_rfc3686, nonce, ctx1_iv_off, true,
156 if (!alg->caam.geniv)
159 /* aead_givencrypt shared descriptor */
160 if (desc_inline_query(DESC_QI_AEAD_GIVENC_LEN +
161 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
162 DESC_JOB_IO_LEN, data_len, &inl_mask,
163 ARRAY_SIZE(data_len)) < 0)
166 ctx->adata.key_inline = !!(inl_mask & 1);
167 ctx->cdata.key_inline = !!(inl_mask & 2);
169 cnstr_shdsc_aead_givencap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata,
170 ivsize, ctx->authsize, is_rfc3686, nonce,
171 ctx1_iv_off, true, ctrlpriv->era);
177 static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
179 struct caam_ctx *ctx = crypto_aead_ctx(authenc);
181 ctx->authsize = authsize;
182 aead_set_sh_desc(authenc);
187 static int aead_setkey(struct crypto_aead *aead, const u8 *key,
190 struct caam_ctx *ctx = crypto_aead_ctx(aead);
191 struct device *jrdev = ctx->jrdev;
192 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
193 struct crypto_authenc_keys keys;
196 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
199 dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n",
200 keys.authkeylen + keys.enckeylen, keys.enckeylen,
202 print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
203 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
206 * If DKP is supported, use it in the shared descriptor to generate
209 if (ctrlpriv->era >= 6) {
210 ctx->adata.keylen = keys.authkeylen;
211 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
214 if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE)
217 memcpy(ctx->key, keys.authkey, keys.authkeylen);
218 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey,
220 dma_sync_single_for_device(jrdev->parent, ctx->key_dma,
221 ctx->adata.keylen_pad +
222 keys.enckeylen, ctx->dir);
226 ret = gen_split_key(jrdev, ctx->key, &ctx->adata, keys.authkey,
227 keys.authkeylen, CAAM_MAX_KEY_SIZE -
232 /* postpend encryption key to auth split key */
233 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
234 dma_sync_single_for_device(jrdev->parent, ctx->key_dma,
235 ctx->adata.keylen_pad + keys.enckeylen,
238 print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ",
239 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
240 ctx->adata.keylen_pad + keys.enckeylen, 1);
243 ctx->cdata.keylen = keys.enckeylen;
245 ret = aead_set_sh_desc(aead);
249 /* Now update the driver contexts with the new shared descriptor */
250 if (ctx->drv_ctx[ENCRYPT]) {
251 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
254 dev_err(jrdev, "driver enc context update failed\n");
259 if (ctx->drv_ctx[DECRYPT]) {
260 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
263 dev_err(jrdev, "driver dec context update failed\n");
268 memzero_explicit(&keys, sizeof(keys));
271 memzero_explicit(&keys, sizeof(keys));
275 static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key,
278 struct crypto_authenc_keys keys;
281 err = crypto_authenc_extractkeys(&keys, key, keylen);
285 err = verify_aead_des3_key(aead, keys.enckey, keys.enckeylen) ?:
286 aead_setkey(aead, key, keylen);
288 memzero_explicit(&keys, sizeof(keys));
292 static int gcm_set_sh_desc(struct crypto_aead *aead)
294 struct caam_ctx *ctx = crypto_aead_ctx(aead);
295 unsigned int ivsize = crypto_aead_ivsize(aead);
296 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
299 if (!ctx->cdata.keylen || !ctx->authsize)
303 * Job Descriptor and Shared Descriptor
304 * must fit into the 64-word Descriptor h/w Buffer
306 if (rem_bytes >= DESC_QI_GCM_ENC_LEN) {
307 ctx->cdata.key_inline = true;
308 ctx->cdata.key_virt = ctx->key;
310 ctx->cdata.key_inline = false;
311 ctx->cdata.key_dma = ctx->key_dma;
314 cnstr_shdsc_gcm_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize,
315 ctx->authsize, true);
318 * Job Descriptor and Shared Descriptor
319 * must fit into the 64-word Descriptor h/w Buffer
321 if (rem_bytes >= DESC_QI_GCM_DEC_LEN) {
322 ctx->cdata.key_inline = true;
323 ctx->cdata.key_virt = ctx->key;
325 ctx->cdata.key_inline = false;
326 ctx->cdata.key_dma = ctx->key_dma;
329 cnstr_shdsc_gcm_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize,
330 ctx->authsize, true);
335 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
337 struct caam_ctx *ctx = crypto_aead_ctx(authenc);
340 err = crypto_gcm_check_authsize(authsize);
344 ctx->authsize = authsize;
345 gcm_set_sh_desc(authenc);
350 static int gcm_setkey(struct crypto_aead *aead,
351 const u8 *key, unsigned int keylen)
353 struct caam_ctx *ctx = crypto_aead_ctx(aead);
354 struct device *jrdev = ctx->jrdev;
357 ret = aes_check_keylen(keylen);
361 print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
362 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
364 memcpy(ctx->key, key, keylen);
365 dma_sync_single_for_device(jrdev->parent, ctx->key_dma, keylen,
367 ctx->cdata.keylen = keylen;
369 ret = gcm_set_sh_desc(aead);
373 /* Now update the driver contexts with the new shared descriptor */
374 if (ctx->drv_ctx[ENCRYPT]) {
375 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
378 dev_err(jrdev, "driver enc context update failed\n");
383 if (ctx->drv_ctx[DECRYPT]) {
384 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
387 dev_err(jrdev, "driver dec context update failed\n");
395 static int rfc4106_set_sh_desc(struct crypto_aead *aead)
397 struct caam_ctx *ctx = crypto_aead_ctx(aead);
398 unsigned int ivsize = crypto_aead_ivsize(aead);
399 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
402 if (!ctx->cdata.keylen || !ctx->authsize)
405 ctx->cdata.key_virt = ctx->key;
408 * Job Descriptor and Shared Descriptor
409 * must fit into the 64-word Descriptor h/w Buffer
411 if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) {
412 ctx->cdata.key_inline = true;
414 ctx->cdata.key_inline = false;
415 ctx->cdata.key_dma = ctx->key_dma;
418 cnstr_shdsc_rfc4106_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize,
419 ctx->authsize, true);
422 * Job Descriptor and Shared Descriptor
423 * must fit into the 64-word Descriptor h/w Buffer
425 if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) {
426 ctx->cdata.key_inline = true;
428 ctx->cdata.key_inline = false;
429 ctx->cdata.key_dma = ctx->key_dma;
432 cnstr_shdsc_rfc4106_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize,
433 ctx->authsize, true);
438 static int rfc4106_setauthsize(struct crypto_aead *authenc,
439 unsigned int authsize)
441 struct caam_ctx *ctx = crypto_aead_ctx(authenc);
444 err = crypto_rfc4106_check_authsize(authsize);
448 ctx->authsize = authsize;
449 rfc4106_set_sh_desc(authenc);
454 static int rfc4106_setkey(struct crypto_aead *aead,
455 const u8 *key, unsigned int keylen)
457 struct caam_ctx *ctx = crypto_aead_ctx(aead);
458 struct device *jrdev = ctx->jrdev;
461 ret = aes_check_keylen(keylen - 4);
465 print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
466 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
468 memcpy(ctx->key, key, keylen);
470 * The last four bytes of the key material are used as the salt value
471 * in the nonce. Update the AES key length.
473 ctx->cdata.keylen = keylen - 4;
474 dma_sync_single_for_device(jrdev->parent, ctx->key_dma,
475 ctx->cdata.keylen, ctx->dir);
477 ret = rfc4106_set_sh_desc(aead);
481 /* Now update the driver contexts with the new shared descriptor */
482 if (ctx->drv_ctx[ENCRYPT]) {
483 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
486 dev_err(jrdev, "driver enc context update failed\n");
491 if (ctx->drv_ctx[DECRYPT]) {
492 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
495 dev_err(jrdev, "driver dec context update failed\n");
503 static int rfc4543_set_sh_desc(struct crypto_aead *aead)
505 struct caam_ctx *ctx = crypto_aead_ctx(aead);
506 unsigned int ivsize = crypto_aead_ivsize(aead);
507 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
510 if (!ctx->cdata.keylen || !ctx->authsize)
513 ctx->cdata.key_virt = ctx->key;
516 * Job Descriptor and Shared Descriptor
517 * must fit into the 64-word Descriptor h/w Buffer
519 if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) {
520 ctx->cdata.key_inline = true;
522 ctx->cdata.key_inline = false;
523 ctx->cdata.key_dma = ctx->key_dma;
526 cnstr_shdsc_rfc4543_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize,
527 ctx->authsize, true);
530 * Job Descriptor and Shared Descriptor
531 * must fit into the 64-word Descriptor h/w Buffer
533 if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) {
534 ctx->cdata.key_inline = true;
536 ctx->cdata.key_inline = false;
537 ctx->cdata.key_dma = ctx->key_dma;
540 cnstr_shdsc_rfc4543_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize,
541 ctx->authsize, true);
546 static int rfc4543_setauthsize(struct crypto_aead *authenc,
547 unsigned int authsize)
549 struct caam_ctx *ctx = crypto_aead_ctx(authenc);
554 ctx->authsize = authsize;
555 rfc4543_set_sh_desc(authenc);
560 static int rfc4543_setkey(struct crypto_aead *aead,
561 const u8 *key, unsigned int keylen)
563 struct caam_ctx *ctx = crypto_aead_ctx(aead);
564 struct device *jrdev = ctx->jrdev;
567 ret = aes_check_keylen(keylen - 4);
571 print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
572 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
574 memcpy(ctx->key, key, keylen);
576 * The last four bytes of the key material are used as the salt value
577 * in the nonce. Update the AES key length.
579 ctx->cdata.keylen = keylen - 4;
580 dma_sync_single_for_device(jrdev->parent, ctx->key_dma,
581 ctx->cdata.keylen, ctx->dir);
583 ret = rfc4543_set_sh_desc(aead);
587 /* Now update the driver contexts with the new shared descriptor */
588 if (ctx->drv_ctx[ENCRYPT]) {
589 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
592 dev_err(jrdev, "driver enc context update failed\n");
597 if (ctx->drv_ctx[DECRYPT]) {
598 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
601 dev_err(jrdev, "driver dec context update failed\n");
609 static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
610 unsigned int keylen, const u32 ctx1_iv_off)
612 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
613 struct caam_skcipher_alg *alg =
614 container_of(crypto_skcipher_alg(skcipher), typeof(*alg),
616 struct device *jrdev = ctx->jrdev;
617 unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
618 const bool is_rfc3686 = alg->caam.rfc3686;
621 print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
622 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
624 ctx->cdata.keylen = keylen;
625 ctx->cdata.key_virt = key;
626 ctx->cdata.key_inline = true;
628 /* skcipher encrypt, decrypt shared descriptors */
629 cnstr_shdsc_skcipher_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize,
630 is_rfc3686, ctx1_iv_off);
631 cnstr_shdsc_skcipher_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize,
632 is_rfc3686, ctx1_iv_off);
634 /* Now update the driver contexts with the new shared descriptor */
635 if (ctx->drv_ctx[ENCRYPT]) {
636 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
639 dev_err(jrdev, "driver enc context update failed\n");
644 if (ctx->drv_ctx[DECRYPT]) {
645 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
648 dev_err(jrdev, "driver dec context update failed\n");
656 static int aes_skcipher_setkey(struct crypto_skcipher *skcipher,
657 const u8 *key, unsigned int keylen)
661 err = aes_check_keylen(keylen);
665 return skcipher_setkey(skcipher, key, keylen, 0);
668 static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher,
669 const u8 *key, unsigned int keylen)
676 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
677 * | *key = {KEY, NONCE}
679 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
680 keylen -= CTR_RFC3686_NONCE_SIZE;
682 err = aes_check_keylen(keylen);
686 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
689 static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher,
690 const u8 *key, unsigned int keylen)
696 * AES-CTR needs to load IV in CONTEXT1 reg
697 * at an offset of 128bits (16bytes)
698 * CONTEXT1[255:128] = IV
702 err = aes_check_keylen(keylen);
706 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
709 static int des3_skcipher_setkey(struct crypto_skcipher *skcipher,
710 const u8 *key, unsigned int keylen)
712 return verify_skcipher_des3_key(skcipher, key) ?:
713 skcipher_setkey(skcipher, key, keylen, 0);
716 static int des_skcipher_setkey(struct crypto_skcipher *skcipher,
717 const u8 *key, unsigned int keylen)
719 return verify_skcipher_des_key(skcipher, key) ?:
720 skcipher_setkey(skcipher, key, keylen, 0);
723 static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
726 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
727 struct device *jrdev = ctx->jrdev;
730 if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) {
731 dev_err(jrdev, "key size mismatch\n");
735 ctx->cdata.keylen = keylen;
736 ctx->cdata.key_virt = key;
737 ctx->cdata.key_inline = true;
739 /* xts skcipher encrypt, decrypt shared descriptors */
740 cnstr_shdsc_xts_skcipher_encap(ctx->sh_desc_enc, &ctx->cdata);
741 cnstr_shdsc_xts_skcipher_decap(ctx->sh_desc_dec, &ctx->cdata);
743 /* Now update the driver contexts with the new shared descriptor */
744 if (ctx->drv_ctx[ENCRYPT]) {
745 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
748 dev_err(jrdev, "driver enc context update failed\n");
753 if (ctx->drv_ctx[DECRYPT]) {
754 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
757 dev_err(jrdev, "driver dec context update failed\n");
766 * aead_edesc - s/w-extended aead descriptor
767 * @src_nents: number of segments in input scatterlist
768 * @dst_nents: number of segments in output scatterlist
769 * @iv_dma: dma address of iv for checking continuity and link table
770 * @qm_sg_bytes: length of dma mapped h/w link table
771 * @qm_sg_dma: bus physical mapped address of h/w link table
772 * @assoclen: associated data length, in CAAM endianness
773 * @assoclen_dma: bus physical mapped address of req->assoclen
774 * @drv_req: driver-specific request structure
775 * @sgt: the h/w link table, followed by IV
782 dma_addr_t qm_sg_dma;
783 unsigned int assoclen;
784 dma_addr_t assoclen_dma;
785 struct caam_drv_req drv_req;
786 struct qm_sg_entry sgt[0];
790 * skcipher_edesc - s/w-extended skcipher descriptor
791 * @src_nents: number of segments in input scatterlist
792 * @dst_nents: number of segments in output scatterlist
793 * @iv_dma: dma address of iv for checking continuity and link table
794 * @qm_sg_bytes: length of dma mapped h/w link table
795 * @qm_sg_dma: bus physical mapped address of h/w link table
796 * @drv_req: driver-specific request structure
797 * @sgt: the h/w link table, followed by IV
799 struct skcipher_edesc {
804 dma_addr_t qm_sg_dma;
805 struct caam_drv_req drv_req;
806 struct qm_sg_entry sgt[0];
809 static struct caam_drv_ctx *get_drv_ctx(struct caam_ctx *ctx,
813 * This function is called on the fast path with values of 'type'
814 * known at compile time. Invalid arguments are not expected and
815 * thus no checks are made.
817 struct caam_drv_ctx *drv_ctx = ctx->drv_ctx[type];
820 if (unlikely(!drv_ctx)) {
821 spin_lock(&ctx->lock);
823 /* Read again to check if some other core init drv_ctx */
824 drv_ctx = ctx->drv_ctx[type];
829 desc = ctx->sh_desc_enc;
830 else /* (type == DECRYPT) */
831 desc = ctx->sh_desc_dec;
833 cpu = smp_processor_id();
834 drv_ctx = caam_drv_ctx_init(ctx->qidev, &cpu, desc);
835 if (!IS_ERR_OR_NULL(drv_ctx))
836 drv_ctx->op_type = type;
838 ctx->drv_ctx[type] = drv_ctx;
841 spin_unlock(&ctx->lock);
847 static void caam_unmap(struct device *dev, struct scatterlist *src,
848 struct scatterlist *dst, int src_nents,
849 int dst_nents, dma_addr_t iv_dma, int ivsize,
850 enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma,
855 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
857 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
859 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
863 dma_unmap_single(dev, iv_dma, ivsize, iv_dir);
865 dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE);
868 static void aead_unmap(struct device *dev,
869 struct aead_edesc *edesc,
870 struct aead_request *req)
872 struct crypto_aead *aead = crypto_aead_reqtfm(req);
873 int ivsize = crypto_aead_ivsize(aead);
875 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
876 edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma,
878 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
881 static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc,
882 struct skcipher_request *req)
884 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
885 int ivsize = crypto_skcipher_ivsize(skcipher);
887 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
888 edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma,
892 static void aead_done(struct caam_drv_req *drv_req, u32 status)
894 struct device *qidev;
895 struct aead_edesc *edesc;
896 struct aead_request *aead_req = drv_req->app_ctx;
897 struct crypto_aead *aead = crypto_aead_reqtfm(aead_req);
898 struct caam_ctx *caam_ctx = crypto_aead_ctx(aead);
901 qidev = caam_ctx->qidev;
903 if (unlikely(status))
904 ecode = caam_jr_strstatus(qidev, status);
906 edesc = container_of(drv_req, typeof(*edesc), drv_req);
907 aead_unmap(qidev, edesc, aead_req);
909 aead_request_complete(aead_req, ecode);
910 qi_cache_free(edesc);
914 * allocate and map the aead extended descriptor
916 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
919 struct crypto_aead *aead = crypto_aead_reqtfm(req);
920 struct caam_ctx *ctx = crypto_aead_ctx(aead);
921 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
923 struct device *qidev = ctx->qidev;
924 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
925 GFP_KERNEL : GFP_ATOMIC;
926 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
927 int src_len, dst_len = 0;
928 struct aead_edesc *edesc;
929 dma_addr_t qm_sg_dma, iv_dma = 0;
931 unsigned int authsize = ctx->authsize;
932 int qm_sg_index = 0, qm_sg_ents = 0, qm_sg_bytes;
934 struct qm_sg_entry *sg_table, *fd_sgt;
935 struct caam_drv_ctx *drv_ctx;
937 drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT);
938 if (IS_ERR_OR_NULL(drv_ctx))
939 return (struct aead_edesc *)drv_ctx;
941 /* allocate space for base edesc and hw desc commands, link tables */
942 edesc = qi_cache_alloc(GFP_DMA | flags);
943 if (unlikely(!edesc)) {
944 dev_err(qidev, "could not allocate extended descriptor\n");
945 return ERR_PTR(-ENOMEM);
948 if (likely(req->src == req->dst)) {
949 src_len = req->assoclen + req->cryptlen +
950 (encrypt ? authsize : 0);
952 src_nents = sg_nents_for_len(req->src, src_len);
953 if (unlikely(src_nents < 0)) {
954 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n",
956 qi_cache_free(edesc);
957 return ERR_PTR(src_nents);
960 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents,
962 if (unlikely(!mapped_src_nents)) {
963 dev_err(qidev, "unable to map source\n");
964 qi_cache_free(edesc);
965 return ERR_PTR(-ENOMEM);
968 src_len = req->assoclen + req->cryptlen;
969 dst_len = src_len + (encrypt ? authsize : (-authsize));
971 src_nents = sg_nents_for_len(req->src, src_len);
972 if (unlikely(src_nents < 0)) {
973 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n",
975 qi_cache_free(edesc);
976 return ERR_PTR(src_nents);
979 dst_nents = sg_nents_for_len(req->dst, dst_len);
980 if (unlikely(dst_nents < 0)) {
981 dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n",
983 qi_cache_free(edesc);
984 return ERR_PTR(dst_nents);
988 mapped_src_nents = dma_map_sg(qidev, req->src,
989 src_nents, DMA_TO_DEVICE);
990 if (unlikely(!mapped_src_nents)) {
991 dev_err(qidev, "unable to map source\n");
992 qi_cache_free(edesc);
993 return ERR_PTR(-ENOMEM);
996 mapped_src_nents = 0;
1000 mapped_dst_nents = dma_map_sg(qidev, req->dst,
1003 if (unlikely(!mapped_dst_nents)) {
1004 dev_err(qidev, "unable to map destination\n");
1005 dma_unmap_sg(qidev, req->src, src_nents,
1007 qi_cache_free(edesc);
1008 return ERR_PTR(-ENOMEM);
1011 mapped_dst_nents = 0;
1015 if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv)
1016 ivsize = crypto_aead_ivsize(aead);
1019 * Create S/G table: req->assoclen, [IV,] req->src [, req->dst].
1020 * Input is not contiguous.
1021 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
1022 * the end of the table by allocating more S/G entries. Logic:
1023 * if (src != dst && output S/G)
1024 * pad output S/G, if needed
1025 * else if (src == dst && S/G)
1026 * overlapping S/Gs; pad one of them
1027 * else if (input S/G) ...
1028 * pad input S/G, if needed
1030 qm_sg_ents = 1 + !!ivsize + mapped_src_nents;
1031 if (mapped_dst_nents > 1)
1032 qm_sg_ents += pad_sg_nents(mapped_dst_nents);
1033 else if ((req->src == req->dst) && (mapped_src_nents > 1))
1034 qm_sg_ents = max(pad_sg_nents(qm_sg_ents),
1035 1 + !!ivsize + pad_sg_nents(mapped_src_nents));
1037 qm_sg_ents = pad_sg_nents(qm_sg_ents);
1039 sg_table = &edesc->sgt[0];
1040 qm_sg_bytes = qm_sg_ents * sizeof(*sg_table);
1041 if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize >
1042 CAAM_QI_MEMCACHE_SIZE)) {
1043 dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n",
1044 qm_sg_ents, ivsize);
1045 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
1047 qi_cache_free(edesc);
1048 return ERR_PTR(-ENOMEM);
1052 u8 *iv = (u8 *)(sg_table + qm_sg_ents);
1054 /* Make sure IV is located in a DMAable area */
1055 memcpy(iv, req->iv, ivsize);
1057 iv_dma = dma_map_single(qidev, iv, ivsize, DMA_TO_DEVICE);
1058 if (dma_mapping_error(qidev, iv_dma)) {
1059 dev_err(qidev, "unable to map IV\n");
1060 caam_unmap(qidev, req->src, req->dst, src_nents,
1061 dst_nents, 0, 0, DMA_NONE, 0, 0);
1062 qi_cache_free(edesc);
1063 return ERR_PTR(-ENOMEM);
1067 edesc->src_nents = src_nents;
1068 edesc->dst_nents = dst_nents;
1069 edesc->iv_dma = iv_dma;
1070 edesc->drv_req.app_ctx = req;
1071 edesc->drv_req.cbk = aead_done;
1072 edesc->drv_req.drv_ctx = drv_ctx;
1074 edesc->assoclen = cpu_to_caam32(req->assoclen);
1075 edesc->assoclen_dma = dma_map_single(qidev, &edesc->assoclen, 4,
1077 if (dma_mapping_error(qidev, edesc->assoclen_dma)) {
1078 dev_err(qidev, "unable to map assoclen\n");
1079 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
1080 iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
1081 qi_cache_free(edesc);
1082 return ERR_PTR(-ENOMEM);
1085 dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0);
1088 dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0);
1091 sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0);
1092 qm_sg_index += mapped_src_nents;
1094 if (mapped_dst_nents > 1)
1095 sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0);
1097 qm_sg_dma = dma_map_single(qidev, sg_table, qm_sg_bytes, DMA_TO_DEVICE);
1098 if (dma_mapping_error(qidev, qm_sg_dma)) {
1099 dev_err(qidev, "unable to map S/G table\n");
1100 dma_unmap_single(qidev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
1101 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
1102 iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
1103 qi_cache_free(edesc);
1104 return ERR_PTR(-ENOMEM);
1107 edesc->qm_sg_dma = qm_sg_dma;
1108 edesc->qm_sg_bytes = qm_sg_bytes;
1110 out_len = req->assoclen + req->cryptlen +
1111 (encrypt ? ctx->authsize : (-ctx->authsize));
1112 in_len = 4 + ivsize + req->assoclen + req->cryptlen;
1114 fd_sgt = &edesc->drv_req.fd_sgt[0];
1115 dma_to_qm_sg_one_last_ext(&fd_sgt[1], qm_sg_dma, in_len, 0);
1117 if (req->dst == req->src) {
1118 if (mapped_src_nents == 1)
1119 dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->src),
1122 dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma +
1123 (1 + !!ivsize) * sizeof(*sg_table),
1125 } else if (mapped_dst_nents <= 1) {
1126 dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), out_len,
1129 dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + sizeof(*sg_table) *
1130 qm_sg_index, out_len, 0);
1136 static inline int aead_crypt(struct aead_request *req, bool encrypt)
1138 struct aead_edesc *edesc;
1139 struct crypto_aead *aead = crypto_aead_reqtfm(req);
1140 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1143 if (unlikely(caam_congested))
1146 /* allocate extended descriptor */
1147 edesc = aead_edesc_alloc(req, encrypt);
1148 if (IS_ERR_OR_NULL(edesc))
1149 return PTR_ERR(edesc);
1151 /* Create and submit job descriptor */
1152 ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req);
1156 aead_unmap(ctx->qidev, edesc, req);
1157 qi_cache_free(edesc);
1163 static int aead_encrypt(struct aead_request *req)
1165 return aead_crypt(req, true);
1168 static int aead_decrypt(struct aead_request *req)
1170 return aead_crypt(req, false);
1173 static int ipsec_gcm_encrypt(struct aead_request *req)
1175 return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_crypt(req,
1179 static int ipsec_gcm_decrypt(struct aead_request *req)
1181 return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_crypt(req,
1185 static void skcipher_done(struct caam_drv_req *drv_req, u32 status)
1187 struct skcipher_edesc *edesc;
1188 struct skcipher_request *req = drv_req->app_ctx;
1189 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1190 struct caam_ctx *caam_ctx = crypto_skcipher_ctx(skcipher);
1191 struct device *qidev = caam_ctx->qidev;
1192 int ivsize = crypto_skcipher_ivsize(skcipher);
1195 dev_dbg(qidev, "%s %d: status 0x%x\n", __func__, __LINE__, status);
1197 edesc = container_of(drv_req, typeof(*edesc), drv_req);
1200 ecode = caam_jr_strstatus(qidev, status);
1202 print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ",
1203 DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
1204 edesc->src_nents > 1 ? 100 : ivsize, 1);
1205 caam_dump_sg("dst @" __stringify(__LINE__)": ",
1206 DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
1207 edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
1209 skcipher_unmap(qidev, edesc, req);
1212 * The crypto API expects us to set the IV (req->iv) to the last
1213 * ciphertext block (CBC mode) or last counter (CTR mode).
1214 * This is used e.g. by the CTS mode.
1217 memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes,
1220 qi_cache_free(edesc);
1221 skcipher_request_complete(req, ecode);
1224 static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
1227 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1228 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1229 struct device *qidev = ctx->qidev;
1230 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1231 GFP_KERNEL : GFP_ATOMIC;
1232 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
1233 struct skcipher_edesc *edesc;
1236 int ivsize = crypto_skcipher_ivsize(skcipher);
1237 int dst_sg_idx, qm_sg_ents, qm_sg_bytes;
1238 struct qm_sg_entry *sg_table, *fd_sgt;
1239 struct caam_drv_ctx *drv_ctx;
1241 drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT);
1242 if (IS_ERR_OR_NULL(drv_ctx))
1243 return (struct skcipher_edesc *)drv_ctx;
1245 src_nents = sg_nents_for_len(req->src, req->cryptlen);
1246 if (unlikely(src_nents < 0)) {
1247 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n",
1249 return ERR_PTR(src_nents);
1252 if (unlikely(req->src != req->dst)) {
1253 dst_nents = sg_nents_for_len(req->dst, req->cryptlen);
1254 if (unlikely(dst_nents < 0)) {
1255 dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n",
1257 return ERR_PTR(dst_nents);
1260 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents,
1262 if (unlikely(!mapped_src_nents)) {
1263 dev_err(qidev, "unable to map source\n");
1264 return ERR_PTR(-ENOMEM);
1267 mapped_dst_nents = dma_map_sg(qidev, req->dst, dst_nents,
1269 if (unlikely(!mapped_dst_nents)) {
1270 dev_err(qidev, "unable to map destination\n");
1271 dma_unmap_sg(qidev, req->src, src_nents, DMA_TO_DEVICE);
1272 return ERR_PTR(-ENOMEM);
1275 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents,
1277 if (unlikely(!mapped_src_nents)) {
1278 dev_err(qidev, "unable to map source\n");
1279 return ERR_PTR(-ENOMEM);
1283 qm_sg_ents = 1 + mapped_src_nents;
1284 dst_sg_idx = qm_sg_ents;
1287 * Input, output HW S/G tables: [IV, src][dst, IV]
1288 * IV entries point to the same buffer
1289 * If src == dst, S/G entries are reused (S/G tables overlap)
1291 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
1292 * the end of the table by allocating more S/G entries.
1294 if (req->src != req->dst)
1295 qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1);
1297 qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents);
1299 qm_sg_bytes = qm_sg_ents * sizeof(struct qm_sg_entry);
1300 if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes +
1301 ivsize > CAAM_QI_MEMCACHE_SIZE)) {
1302 dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n",
1303 qm_sg_ents, ivsize);
1304 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
1306 return ERR_PTR(-ENOMEM);
1309 /* allocate space for base edesc, link tables and IV */
1310 edesc = qi_cache_alloc(GFP_DMA | flags);
1311 if (unlikely(!edesc)) {
1312 dev_err(qidev, "could not allocate extended descriptor\n");
1313 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
1315 return ERR_PTR(-ENOMEM);
1318 /* Make sure IV is located in a DMAable area */
1319 sg_table = &edesc->sgt[0];
1320 iv = (u8 *)(sg_table + qm_sg_ents);
1321 memcpy(iv, req->iv, ivsize);
1323 iv_dma = dma_map_single(qidev, iv, ivsize, DMA_BIDIRECTIONAL);
1324 if (dma_mapping_error(qidev, iv_dma)) {
1325 dev_err(qidev, "unable to map IV\n");
1326 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
1328 qi_cache_free(edesc);
1329 return ERR_PTR(-ENOMEM);
1332 edesc->src_nents = src_nents;
1333 edesc->dst_nents = dst_nents;
1334 edesc->iv_dma = iv_dma;
1335 edesc->qm_sg_bytes = qm_sg_bytes;
1336 edesc->drv_req.app_ctx = req;
1337 edesc->drv_req.cbk = skcipher_done;
1338 edesc->drv_req.drv_ctx = drv_ctx;
1340 dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0);
1341 sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0);
1343 if (req->src != req->dst)
1344 sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0);
1346 dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma,
1349 edesc->qm_sg_dma = dma_map_single(qidev, sg_table, edesc->qm_sg_bytes,
1351 if (dma_mapping_error(qidev, edesc->qm_sg_dma)) {
1352 dev_err(qidev, "unable to map S/G table\n");
1353 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
1354 iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0);
1355 qi_cache_free(edesc);
1356 return ERR_PTR(-ENOMEM);
1359 fd_sgt = &edesc->drv_req.fd_sgt[0];
1361 dma_to_qm_sg_one_last_ext(&fd_sgt[1], edesc->qm_sg_dma,
1362 ivsize + req->cryptlen, 0);
1364 if (req->src == req->dst)
1365 dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma +
1366 sizeof(*sg_table), req->cryptlen + ivsize,
1369 dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + dst_sg_idx *
1370 sizeof(*sg_table), req->cryptlen + ivsize,
1376 static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt)
1378 struct skcipher_edesc *edesc;
1379 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1380 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1386 if (unlikely(caam_congested))
1389 /* allocate extended descriptor */
1390 edesc = skcipher_edesc_alloc(req, encrypt);
1392 return PTR_ERR(edesc);
1394 ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req);
1398 skcipher_unmap(ctx->qidev, edesc, req);
1399 qi_cache_free(edesc);
1405 static int skcipher_encrypt(struct skcipher_request *req)
1407 return skcipher_crypt(req, true);
1410 static int skcipher_decrypt(struct skcipher_request *req)
1412 return skcipher_crypt(req, false);
1415 static struct caam_skcipher_alg driver_algs[] = {
1419 .cra_name = "cbc(aes)",
1420 .cra_driver_name = "cbc-aes-caam-qi",
1421 .cra_blocksize = AES_BLOCK_SIZE,
1423 .setkey = aes_skcipher_setkey,
1424 .encrypt = skcipher_encrypt,
1425 .decrypt = skcipher_decrypt,
1426 .min_keysize = AES_MIN_KEY_SIZE,
1427 .max_keysize = AES_MAX_KEY_SIZE,
1428 .ivsize = AES_BLOCK_SIZE,
1430 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1435 .cra_name = "cbc(des3_ede)",
1436 .cra_driver_name = "cbc-3des-caam-qi",
1437 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1439 .setkey = des3_skcipher_setkey,
1440 .encrypt = skcipher_encrypt,
1441 .decrypt = skcipher_decrypt,
1442 .min_keysize = DES3_EDE_KEY_SIZE,
1443 .max_keysize = DES3_EDE_KEY_SIZE,
1444 .ivsize = DES3_EDE_BLOCK_SIZE,
1446 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1451 .cra_name = "cbc(des)",
1452 .cra_driver_name = "cbc-des-caam-qi",
1453 .cra_blocksize = DES_BLOCK_SIZE,
1455 .setkey = des_skcipher_setkey,
1456 .encrypt = skcipher_encrypt,
1457 .decrypt = skcipher_decrypt,
1458 .min_keysize = DES_KEY_SIZE,
1459 .max_keysize = DES_KEY_SIZE,
1460 .ivsize = DES_BLOCK_SIZE,
1462 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
1467 .cra_name = "ctr(aes)",
1468 .cra_driver_name = "ctr-aes-caam-qi",
1471 .setkey = ctr_skcipher_setkey,
1472 .encrypt = skcipher_encrypt,
1473 .decrypt = skcipher_decrypt,
1474 .min_keysize = AES_MIN_KEY_SIZE,
1475 .max_keysize = AES_MAX_KEY_SIZE,
1476 .ivsize = AES_BLOCK_SIZE,
1477 .chunksize = AES_BLOCK_SIZE,
1479 .caam.class1_alg_type = OP_ALG_ALGSEL_AES |
1480 OP_ALG_AAI_CTR_MOD128,
1485 .cra_name = "rfc3686(ctr(aes))",
1486 .cra_driver_name = "rfc3686-ctr-aes-caam-qi",
1489 .setkey = rfc3686_skcipher_setkey,
1490 .encrypt = skcipher_encrypt,
1491 .decrypt = skcipher_decrypt,
1492 .min_keysize = AES_MIN_KEY_SIZE +
1493 CTR_RFC3686_NONCE_SIZE,
1494 .max_keysize = AES_MAX_KEY_SIZE +
1495 CTR_RFC3686_NONCE_SIZE,
1496 .ivsize = CTR_RFC3686_IV_SIZE,
1497 .chunksize = AES_BLOCK_SIZE,
1500 .class1_alg_type = OP_ALG_ALGSEL_AES |
1501 OP_ALG_AAI_CTR_MOD128,
1508 .cra_name = "xts(aes)",
1509 .cra_driver_name = "xts-aes-caam-qi",
1510 .cra_blocksize = AES_BLOCK_SIZE,
1512 .setkey = xts_skcipher_setkey,
1513 .encrypt = skcipher_encrypt,
1514 .decrypt = skcipher_decrypt,
1515 .min_keysize = 2 * AES_MIN_KEY_SIZE,
1516 .max_keysize = 2 * AES_MAX_KEY_SIZE,
1517 .ivsize = AES_BLOCK_SIZE,
1519 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS,
1523 static struct caam_aead_alg driver_aeads[] = {
1527 .cra_name = "rfc4106(gcm(aes))",
1528 .cra_driver_name = "rfc4106-gcm-aes-caam-qi",
1531 .setkey = rfc4106_setkey,
1532 .setauthsize = rfc4106_setauthsize,
1533 .encrypt = ipsec_gcm_encrypt,
1534 .decrypt = ipsec_gcm_decrypt,
1536 .maxauthsize = AES_BLOCK_SIZE,
1539 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1546 .cra_name = "rfc4543(gcm(aes))",
1547 .cra_driver_name = "rfc4543-gcm-aes-caam-qi",
1550 .setkey = rfc4543_setkey,
1551 .setauthsize = rfc4543_setauthsize,
1552 .encrypt = ipsec_gcm_encrypt,
1553 .decrypt = ipsec_gcm_decrypt,
1555 .maxauthsize = AES_BLOCK_SIZE,
1558 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1562 /* Galois Counter Mode */
1566 .cra_name = "gcm(aes)",
1567 .cra_driver_name = "gcm-aes-caam-qi",
1570 .setkey = gcm_setkey,
1571 .setauthsize = gcm_setauthsize,
1572 .encrypt = aead_encrypt,
1573 .decrypt = aead_decrypt,
1575 .maxauthsize = AES_BLOCK_SIZE,
1578 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1582 /* single-pass ipsec_esp descriptor */
1586 .cra_name = "authenc(hmac(md5),cbc(aes))",
1587 .cra_driver_name = "authenc-hmac-md5-"
1589 .cra_blocksize = AES_BLOCK_SIZE,
1591 .setkey = aead_setkey,
1592 .setauthsize = aead_setauthsize,
1593 .encrypt = aead_encrypt,
1594 .decrypt = aead_decrypt,
1595 .ivsize = AES_BLOCK_SIZE,
1596 .maxauthsize = MD5_DIGEST_SIZE,
1599 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1600 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
1601 OP_ALG_AAI_HMAC_PRECOMP,
1607 .cra_name = "echainiv(authenc(hmac(md5),"
1609 .cra_driver_name = "echainiv-authenc-hmac-md5-"
1611 .cra_blocksize = AES_BLOCK_SIZE,
1613 .setkey = aead_setkey,
1614 .setauthsize = aead_setauthsize,
1615 .encrypt = aead_encrypt,
1616 .decrypt = aead_decrypt,
1617 .ivsize = AES_BLOCK_SIZE,
1618 .maxauthsize = MD5_DIGEST_SIZE,
1621 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1622 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
1623 OP_ALG_AAI_HMAC_PRECOMP,
1630 .cra_name = "authenc(hmac(sha1),cbc(aes))",
1631 .cra_driver_name = "authenc-hmac-sha1-"
1633 .cra_blocksize = AES_BLOCK_SIZE,
1635 .setkey = aead_setkey,
1636 .setauthsize = aead_setauthsize,
1637 .encrypt = aead_encrypt,
1638 .decrypt = aead_decrypt,
1639 .ivsize = AES_BLOCK_SIZE,
1640 .maxauthsize = SHA1_DIGEST_SIZE,
1643 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1644 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1645 OP_ALG_AAI_HMAC_PRECOMP,
1651 .cra_name = "echainiv(authenc(hmac(sha1),"
1653 .cra_driver_name = "echainiv-authenc-"
1654 "hmac-sha1-cbc-aes-caam-qi",
1655 .cra_blocksize = AES_BLOCK_SIZE,
1657 .setkey = aead_setkey,
1658 .setauthsize = aead_setauthsize,
1659 .encrypt = aead_encrypt,
1660 .decrypt = aead_decrypt,
1661 .ivsize = AES_BLOCK_SIZE,
1662 .maxauthsize = SHA1_DIGEST_SIZE,
1665 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1666 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1667 OP_ALG_AAI_HMAC_PRECOMP,
1674 .cra_name = "authenc(hmac(sha224),cbc(aes))",
1675 .cra_driver_name = "authenc-hmac-sha224-"
1677 .cra_blocksize = AES_BLOCK_SIZE,
1679 .setkey = aead_setkey,
1680 .setauthsize = aead_setauthsize,
1681 .encrypt = aead_encrypt,
1682 .decrypt = aead_decrypt,
1683 .ivsize = AES_BLOCK_SIZE,
1684 .maxauthsize = SHA224_DIGEST_SIZE,
1687 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1688 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1689 OP_ALG_AAI_HMAC_PRECOMP,
1695 .cra_name = "echainiv(authenc(hmac(sha224),"
1697 .cra_driver_name = "echainiv-authenc-"
1698 "hmac-sha224-cbc-aes-caam-qi",
1699 .cra_blocksize = AES_BLOCK_SIZE,
1701 .setkey = aead_setkey,
1702 .setauthsize = aead_setauthsize,
1703 .encrypt = aead_encrypt,
1704 .decrypt = aead_decrypt,
1705 .ivsize = AES_BLOCK_SIZE,
1706 .maxauthsize = SHA224_DIGEST_SIZE,
1709 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1710 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1711 OP_ALG_AAI_HMAC_PRECOMP,
1718 .cra_name = "authenc(hmac(sha256),cbc(aes))",
1719 .cra_driver_name = "authenc-hmac-sha256-"
1721 .cra_blocksize = AES_BLOCK_SIZE,
1723 .setkey = aead_setkey,
1724 .setauthsize = aead_setauthsize,
1725 .encrypt = aead_encrypt,
1726 .decrypt = aead_decrypt,
1727 .ivsize = AES_BLOCK_SIZE,
1728 .maxauthsize = SHA256_DIGEST_SIZE,
1731 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1732 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
1733 OP_ALG_AAI_HMAC_PRECOMP,
1739 .cra_name = "echainiv(authenc(hmac(sha256),"
1741 .cra_driver_name = "echainiv-authenc-"
1742 "hmac-sha256-cbc-aes-"
1744 .cra_blocksize = AES_BLOCK_SIZE,
1746 .setkey = aead_setkey,
1747 .setauthsize = aead_setauthsize,
1748 .encrypt = aead_encrypt,
1749 .decrypt = aead_decrypt,
1750 .ivsize = AES_BLOCK_SIZE,
1751 .maxauthsize = SHA256_DIGEST_SIZE,
1754 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1755 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
1756 OP_ALG_AAI_HMAC_PRECOMP,
1763 .cra_name = "authenc(hmac(sha384),cbc(aes))",
1764 .cra_driver_name = "authenc-hmac-sha384-"
1766 .cra_blocksize = AES_BLOCK_SIZE,
1768 .setkey = aead_setkey,
1769 .setauthsize = aead_setauthsize,
1770 .encrypt = aead_encrypt,
1771 .decrypt = aead_decrypt,
1772 .ivsize = AES_BLOCK_SIZE,
1773 .maxauthsize = SHA384_DIGEST_SIZE,
1776 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1777 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
1778 OP_ALG_AAI_HMAC_PRECOMP,
1784 .cra_name = "echainiv(authenc(hmac(sha384),"
1786 .cra_driver_name = "echainiv-authenc-"
1787 "hmac-sha384-cbc-aes-"
1789 .cra_blocksize = AES_BLOCK_SIZE,
1791 .setkey = aead_setkey,
1792 .setauthsize = aead_setauthsize,
1793 .encrypt = aead_encrypt,
1794 .decrypt = aead_decrypt,
1795 .ivsize = AES_BLOCK_SIZE,
1796 .maxauthsize = SHA384_DIGEST_SIZE,
1799 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1800 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
1801 OP_ALG_AAI_HMAC_PRECOMP,
1808 .cra_name = "authenc(hmac(sha512),cbc(aes))",
1809 .cra_driver_name = "authenc-hmac-sha512-"
1811 .cra_blocksize = AES_BLOCK_SIZE,
1813 .setkey = aead_setkey,
1814 .setauthsize = aead_setauthsize,
1815 .encrypt = aead_encrypt,
1816 .decrypt = aead_decrypt,
1817 .ivsize = AES_BLOCK_SIZE,
1818 .maxauthsize = SHA512_DIGEST_SIZE,
1821 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1822 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
1823 OP_ALG_AAI_HMAC_PRECOMP,
1829 .cra_name = "echainiv(authenc(hmac(sha512),"
1831 .cra_driver_name = "echainiv-authenc-"
1832 "hmac-sha512-cbc-aes-"
1834 .cra_blocksize = AES_BLOCK_SIZE,
1836 .setkey = aead_setkey,
1837 .setauthsize = aead_setauthsize,
1838 .encrypt = aead_encrypt,
1839 .decrypt = aead_decrypt,
1840 .ivsize = AES_BLOCK_SIZE,
1841 .maxauthsize = SHA512_DIGEST_SIZE,
1844 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1845 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
1846 OP_ALG_AAI_HMAC_PRECOMP,
1853 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
1854 .cra_driver_name = "authenc-hmac-md5-"
1855 "cbc-des3_ede-caam-qi",
1856 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1858 .setkey = des3_aead_setkey,
1859 .setauthsize = aead_setauthsize,
1860 .encrypt = aead_encrypt,
1861 .decrypt = aead_decrypt,
1862 .ivsize = DES3_EDE_BLOCK_SIZE,
1863 .maxauthsize = MD5_DIGEST_SIZE,
1866 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1867 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
1868 OP_ALG_AAI_HMAC_PRECOMP,
1874 .cra_name = "echainiv(authenc(hmac(md5),"
1876 .cra_driver_name = "echainiv-authenc-hmac-md5-"
1877 "cbc-des3_ede-caam-qi",
1878 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1880 .setkey = des3_aead_setkey,
1881 .setauthsize = aead_setauthsize,
1882 .encrypt = aead_encrypt,
1883 .decrypt = aead_decrypt,
1884 .ivsize = DES3_EDE_BLOCK_SIZE,
1885 .maxauthsize = MD5_DIGEST_SIZE,
1888 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1889 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
1890 OP_ALG_AAI_HMAC_PRECOMP,
1897 .cra_name = "authenc(hmac(sha1),"
1899 .cra_driver_name = "authenc-hmac-sha1-"
1900 "cbc-des3_ede-caam-qi",
1901 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1903 .setkey = des3_aead_setkey,
1904 .setauthsize = aead_setauthsize,
1905 .encrypt = aead_encrypt,
1906 .decrypt = aead_decrypt,
1907 .ivsize = DES3_EDE_BLOCK_SIZE,
1908 .maxauthsize = SHA1_DIGEST_SIZE,
1911 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1912 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1913 OP_ALG_AAI_HMAC_PRECOMP,
1919 .cra_name = "echainiv(authenc(hmac(sha1),"
1921 .cra_driver_name = "echainiv-authenc-"
1923 "cbc-des3_ede-caam-qi",
1924 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1926 .setkey = des3_aead_setkey,
1927 .setauthsize = aead_setauthsize,
1928 .encrypt = aead_encrypt,
1929 .decrypt = aead_decrypt,
1930 .ivsize = DES3_EDE_BLOCK_SIZE,
1931 .maxauthsize = SHA1_DIGEST_SIZE,
1934 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1935 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1936 OP_ALG_AAI_HMAC_PRECOMP,
1943 .cra_name = "authenc(hmac(sha224),"
1945 .cra_driver_name = "authenc-hmac-sha224-"
1946 "cbc-des3_ede-caam-qi",
1947 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1949 .setkey = des3_aead_setkey,
1950 .setauthsize = aead_setauthsize,
1951 .encrypt = aead_encrypt,
1952 .decrypt = aead_decrypt,
1953 .ivsize = DES3_EDE_BLOCK_SIZE,
1954 .maxauthsize = SHA224_DIGEST_SIZE,
1957 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1958 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1959 OP_ALG_AAI_HMAC_PRECOMP,
1965 .cra_name = "echainiv(authenc(hmac(sha224),"
1967 .cra_driver_name = "echainiv-authenc-"
1969 "cbc-des3_ede-caam-qi",
1970 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1972 .setkey = des3_aead_setkey,
1973 .setauthsize = aead_setauthsize,
1974 .encrypt = aead_encrypt,
1975 .decrypt = aead_decrypt,
1976 .ivsize = DES3_EDE_BLOCK_SIZE,
1977 .maxauthsize = SHA224_DIGEST_SIZE,
1980 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1981 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1982 OP_ALG_AAI_HMAC_PRECOMP,
1989 .cra_name = "authenc(hmac(sha256),"
1991 .cra_driver_name = "authenc-hmac-sha256-"
1992 "cbc-des3_ede-caam-qi",
1993 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1995 .setkey = des3_aead_setkey,
1996 .setauthsize = aead_setauthsize,
1997 .encrypt = aead_encrypt,
1998 .decrypt = aead_decrypt,
1999 .ivsize = DES3_EDE_BLOCK_SIZE,
2000 .maxauthsize = SHA256_DIGEST_SIZE,
2003 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2004 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2005 OP_ALG_AAI_HMAC_PRECOMP,
2011 .cra_name = "echainiv(authenc(hmac(sha256),"
2013 .cra_driver_name = "echainiv-authenc-"
2015 "cbc-des3_ede-caam-qi",
2016 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2018 .setkey = des3_aead_setkey,
2019 .setauthsize = aead_setauthsize,
2020 .encrypt = aead_encrypt,
2021 .decrypt = aead_decrypt,
2022 .ivsize = DES3_EDE_BLOCK_SIZE,
2023 .maxauthsize = SHA256_DIGEST_SIZE,
2026 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2027 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2028 OP_ALG_AAI_HMAC_PRECOMP,
2035 .cra_name = "authenc(hmac(sha384),"
2037 .cra_driver_name = "authenc-hmac-sha384-"
2038 "cbc-des3_ede-caam-qi",
2039 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2041 .setkey = des3_aead_setkey,
2042 .setauthsize = aead_setauthsize,
2043 .encrypt = aead_encrypt,
2044 .decrypt = aead_decrypt,
2045 .ivsize = DES3_EDE_BLOCK_SIZE,
2046 .maxauthsize = SHA384_DIGEST_SIZE,
2049 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2050 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2051 OP_ALG_AAI_HMAC_PRECOMP,
2057 .cra_name = "echainiv(authenc(hmac(sha384),"
2059 .cra_driver_name = "echainiv-authenc-"
2061 "cbc-des3_ede-caam-qi",
2062 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2064 .setkey = des3_aead_setkey,
2065 .setauthsize = aead_setauthsize,
2066 .encrypt = aead_encrypt,
2067 .decrypt = aead_decrypt,
2068 .ivsize = DES3_EDE_BLOCK_SIZE,
2069 .maxauthsize = SHA384_DIGEST_SIZE,
2072 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2073 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2074 OP_ALG_AAI_HMAC_PRECOMP,
2081 .cra_name = "authenc(hmac(sha512),"
2083 .cra_driver_name = "authenc-hmac-sha512-"
2084 "cbc-des3_ede-caam-qi",
2085 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2087 .setkey = des3_aead_setkey,
2088 .setauthsize = aead_setauthsize,
2089 .encrypt = aead_encrypt,
2090 .decrypt = aead_decrypt,
2091 .ivsize = DES3_EDE_BLOCK_SIZE,
2092 .maxauthsize = SHA512_DIGEST_SIZE,
2095 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2096 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2097 OP_ALG_AAI_HMAC_PRECOMP,
2103 .cra_name = "echainiv(authenc(hmac(sha512),"
2105 .cra_driver_name = "echainiv-authenc-"
2107 "cbc-des3_ede-caam-qi",
2108 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2110 .setkey = des3_aead_setkey,
2111 .setauthsize = aead_setauthsize,
2112 .encrypt = aead_encrypt,
2113 .decrypt = aead_decrypt,
2114 .ivsize = DES3_EDE_BLOCK_SIZE,
2115 .maxauthsize = SHA512_DIGEST_SIZE,
2118 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2119 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2120 OP_ALG_AAI_HMAC_PRECOMP,
2127 .cra_name = "authenc(hmac(md5),cbc(des))",
2128 .cra_driver_name = "authenc-hmac-md5-"
2130 .cra_blocksize = DES_BLOCK_SIZE,
2132 .setkey = aead_setkey,
2133 .setauthsize = aead_setauthsize,
2134 .encrypt = aead_encrypt,
2135 .decrypt = aead_decrypt,
2136 .ivsize = DES_BLOCK_SIZE,
2137 .maxauthsize = MD5_DIGEST_SIZE,
2140 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2141 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2142 OP_ALG_AAI_HMAC_PRECOMP,
2148 .cra_name = "echainiv(authenc(hmac(md5),"
2150 .cra_driver_name = "echainiv-authenc-hmac-md5-"
2152 .cra_blocksize = DES_BLOCK_SIZE,
2154 .setkey = aead_setkey,
2155 .setauthsize = aead_setauthsize,
2156 .encrypt = aead_encrypt,
2157 .decrypt = aead_decrypt,
2158 .ivsize = DES_BLOCK_SIZE,
2159 .maxauthsize = MD5_DIGEST_SIZE,
2162 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2163 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2164 OP_ALG_AAI_HMAC_PRECOMP,
2171 .cra_name = "authenc(hmac(sha1),cbc(des))",
2172 .cra_driver_name = "authenc-hmac-sha1-"
2174 .cra_blocksize = DES_BLOCK_SIZE,
2176 .setkey = aead_setkey,
2177 .setauthsize = aead_setauthsize,
2178 .encrypt = aead_encrypt,
2179 .decrypt = aead_decrypt,
2180 .ivsize = DES_BLOCK_SIZE,
2181 .maxauthsize = SHA1_DIGEST_SIZE,
2184 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2185 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2186 OP_ALG_AAI_HMAC_PRECOMP,
2192 .cra_name = "echainiv(authenc(hmac(sha1),"
2194 .cra_driver_name = "echainiv-authenc-"
2195 "hmac-sha1-cbc-des-caam-qi",
2196 .cra_blocksize = DES_BLOCK_SIZE,
2198 .setkey = aead_setkey,
2199 .setauthsize = aead_setauthsize,
2200 .encrypt = aead_encrypt,
2201 .decrypt = aead_decrypt,
2202 .ivsize = DES_BLOCK_SIZE,
2203 .maxauthsize = SHA1_DIGEST_SIZE,
2206 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2207 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2208 OP_ALG_AAI_HMAC_PRECOMP,
2215 .cra_name = "authenc(hmac(sha224),cbc(des))",
2216 .cra_driver_name = "authenc-hmac-sha224-"
2218 .cra_blocksize = DES_BLOCK_SIZE,
2220 .setkey = aead_setkey,
2221 .setauthsize = aead_setauthsize,
2222 .encrypt = aead_encrypt,
2223 .decrypt = aead_decrypt,
2224 .ivsize = DES_BLOCK_SIZE,
2225 .maxauthsize = SHA224_DIGEST_SIZE,
2228 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2229 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2230 OP_ALG_AAI_HMAC_PRECOMP,
2236 .cra_name = "echainiv(authenc(hmac(sha224),"
2238 .cra_driver_name = "echainiv-authenc-"
2239 "hmac-sha224-cbc-des-"
2241 .cra_blocksize = DES_BLOCK_SIZE,
2243 .setkey = aead_setkey,
2244 .setauthsize = aead_setauthsize,
2245 .encrypt = aead_encrypt,
2246 .decrypt = aead_decrypt,
2247 .ivsize = DES_BLOCK_SIZE,
2248 .maxauthsize = SHA224_DIGEST_SIZE,
2251 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2252 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2253 OP_ALG_AAI_HMAC_PRECOMP,
2260 .cra_name = "authenc(hmac(sha256),cbc(des))",
2261 .cra_driver_name = "authenc-hmac-sha256-"
2263 .cra_blocksize = DES_BLOCK_SIZE,
2265 .setkey = aead_setkey,
2266 .setauthsize = aead_setauthsize,
2267 .encrypt = aead_encrypt,
2268 .decrypt = aead_decrypt,
2269 .ivsize = DES_BLOCK_SIZE,
2270 .maxauthsize = SHA256_DIGEST_SIZE,
2273 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2274 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2275 OP_ALG_AAI_HMAC_PRECOMP,
2281 .cra_name = "echainiv(authenc(hmac(sha256),"
2283 .cra_driver_name = "echainiv-authenc-"
2284 "hmac-sha256-cbc-des-"
2286 .cra_blocksize = DES_BLOCK_SIZE,
2288 .setkey = aead_setkey,
2289 .setauthsize = aead_setauthsize,
2290 .encrypt = aead_encrypt,
2291 .decrypt = aead_decrypt,
2292 .ivsize = DES_BLOCK_SIZE,
2293 .maxauthsize = SHA256_DIGEST_SIZE,
2296 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2297 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2298 OP_ALG_AAI_HMAC_PRECOMP,
2305 .cra_name = "authenc(hmac(sha384),cbc(des))",
2306 .cra_driver_name = "authenc-hmac-sha384-"
2308 .cra_blocksize = DES_BLOCK_SIZE,
2310 .setkey = aead_setkey,
2311 .setauthsize = aead_setauthsize,
2312 .encrypt = aead_encrypt,
2313 .decrypt = aead_decrypt,
2314 .ivsize = DES_BLOCK_SIZE,
2315 .maxauthsize = SHA384_DIGEST_SIZE,
2318 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2319 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2320 OP_ALG_AAI_HMAC_PRECOMP,
2326 .cra_name = "echainiv(authenc(hmac(sha384),"
2328 .cra_driver_name = "echainiv-authenc-"
2329 "hmac-sha384-cbc-des-"
2331 .cra_blocksize = DES_BLOCK_SIZE,
2333 .setkey = aead_setkey,
2334 .setauthsize = aead_setauthsize,
2335 .encrypt = aead_encrypt,
2336 .decrypt = aead_decrypt,
2337 .ivsize = DES_BLOCK_SIZE,
2338 .maxauthsize = SHA384_DIGEST_SIZE,
2341 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2342 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2343 OP_ALG_AAI_HMAC_PRECOMP,
2350 .cra_name = "authenc(hmac(sha512),cbc(des))",
2351 .cra_driver_name = "authenc-hmac-sha512-"
2353 .cra_blocksize = DES_BLOCK_SIZE,
2355 .setkey = aead_setkey,
2356 .setauthsize = aead_setauthsize,
2357 .encrypt = aead_encrypt,
2358 .decrypt = aead_decrypt,
2359 .ivsize = DES_BLOCK_SIZE,
2360 .maxauthsize = SHA512_DIGEST_SIZE,
2363 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2364 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2365 OP_ALG_AAI_HMAC_PRECOMP,
2371 .cra_name = "echainiv(authenc(hmac(sha512),"
2373 .cra_driver_name = "echainiv-authenc-"
2374 "hmac-sha512-cbc-des-"
2376 .cra_blocksize = DES_BLOCK_SIZE,
2378 .setkey = aead_setkey,
2379 .setauthsize = aead_setauthsize,
2380 .encrypt = aead_encrypt,
2381 .decrypt = aead_decrypt,
2382 .ivsize = DES_BLOCK_SIZE,
2383 .maxauthsize = SHA512_DIGEST_SIZE,
2386 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2387 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2388 OP_ALG_AAI_HMAC_PRECOMP,
2394 static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam,
2397 struct caam_drv_private *priv;
2401 * distribute tfms across job rings to ensure in-order
2402 * crypto request processing per tfm
2404 ctx->jrdev = caam_jr_alloc();
2405 if (IS_ERR(ctx->jrdev)) {
2406 pr_err("Job Ring Device allocation for transform failed\n");
2407 return PTR_ERR(ctx->jrdev);
2410 dev = ctx->jrdev->parent;
2411 priv = dev_get_drvdata(dev);
2412 if (priv->era >= 6 && uses_dkp)
2413 ctx->dir = DMA_BIDIRECTIONAL;
2415 ctx->dir = DMA_TO_DEVICE;
2417 ctx->key_dma = dma_map_single(dev, ctx->key, sizeof(ctx->key),
2419 if (dma_mapping_error(dev, ctx->key_dma)) {
2420 dev_err(dev, "unable to map key\n");
2421 caam_jr_free(ctx->jrdev);
2425 /* copy descriptor header template value */
2426 ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
2427 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
2431 spin_lock_init(&ctx->lock);
2432 ctx->drv_ctx[ENCRYPT] = NULL;
2433 ctx->drv_ctx[DECRYPT] = NULL;
2438 static int caam_cra_init(struct crypto_skcipher *tfm)
2440 struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
2441 struct caam_skcipher_alg *caam_alg =
2442 container_of(alg, typeof(*caam_alg), skcipher);
2444 return caam_init_common(crypto_skcipher_ctx(tfm), &caam_alg->caam,
2448 static int caam_aead_init(struct crypto_aead *tfm)
2450 struct aead_alg *alg = crypto_aead_alg(tfm);
2451 struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg),
2453 struct caam_ctx *ctx = crypto_aead_ctx(tfm);
2455 return caam_init_common(ctx, &caam_alg->caam, !caam_alg->caam.nodkp);
2458 static void caam_exit_common(struct caam_ctx *ctx)
2460 caam_drv_ctx_rel(ctx->drv_ctx[ENCRYPT]);
2461 caam_drv_ctx_rel(ctx->drv_ctx[DECRYPT]);
2463 dma_unmap_single(ctx->jrdev->parent, ctx->key_dma, sizeof(ctx->key),
2466 caam_jr_free(ctx->jrdev);
2469 static void caam_cra_exit(struct crypto_skcipher *tfm)
2471 caam_exit_common(crypto_skcipher_ctx(tfm));
2474 static void caam_aead_exit(struct crypto_aead *tfm)
2476 caam_exit_common(crypto_aead_ctx(tfm));
2479 void caam_qi_algapi_exit(void)
2483 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
2484 struct caam_aead_alg *t_alg = driver_aeads + i;
2486 if (t_alg->registered)
2487 crypto_unregister_aead(&t_alg->aead);
2490 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
2491 struct caam_skcipher_alg *t_alg = driver_algs + i;
2493 if (t_alg->registered)
2494 crypto_unregister_skcipher(&t_alg->skcipher);
2498 static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg)
2500 struct skcipher_alg *alg = &t_alg->skcipher;
2502 alg->base.cra_module = THIS_MODULE;
2503 alg->base.cra_priority = CAAM_CRA_PRIORITY;
2504 alg->base.cra_ctxsize = sizeof(struct caam_ctx);
2505 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
2507 alg->init = caam_cra_init;
2508 alg->exit = caam_cra_exit;
2511 static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
2513 struct aead_alg *alg = &t_alg->aead;
2515 alg->base.cra_module = THIS_MODULE;
2516 alg->base.cra_priority = CAAM_CRA_PRIORITY;
2517 alg->base.cra_ctxsize = sizeof(struct caam_ctx);
2518 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
2520 alg->init = caam_aead_init;
2521 alg->exit = caam_aead_exit;
2524 int caam_qi_algapi_init(struct device *ctrldev)
2526 struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
2528 u32 aes_vid, aes_inst, des_inst, md_vid, md_inst;
2529 unsigned int md_limit = SHA512_DIGEST_SIZE;
2530 bool registered = false;
2532 /* Make sure this runs only on (DPAA 1.x) QI */
2533 if (!priv->qi_present || caam_dpaa2)
2537 * Register crypto algorithms the device supports.
2538 * First, detect presence and attributes of DES, AES, and MD blocks.
2540 if (priv->era < 10) {
2541 u32 cha_vid, cha_inst;
2543 cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls);
2544 aes_vid = cha_vid & CHA_ID_LS_AES_MASK;
2545 md_vid = (cha_vid & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
2547 cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls);
2548 des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >>
2549 CHA_ID_LS_DES_SHIFT;
2550 aes_inst = cha_inst & CHA_ID_LS_AES_MASK;
2551 md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
2555 aesa = rd_reg32(&priv->ctrl->vreg.aesa);
2556 mdha = rd_reg32(&priv->ctrl->vreg.mdha);
2558 aes_vid = (aesa & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT;
2559 md_vid = (mdha & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT;
2561 des_inst = rd_reg32(&priv->ctrl->vreg.desa) & CHA_VER_NUM_MASK;
2562 aes_inst = aesa & CHA_VER_NUM_MASK;
2563 md_inst = mdha & CHA_VER_NUM_MASK;
2566 /* If MD is present, limit digest size based on LP256 */
2567 if (md_inst && md_vid == CHA_VER_VID_MD_LP256)
2568 md_limit = SHA256_DIGEST_SIZE;
2570 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
2571 struct caam_skcipher_alg *t_alg = driver_algs + i;
2572 u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK;
2574 /* Skip DES algorithms if not supported by device */
2576 ((alg_sel == OP_ALG_ALGSEL_3DES) ||
2577 (alg_sel == OP_ALG_ALGSEL_DES)))
2580 /* Skip AES algorithms if not supported by device */
2581 if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES))
2584 caam_skcipher_alg_init(t_alg);
2586 err = crypto_register_skcipher(&t_alg->skcipher);
2588 dev_warn(ctrldev, "%s alg registration failed\n",
2589 t_alg->skcipher.base.cra_driver_name);
2593 t_alg->registered = true;
2597 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
2598 struct caam_aead_alg *t_alg = driver_aeads + i;
2599 u32 c1_alg_sel = t_alg->caam.class1_alg_type &
2601 u32 c2_alg_sel = t_alg->caam.class2_alg_type &
2603 u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK;
2605 /* Skip DES algorithms if not supported by device */
2607 ((c1_alg_sel == OP_ALG_ALGSEL_3DES) ||
2608 (c1_alg_sel == OP_ALG_ALGSEL_DES)))
2611 /* Skip AES algorithms if not supported by device */
2612 if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES))
2616 * Check support for AES algorithms not available
2619 if (aes_vid == CHA_VER_VID_AES_LP && alg_aai == OP_ALG_AAI_GCM)
2623 * Skip algorithms requiring message digests
2624 * if MD or MD size is not supported by device.
2627 (!md_inst || (t_alg->aead.maxauthsize > md_limit)))
2630 caam_aead_alg_init(t_alg);
2632 err = crypto_register_aead(&t_alg->aead);
2634 pr_warn("%s alg registration failed\n",
2635 t_alg->aead.base.cra_driver_name);
2639 t_alg->registered = true;
2644 dev_info(ctrldev, "algorithms registered in /proc/crypto\n");