1 // SPDX-License-Identifier: GPL-2.0+
3 * caam - Freescale FSL CAAM support for ahash functions of crypto API
5 * Copyright 2011 Freescale Semiconductor, Inc.
6 * Copyright 2018-2019 NXP
8 * Based on caamalg.c crypto API driver.
10 * relationship of digest job descriptor or first job descriptor after init to
13 * --------------- ---------------
14 * | JobDesc #1 |-------------------->| ShareDesc |
15 * | *(packet 1) | | (hashKey) |
16 * --------------- | (operation) |
19 * relationship of subsequent job descriptors to shared descriptors:
21 * --------------- ---------------
22 * | JobDesc #2 |-------------------->| ShareDesc |
23 * | *(packet 2) | |------------->| (hashKey) |
24 * --------------- | |-------->| (operation) |
25 * . | | | (load ctx2) |
26 * . | | ---------------
28 * | JobDesc #3 |------| |
34 * | JobDesc #4 |------------
38 * The SharedDesc never changes for a connection unless rekeyed, but
39 * each packet will likely be in a different place. So all we need
40 * to know to process the packet is where the input is, where the
41 * output goes, and what context we want to process with. Context is
42 * in the SharedDesc, packet references in the JobDesc.
44 * So, a job desc looks like:
46 * ---------------------
48 * | ShareDesc Pointer |
55 * ---------------------
62 #include "desc_constr.h"
65 #include "sg_sw_sec4.h"
67 #include "caamhash_desc.h"
69 #define CAAM_CRA_PRIORITY 3000
71 /* max hash key is max split key size */
72 #define CAAM_MAX_HASH_KEY_SIZE (SHA512_DIGEST_SIZE * 2)
74 #define CAAM_MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE
75 #define CAAM_MAX_HASH_DIGEST_SIZE SHA512_DIGEST_SIZE
77 #define DESC_HASH_MAX_USED_BYTES (DESC_AHASH_FINAL_LEN + \
78 CAAM_MAX_HASH_KEY_SIZE)
79 #define DESC_HASH_MAX_USED_LEN (DESC_HASH_MAX_USED_BYTES / CAAM_CMD_SZ)
81 /* caam context sizes for hashes: running digest + 8 */
82 #define HASH_MSG_LEN 8
83 #define MAX_CTX_LEN (HASH_MSG_LEN + SHA512_DIGEST_SIZE)
85 static struct list_head hash_list;
87 /* ahash per-session context */
88 struct caam_hash_ctx {
89 u32 sh_desc_update[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
90 u32 sh_desc_update_first[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
91 u32 sh_desc_fin[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
92 u32 sh_desc_digest[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
93 u8 key[CAAM_MAX_HASH_KEY_SIZE] ____cacheline_aligned;
94 dma_addr_t sh_desc_update_dma ____cacheline_aligned;
95 dma_addr_t sh_desc_update_first_dma;
96 dma_addr_t sh_desc_fin_dma;
97 dma_addr_t sh_desc_digest_dma;
98 enum dma_data_direction dir;
99 enum dma_data_direction key_dir;
100 struct device *jrdev;
102 struct alginfo adata;
106 struct caam_hash_state {
110 u8 buf_0[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
112 u8 buf_1[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
114 u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned;
115 int (*update)(struct ahash_request *req);
116 int (*final)(struct ahash_request *req);
117 int (*finup)(struct ahash_request *req);
121 struct caam_export_state {
122 u8 buf[CAAM_MAX_HASH_BLOCK_SIZE];
123 u8 caam_ctx[MAX_CTX_LEN];
125 int (*update)(struct ahash_request *req);
126 int (*final)(struct ahash_request *req);
127 int (*finup)(struct ahash_request *req);
130 static inline void switch_buf(struct caam_hash_state *state)
132 state->current_buf ^= 1;
135 static inline u8 *current_buf(struct caam_hash_state *state)
137 return state->current_buf ? state->buf_1 : state->buf_0;
140 static inline u8 *alt_buf(struct caam_hash_state *state)
142 return state->current_buf ? state->buf_0 : state->buf_1;
145 static inline int *current_buflen(struct caam_hash_state *state)
147 return state->current_buf ? &state->buflen_1 : &state->buflen_0;
150 static inline int *alt_buflen(struct caam_hash_state *state)
152 return state->current_buf ? &state->buflen_0 : &state->buflen_1;
155 static inline bool is_cmac_aes(u32 algtype)
157 return (algtype & (OP_ALG_ALGSEL_MASK | OP_ALG_AAI_MASK)) ==
158 (OP_ALG_ALGSEL_AES | OP_ALG_AAI_CMAC);
160 /* Common job descriptor seq in/out ptr routines */
162 /* Map state->caam_ctx, and append seq_out_ptr command that points to it */
163 static inline int map_seq_out_ptr_ctx(u32 *desc, struct device *jrdev,
164 struct caam_hash_state *state,
167 state->ctx_dma_len = ctx_len;
168 state->ctx_dma = dma_map_single(jrdev, state->caam_ctx,
169 ctx_len, DMA_FROM_DEVICE);
170 if (dma_mapping_error(jrdev, state->ctx_dma)) {
171 dev_err(jrdev, "unable to map ctx\n");
176 append_seq_out_ptr(desc, state->ctx_dma, ctx_len, 0);
181 /* Map current buffer in state (if length > 0) and put it in link table */
182 static inline int buf_map_to_sec4_sg(struct device *jrdev,
183 struct sec4_sg_entry *sec4_sg,
184 struct caam_hash_state *state)
186 int buflen = *current_buflen(state);
191 state->buf_dma = dma_map_single(jrdev, current_buf(state), buflen,
193 if (dma_mapping_error(jrdev, state->buf_dma)) {
194 dev_err(jrdev, "unable to map buf\n");
199 dma_to_sec4_sg_one(sec4_sg, state->buf_dma, buflen, 0);
204 /* Map state->caam_ctx, and add it to link table */
205 static inline int ctx_map_to_sec4_sg(struct device *jrdev,
206 struct caam_hash_state *state, int ctx_len,
207 struct sec4_sg_entry *sec4_sg, u32 flag)
209 state->ctx_dma_len = ctx_len;
210 state->ctx_dma = dma_map_single(jrdev, state->caam_ctx, ctx_len, flag);
211 if (dma_mapping_error(jrdev, state->ctx_dma)) {
212 dev_err(jrdev, "unable to map ctx\n");
217 dma_to_sec4_sg_one(sec4_sg, state->ctx_dma, ctx_len, 0);
222 static int ahash_set_sh_desc(struct crypto_ahash *ahash)
224 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
225 int digestsize = crypto_ahash_digestsize(ahash);
226 struct device *jrdev = ctx->jrdev;
227 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
230 ctx->adata.key_virt = ctx->key;
232 /* ahash_update shared descriptor */
233 desc = ctx->sh_desc_update;
234 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_UPDATE, ctx->ctx_len,
235 ctx->ctx_len, true, ctrlpriv->era);
236 dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma,
237 desc_bytes(desc), ctx->dir);
239 print_hex_dump_debug("ahash update shdesc@"__stringify(__LINE__)": ",
240 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
243 /* ahash_update_first shared descriptor */
244 desc = ctx->sh_desc_update_first;
245 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len,
246 ctx->ctx_len, false, ctrlpriv->era);
247 dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma,
248 desc_bytes(desc), ctx->dir);
249 print_hex_dump_debug("ahash update first shdesc@"__stringify(__LINE__)
250 ": ", DUMP_PREFIX_ADDRESS, 16, 4, desc,
251 desc_bytes(desc), 1);
253 /* ahash_final shared descriptor */
254 desc = ctx->sh_desc_fin;
255 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, digestsize,
256 ctx->ctx_len, true, ctrlpriv->era);
257 dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma,
258 desc_bytes(desc), ctx->dir);
260 print_hex_dump_debug("ahash final shdesc@"__stringify(__LINE__)": ",
261 DUMP_PREFIX_ADDRESS, 16, 4, desc,
262 desc_bytes(desc), 1);
264 /* ahash_digest shared descriptor */
265 desc = ctx->sh_desc_digest;
266 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, digestsize,
267 ctx->ctx_len, false, ctrlpriv->era);
268 dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma,
269 desc_bytes(desc), ctx->dir);
271 print_hex_dump_debug("ahash digest shdesc@"__stringify(__LINE__)": ",
272 DUMP_PREFIX_ADDRESS, 16, 4, desc,
273 desc_bytes(desc), 1);
278 static int axcbc_set_sh_desc(struct crypto_ahash *ahash)
280 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
281 int digestsize = crypto_ahash_digestsize(ahash);
282 struct device *jrdev = ctx->jrdev;
285 /* shared descriptor for ahash_update */
286 desc = ctx->sh_desc_update;
287 cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_UPDATE,
288 ctx->ctx_len, ctx->ctx_len);
289 dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma,
290 desc_bytes(desc), ctx->dir);
291 print_hex_dump_debug("axcbc update shdesc@" __stringify(__LINE__)" : ",
292 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
295 /* shared descriptor for ahash_{final,finup} */
296 desc = ctx->sh_desc_fin;
297 cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_FINALIZE,
298 digestsize, ctx->ctx_len);
299 dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma,
300 desc_bytes(desc), ctx->dir);
301 print_hex_dump_debug("axcbc finup shdesc@" __stringify(__LINE__)" : ",
302 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
305 /* key is immediate data for INIT and INITFINAL states */
306 ctx->adata.key_virt = ctx->key;
308 /* shared descriptor for first invocation of ahash_update */
309 desc = ctx->sh_desc_update_first;
310 cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len,
312 dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma,
313 desc_bytes(desc), ctx->dir);
314 print_hex_dump_debug("axcbc update first shdesc@" __stringify(__LINE__)
315 " : ", DUMP_PREFIX_ADDRESS, 16, 4, desc,
316 desc_bytes(desc), 1);
318 /* shared descriptor for ahash_digest */
319 desc = ctx->sh_desc_digest;
320 cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_INITFINAL,
321 digestsize, ctx->ctx_len);
322 dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma,
323 desc_bytes(desc), ctx->dir);
324 print_hex_dump_debug("axcbc digest shdesc@" __stringify(__LINE__)" : ",
325 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
330 static int acmac_set_sh_desc(struct crypto_ahash *ahash)
332 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
333 int digestsize = crypto_ahash_digestsize(ahash);
334 struct device *jrdev = ctx->jrdev;
337 /* shared descriptor for ahash_update */
338 desc = ctx->sh_desc_update;
339 cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_UPDATE,
340 ctx->ctx_len, ctx->ctx_len);
341 dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma,
342 desc_bytes(desc), ctx->dir);
343 print_hex_dump_debug("acmac update shdesc@" __stringify(__LINE__)" : ",
344 DUMP_PREFIX_ADDRESS, 16, 4, desc,
345 desc_bytes(desc), 1);
347 /* shared descriptor for ahash_{final,finup} */
348 desc = ctx->sh_desc_fin;
349 cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_FINALIZE,
350 digestsize, ctx->ctx_len);
351 dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma,
352 desc_bytes(desc), ctx->dir);
353 print_hex_dump_debug("acmac finup shdesc@" __stringify(__LINE__)" : ",
354 DUMP_PREFIX_ADDRESS, 16, 4, desc,
355 desc_bytes(desc), 1);
357 /* shared descriptor for first invocation of ahash_update */
358 desc = ctx->sh_desc_update_first;
359 cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len,
361 dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma,
362 desc_bytes(desc), ctx->dir);
363 print_hex_dump_debug("acmac update first shdesc@" __stringify(__LINE__)
364 " : ", DUMP_PREFIX_ADDRESS, 16, 4, desc,
365 desc_bytes(desc), 1);
367 /* shared descriptor for ahash_digest */
368 desc = ctx->sh_desc_digest;
369 cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_INITFINAL,
370 digestsize, ctx->ctx_len);
371 dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma,
372 desc_bytes(desc), ctx->dir);
373 print_hex_dump_debug("acmac digest shdesc@" __stringify(__LINE__)" : ",
374 DUMP_PREFIX_ADDRESS, 16, 4, desc,
375 desc_bytes(desc), 1);
380 /* Digest hash size if it is too large */
381 static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key,
384 struct device *jrdev = ctx->jrdev;
386 struct split_key_result result;
390 desc = kmalloc(CAAM_CMD_SZ * 8 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA);
392 dev_err(jrdev, "unable to allocate key input memory\n");
396 init_job_desc(desc, 0);
398 key_dma = dma_map_single(jrdev, key, *keylen, DMA_BIDIRECTIONAL);
399 if (dma_mapping_error(jrdev, key_dma)) {
400 dev_err(jrdev, "unable to map key memory\n");
405 /* Job descriptor to perform unkeyed hash on key_in */
406 append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT |
407 OP_ALG_AS_INITFINAL);
408 append_seq_in_ptr(desc, key_dma, *keylen, 0);
409 append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 |
410 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG);
411 append_seq_out_ptr(desc, key_dma, digestsize, 0);
412 append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
413 LDST_SRCDST_BYTE_CONTEXT);
415 print_hex_dump_debug("key_in@"__stringify(__LINE__)": ",
416 DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1);
417 print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
418 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
422 init_completion(&result.completion);
424 ret = caam_jr_enqueue(jrdev, desc, split_key_done, &result);
427 wait_for_completion(&result.completion);
430 print_hex_dump_debug("digested key@"__stringify(__LINE__)": ",
431 DUMP_PREFIX_ADDRESS, 16, 4, key,
434 dma_unmap_single(jrdev, key_dma, *keylen, DMA_BIDIRECTIONAL);
436 *keylen = digestsize;
443 static int ahash_setkey(struct crypto_ahash *ahash,
444 const u8 *key, unsigned int keylen)
446 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
447 struct device *jrdev = ctx->jrdev;
448 int blocksize = crypto_tfm_alg_blocksize(&ahash->base);
449 int digestsize = crypto_ahash_digestsize(ahash);
450 struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent);
452 u8 *hashed_key = NULL;
454 dev_dbg(jrdev, "keylen %d\n", keylen);
456 if (keylen > blocksize) {
457 hashed_key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);
460 ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize);
467 * If DKP is supported, use it in the shared descriptor to generate
470 if (ctrlpriv->era >= 6) {
471 ctx->adata.key_inline = true;
472 ctx->adata.keylen = keylen;
473 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
476 if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE)
479 memcpy(ctx->key, key, keylen);
482 * In case |user key| > |derived key|, using DKP<imm,imm>
483 * would result in invalid opcodes (last bytes of user key) in
484 * the resulting descriptor. Use DKP<ptr,imm> instead => both
485 * virtual and dma key addresses are needed.
487 if (keylen > ctx->adata.keylen_pad)
488 dma_sync_single_for_device(ctx->jrdev,
490 ctx->adata.keylen_pad,
493 ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, key,
494 keylen, CAAM_MAX_HASH_KEY_SIZE);
500 return ahash_set_sh_desc(ahash);
503 crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN);
507 static int axcbc_setkey(struct crypto_ahash *ahash, const u8 *key,
510 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
511 struct device *jrdev = ctx->jrdev;
513 if (keylen != AES_KEYSIZE_128) {
514 crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN);
518 memcpy(ctx->key, key, keylen);
519 dma_sync_single_for_device(jrdev, ctx->adata.key_dma, keylen,
521 ctx->adata.keylen = keylen;
523 print_hex_dump_debug("axcbc ctx.key@" __stringify(__LINE__)" : ",
524 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, keylen, 1);
526 return axcbc_set_sh_desc(ahash);
529 static int acmac_setkey(struct crypto_ahash *ahash, const u8 *key,
532 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
535 err = aes_check_keylen(keylen);
537 crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN);
541 /* key is immediate data for all cmac shared descriptors */
542 ctx->adata.key_virt = key;
543 ctx->adata.keylen = keylen;
545 print_hex_dump_debug("acmac ctx.key@" __stringify(__LINE__)" : ",
546 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
548 return acmac_set_sh_desc(ahash);
552 * ahash_edesc - s/w-extended ahash descriptor
553 * @sec4_sg_dma: physical mapped address of h/w link table
554 * @src_nents: number of segments in input scatterlist
555 * @sec4_sg_bytes: length of dma mapped sec4_sg space
556 * @hw_desc: the h/w job descriptor followed by any referenced link tables
557 * @sec4_sg: h/w link table
560 dma_addr_t sec4_sg_dma;
563 u32 hw_desc[DESC_JOB_IO_LEN_MAX / sizeof(u32)] ____cacheline_aligned;
564 struct sec4_sg_entry sec4_sg[0];
567 static inline void ahash_unmap(struct device *dev,
568 struct ahash_edesc *edesc,
569 struct ahash_request *req, int dst_len)
571 struct caam_hash_state *state = ahash_request_ctx(req);
573 if (edesc->src_nents)
574 dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE);
576 if (edesc->sec4_sg_bytes)
577 dma_unmap_single(dev, edesc->sec4_sg_dma,
578 edesc->sec4_sg_bytes, DMA_TO_DEVICE);
580 if (state->buf_dma) {
581 dma_unmap_single(dev, state->buf_dma, *current_buflen(state),
587 static inline void ahash_unmap_ctx(struct device *dev,
588 struct ahash_edesc *edesc,
589 struct ahash_request *req, int dst_len, u32 flag)
591 struct caam_hash_state *state = ahash_request_ctx(req);
593 if (state->ctx_dma) {
594 dma_unmap_single(dev, state->ctx_dma, state->ctx_dma_len, flag);
597 ahash_unmap(dev, edesc, req, dst_len);
600 static void ahash_done(struct device *jrdev, u32 *desc, u32 err,
603 struct ahash_request *req = context;
604 struct ahash_edesc *edesc;
605 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
606 int digestsize = crypto_ahash_digestsize(ahash);
607 struct caam_hash_state *state = ahash_request_ctx(req);
608 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
611 dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
613 edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
615 ecode = caam_jr_strstatus(jrdev, err);
617 ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);
618 memcpy(req->result, state->caam_ctx, digestsize);
621 print_hex_dump_debug("ctx@"__stringify(__LINE__)": ",
622 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
625 req->base.complete(&req->base, ecode);
628 static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err,
631 struct ahash_request *req = context;
632 struct ahash_edesc *edesc;
633 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
634 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
635 struct caam_hash_state *state = ahash_request_ctx(req);
636 int digestsize = crypto_ahash_digestsize(ahash);
639 dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
641 edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
643 ecode = caam_jr_strstatus(jrdev, err);
645 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL);
649 print_hex_dump_debug("ctx@"__stringify(__LINE__)": ",
650 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
653 print_hex_dump_debug("result@"__stringify(__LINE__)": ",
654 DUMP_PREFIX_ADDRESS, 16, 4, req->result,
657 req->base.complete(&req->base, ecode);
660 static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err,
663 struct ahash_request *req = context;
664 struct ahash_edesc *edesc;
665 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
666 int digestsize = crypto_ahash_digestsize(ahash);
667 struct caam_hash_state *state = ahash_request_ctx(req);
668 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
671 dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
673 edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
675 ecode = caam_jr_strstatus(jrdev, err);
677 ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_BIDIRECTIONAL);
678 memcpy(req->result, state->caam_ctx, digestsize);
681 print_hex_dump_debug("ctx@"__stringify(__LINE__)": ",
682 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
685 req->base.complete(&req->base, ecode);
688 static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err,
691 struct ahash_request *req = context;
692 struct ahash_edesc *edesc;
693 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
694 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
695 struct caam_hash_state *state = ahash_request_ctx(req);
696 int digestsize = crypto_ahash_digestsize(ahash);
699 dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
701 edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
703 ecode = caam_jr_strstatus(jrdev, err);
705 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_FROM_DEVICE);
709 print_hex_dump_debug("ctx@"__stringify(__LINE__)": ",
710 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
713 print_hex_dump_debug("result@"__stringify(__LINE__)": ",
714 DUMP_PREFIX_ADDRESS, 16, 4, req->result,
717 req->base.complete(&req->base, ecode);
721 * Allocate an enhanced descriptor, which contains the hardware descriptor
722 * and space for hardware scatter table containing sg_num entries.
724 static struct ahash_edesc *ahash_edesc_alloc(struct caam_hash_ctx *ctx,
725 int sg_num, u32 *sh_desc,
726 dma_addr_t sh_desc_dma,
729 struct ahash_edesc *edesc;
730 unsigned int sg_size = sg_num * sizeof(struct sec4_sg_entry);
732 edesc = kzalloc(sizeof(*edesc) + sg_size, GFP_DMA | flags);
734 dev_err(ctx->jrdev, "could not allocate extended descriptor\n");
738 init_job_desc_shared(edesc->hw_desc, sh_desc_dma, desc_len(sh_desc),
739 HDR_SHARE_DEFER | HDR_REVERSE);
744 static int ahash_edesc_add_src(struct caam_hash_ctx *ctx,
745 struct ahash_edesc *edesc,
746 struct ahash_request *req, int nents,
747 unsigned int first_sg,
748 unsigned int first_bytes, size_t to_hash)
753 if (nents > 1 || first_sg) {
754 struct sec4_sg_entry *sg = edesc->sec4_sg;
755 unsigned int sgsize = sizeof(*sg) *
756 pad_sg_nents(first_sg + nents);
758 sg_to_sec4_sg_last(req->src, to_hash, sg + first_sg, 0);
760 src_dma = dma_map_single(ctx->jrdev, sg, sgsize, DMA_TO_DEVICE);
761 if (dma_mapping_error(ctx->jrdev, src_dma)) {
762 dev_err(ctx->jrdev, "unable to map S/G table\n");
766 edesc->sec4_sg_bytes = sgsize;
767 edesc->sec4_sg_dma = src_dma;
770 src_dma = sg_dma_address(req->src);
774 append_seq_in_ptr(edesc->hw_desc, src_dma, first_bytes + to_hash,
780 /* submit update job descriptor */
781 static int ahash_update_ctx(struct ahash_request *req)
783 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
784 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
785 struct caam_hash_state *state = ahash_request_ctx(req);
786 struct device *jrdev = ctx->jrdev;
787 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
788 GFP_KERNEL : GFP_ATOMIC;
789 u8 *buf = current_buf(state);
790 int *buflen = current_buflen(state);
791 u8 *next_buf = alt_buf(state);
792 int blocksize = crypto_ahash_blocksize(ahash);
793 int *next_buflen = alt_buflen(state), last_buflen;
794 int in_len = *buflen + req->nbytes, to_hash;
796 int src_nents, mapped_nents, sec4_sg_bytes, sec4_sg_src_index;
797 struct ahash_edesc *edesc;
800 last_buflen = *next_buflen;
801 *next_buflen = in_len & (blocksize - 1);
802 to_hash = in_len - *next_buflen;
805 * For XCBC and CMAC, if to_hash is multiple of block size,
806 * keep last block in internal buffer
808 if ((is_xcbc_aes(ctx->adata.algtype) ||
809 is_cmac_aes(ctx->adata.algtype)) && to_hash >= blocksize &&
810 (*next_buflen == 0)) {
811 *next_buflen = blocksize;
812 to_hash -= blocksize;
817 int src_len = req->nbytes - *next_buflen;
819 src_nents = sg_nents_for_len(req->src, src_len);
821 dev_err(jrdev, "Invalid number of src SG.\n");
826 mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
829 dev_err(jrdev, "unable to DMA map source\n");
836 sec4_sg_src_index = 1 + (*buflen ? 1 : 0);
837 pad_nents = pad_sg_nents(sec4_sg_src_index + mapped_nents);
838 sec4_sg_bytes = pad_nents * sizeof(struct sec4_sg_entry);
841 * allocate space for base edesc and hw desc commands,
844 edesc = ahash_edesc_alloc(ctx, pad_nents, ctx->sh_desc_update,
845 ctx->sh_desc_update_dma, flags);
847 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
851 edesc->src_nents = src_nents;
852 edesc->sec4_sg_bytes = sec4_sg_bytes;
854 ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len,
855 edesc->sec4_sg, DMA_BIDIRECTIONAL);
859 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state);
864 sg_to_sec4_sg_last(req->src, src_len,
865 edesc->sec4_sg + sec4_sg_src_index,
868 sg_to_sec4_set_last(edesc->sec4_sg + sec4_sg_src_index -
872 scatterwalk_map_and_copy(next_buf, req->src,
875 desc = edesc->hw_desc;
877 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
880 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
881 dev_err(jrdev, "unable to map S/G table\n");
886 append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len +
889 append_seq_out_ptr(desc, state->ctx_dma, ctx->ctx_len, 0);
891 print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
892 DUMP_PREFIX_ADDRESS, 16, 4, desc,
893 desc_bytes(desc), 1);
895 ret = caam_jr_enqueue(jrdev, desc, ahash_done_bi, req);
900 } else if (*next_buflen) {
901 scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
903 *buflen = *next_buflen;
904 *next_buflen = last_buflen;
907 print_hex_dump_debug("buf@"__stringify(__LINE__)": ",
908 DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
909 print_hex_dump_debug("next buf@"__stringify(__LINE__)": ",
910 DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
915 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL);
920 static int ahash_final_ctx(struct ahash_request *req)
922 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
923 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
924 struct caam_hash_state *state = ahash_request_ctx(req);
925 struct device *jrdev = ctx->jrdev;
926 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
927 GFP_KERNEL : GFP_ATOMIC;
928 int buflen = *current_buflen(state);
931 int digestsize = crypto_ahash_digestsize(ahash);
932 struct ahash_edesc *edesc;
935 sec4_sg_bytes = pad_sg_nents(1 + (buflen ? 1 : 0)) *
936 sizeof(struct sec4_sg_entry);
938 /* allocate space for base edesc and hw desc commands, link tables */
939 edesc = ahash_edesc_alloc(ctx, 4, ctx->sh_desc_fin,
940 ctx->sh_desc_fin_dma, flags);
944 desc = edesc->hw_desc;
946 edesc->sec4_sg_bytes = sec4_sg_bytes;
948 ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len,
949 edesc->sec4_sg, DMA_BIDIRECTIONAL);
953 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state);
957 sg_to_sec4_set_last(edesc->sec4_sg + (buflen ? 1 : 0));
959 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
960 sec4_sg_bytes, DMA_TO_DEVICE);
961 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
962 dev_err(jrdev, "unable to map S/G table\n");
967 append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len + buflen,
969 append_seq_out_ptr(desc, state->ctx_dma, digestsize, 0);
971 print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
972 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
975 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req);
981 ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_BIDIRECTIONAL);
986 static int ahash_finup_ctx(struct ahash_request *req)
988 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
989 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
990 struct caam_hash_state *state = ahash_request_ctx(req);
991 struct device *jrdev = ctx->jrdev;
992 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
993 GFP_KERNEL : GFP_ATOMIC;
994 int buflen = *current_buflen(state);
996 int sec4_sg_src_index;
997 int src_nents, mapped_nents;
998 int digestsize = crypto_ahash_digestsize(ahash);
999 struct ahash_edesc *edesc;
1002 src_nents = sg_nents_for_len(req->src, req->nbytes);
1003 if (src_nents < 0) {
1004 dev_err(jrdev, "Invalid number of src SG.\n");
1009 mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
1011 if (!mapped_nents) {
1012 dev_err(jrdev, "unable to DMA map source\n");
1019 sec4_sg_src_index = 1 + (buflen ? 1 : 0);
1021 /* allocate space for base edesc and hw desc commands, link tables */
1022 edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents,
1023 ctx->sh_desc_fin, ctx->sh_desc_fin_dma,
1026 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1030 desc = edesc->hw_desc;
1032 edesc->src_nents = src_nents;
1034 ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len,
1035 edesc->sec4_sg, DMA_BIDIRECTIONAL);
1039 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state);
1043 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents,
1044 sec4_sg_src_index, ctx->ctx_len + buflen,
1049 append_seq_out_ptr(desc, state->ctx_dma, digestsize, 0);
1051 print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
1052 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1055 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req);
1059 return -EINPROGRESS;
1061 ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_BIDIRECTIONAL);
1066 static int ahash_digest(struct ahash_request *req)
1068 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1069 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1070 struct caam_hash_state *state = ahash_request_ctx(req);
1071 struct device *jrdev = ctx->jrdev;
1072 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1073 GFP_KERNEL : GFP_ATOMIC;
1075 int digestsize = crypto_ahash_digestsize(ahash);
1076 int src_nents, mapped_nents;
1077 struct ahash_edesc *edesc;
1082 src_nents = sg_nents_for_len(req->src, req->nbytes);
1083 if (src_nents < 0) {
1084 dev_err(jrdev, "Invalid number of src SG.\n");
1089 mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
1091 if (!mapped_nents) {
1092 dev_err(jrdev, "unable to map source for DMA\n");
1099 /* allocate space for base edesc and hw desc commands, link tables */
1100 edesc = ahash_edesc_alloc(ctx, mapped_nents > 1 ? mapped_nents : 0,
1101 ctx->sh_desc_digest, ctx->sh_desc_digest_dma,
1104 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1108 edesc->src_nents = src_nents;
1110 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0,
1113 ahash_unmap(jrdev, edesc, req, digestsize);
1118 desc = edesc->hw_desc;
1120 ret = map_seq_out_ptr_ctx(desc, jrdev, state, digestsize);
1122 ahash_unmap(jrdev, edesc, req, digestsize);
1127 print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
1128 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1131 ret = caam_jr_enqueue(jrdev, desc, ahash_done, req);
1135 ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);
1142 /* submit ahash final if it the first job descriptor */
1143 static int ahash_final_no_ctx(struct ahash_request *req)
1145 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1146 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1147 struct caam_hash_state *state = ahash_request_ctx(req);
1148 struct device *jrdev = ctx->jrdev;
1149 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1150 GFP_KERNEL : GFP_ATOMIC;
1151 u8 *buf = current_buf(state);
1152 int buflen = *current_buflen(state);
1154 int digestsize = crypto_ahash_digestsize(ahash);
1155 struct ahash_edesc *edesc;
1158 /* allocate space for base edesc and hw desc commands, link tables */
1159 edesc = ahash_edesc_alloc(ctx, 0, ctx->sh_desc_digest,
1160 ctx->sh_desc_digest_dma, flags);
1164 desc = edesc->hw_desc;
1167 state->buf_dma = dma_map_single(jrdev, buf, buflen,
1169 if (dma_mapping_error(jrdev, state->buf_dma)) {
1170 dev_err(jrdev, "unable to map src\n");
1174 append_seq_in_ptr(desc, state->buf_dma, buflen, 0);
1177 ret = map_seq_out_ptr_ctx(desc, jrdev, state, digestsize);
1181 print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
1182 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1185 ret = caam_jr_enqueue(jrdev, desc, ahash_done, req);
1189 ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);
1195 ahash_unmap(jrdev, edesc, req, digestsize);
1201 /* submit ahash update if it the first job descriptor after update */
1202 static int ahash_update_no_ctx(struct ahash_request *req)
1204 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1205 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1206 struct caam_hash_state *state = ahash_request_ctx(req);
1207 struct device *jrdev = ctx->jrdev;
1208 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1209 GFP_KERNEL : GFP_ATOMIC;
1210 u8 *buf = current_buf(state);
1211 int *buflen = current_buflen(state);
1212 int blocksize = crypto_ahash_blocksize(ahash);
1213 u8 *next_buf = alt_buf(state);
1214 int *next_buflen = alt_buflen(state);
1215 int in_len = *buflen + req->nbytes, to_hash;
1216 int sec4_sg_bytes, src_nents, mapped_nents;
1217 struct ahash_edesc *edesc;
1221 *next_buflen = in_len & (blocksize - 1);
1222 to_hash = in_len - *next_buflen;
1225 * For XCBC and CMAC, if to_hash is multiple of block size,
1226 * keep last block in internal buffer
1228 if ((is_xcbc_aes(ctx->adata.algtype) ||
1229 is_cmac_aes(ctx->adata.algtype)) && to_hash >= blocksize &&
1230 (*next_buflen == 0)) {
1231 *next_buflen = blocksize;
1232 to_hash -= blocksize;
1237 int src_len = req->nbytes - *next_buflen;
1239 src_nents = sg_nents_for_len(req->src, src_len);
1240 if (src_nents < 0) {
1241 dev_err(jrdev, "Invalid number of src SG.\n");
1246 mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
1248 if (!mapped_nents) {
1249 dev_err(jrdev, "unable to DMA map source\n");
1256 pad_nents = pad_sg_nents(1 + mapped_nents);
1257 sec4_sg_bytes = pad_nents * sizeof(struct sec4_sg_entry);
1260 * allocate space for base edesc and hw desc commands,
1263 edesc = ahash_edesc_alloc(ctx, pad_nents,
1264 ctx->sh_desc_update_first,
1265 ctx->sh_desc_update_first_dma,
1268 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1272 edesc->src_nents = src_nents;
1273 edesc->sec4_sg_bytes = sec4_sg_bytes;
1275 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, state);
1279 sg_to_sec4_sg_last(req->src, src_len, edesc->sec4_sg + 1, 0);
1282 scatterwalk_map_and_copy(next_buf, req->src,
1287 desc = edesc->hw_desc;
1289 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
1292 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
1293 dev_err(jrdev, "unable to map S/G table\n");
1298 append_seq_in_ptr(desc, edesc->sec4_sg_dma, to_hash, LDST_SGF);
1300 ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len);
1304 print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
1305 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1306 desc_bytes(desc), 1);
1308 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req);
1313 state->update = ahash_update_ctx;
1314 state->finup = ahash_finup_ctx;
1315 state->final = ahash_final_ctx;
1316 } else if (*next_buflen) {
1317 scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
1319 *buflen = *next_buflen;
1323 print_hex_dump_debug("buf@"__stringify(__LINE__)": ",
1324 DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
1325 print_hex_dump_debug("next buf@"__stringify(__LINE__)": ",
1326 DUMP_PREFIX_ADDRESS, 16, 4, next_buf, *next_buflen,
1331 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE);
1336 /* submit ahash finup if it the first job descriptor after update */
1337 static int ahash_finup_no_ctx(struct ahash_request *req)
1339 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1340 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1341 struct caam_hash_state *state = ahash_request_ctx(req);
1342 struct device *jrdev = ctx->jrdev;
1343 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1344 GFP_KERNEL : GFP_ATOMIC;
1345 int buflen = *current_buflen(state);
1347 int sec4_sg_bytes, sec4_sg_src_index, src_nents, mapped_nents;
1348 int digestsize = crypto_ahash_digestsize(ahash);
1349 struct ahash_edesc *edesc;
1352 src_nents = sg_nents_for_len(req->src, req->nbytes);
1353 if (src_nents < 0) {
1354 dev_err(jrdev, "Invalid number of src SG.\n");
1359 mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
1361 if (!mapped_nents) {
1362 dev_err(jrdev, "unable to DMA map source\n");
1369 sec4_sg_src_index = 2;
1370 sec4_sg_bytes = (sec4_sg_src_index + mapped_nents) *
1371 sizeof(struct sec4_sg_entry);
1373 /* allocate space for base edesc and hw desc commands, link tables */
1374 edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents,
1375 ctx->sh_desc_digest, ctx->sh_desc_digest_dma,
1378 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1382 desc = edesc->hw_desc;
1384 edesc->src_nents = src_nents;
1385 edesc->sec4_sg_bytes = sec4_sg_bytes;
1387 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, state);
1391 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 1, buflen,
1394 dev_err(jrdev, "unable to map S/G table\n");
1398 ret = map_seq_out_ptr_ctx(desc, jrdev, state, digestsize);
1402 print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
1403 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1406 ret = caam_jr_enqueue(jrdev, desc, ahash_done, req);
1410 ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);
1416 ahash_unmap(jrdev, edesc, req, digestsize);
1422 /* submit first update job descriptor after init */
1423 static int ahash_update_first(struct ahash_request *req)
1425 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1426 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1427 struct caam_hash_state *state = ahash_request_ctx(req);
1428 struct device *jrdev = ctx->jrdev;
1429 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1430 GFP_KERNEL : GFP_ATOMIC;
1431 u8 *next_buf = alt_buf(state);
1432 int *next_buflen = alt_buflen(state);
1434 int blocksize = crypto_ahash_blocksize(ahash);
1436 int src_nents, mapped_nents;
1437 struct ahash_edesc *edesc;
1440 *next_buflen = req->nbytes & (blocksize - 1);
1441 to_hash = req->nbytes - *next_buflen;
1444 * For XCBC and CMAC, if to_hash is multiple of block size,
1445 * keep last block in internal buffer
1447 if ((is_xcbc_aes(ctx->adata.algtype) ||
1448 is_cmac_aes(ctx->adata.algtype)) && to_hash >= blocksize &&
1449 (*next_buflen == 0)) {
1450 *next_buflen = blocksize;
1451 to_hash -= blocksize;
1455 src_nents = sg_nents_for_len(req->src,
1456 req->nbytes - *next_buflen);
1457 if (src_nents < 0) {
1458 dev_err(jrdev, "Invalid number of src SG.\n");
1463 mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
1465 if (!mapped_nents) {
1466 dev_err(jrdev, "unable to map source for DMA\n");
1474 * allocate space for base edesc and hw desc commands,
1477 edesc = ahash_edesc_alloc(ctx, mapped_nents > 1 ?
1479 ctx->sh_desc_update_first,
1480 ctx->sh_desc_update_first_dma,
1483 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1487 edesc->src_nents = src_nents;
1489 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0,
1495 scatterwalk_map_and_copy(next_buf, req->src, to_hash,
1498 desc = edesc->hw_desc;
1500 ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len);
1504 print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
1505 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1506 desc_bytes(desc), 1);
1508 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req);
1513 state->update = ahash_update_ctx;
1514 state->finup = ahash_finup_ctx;
1515 state->final = ahash_final_ctx;
1516 } else if (*next_buflen) {
1517 state->update = ahash_update_no_ctx;
1518 state->finup = ahash_finup_no_ctx;
1519 state->final = ahash_final_no_ctx;
1520 scatterwalk_map_and_copy(next_buf, req->src, 0,
1525 print_hex_dump_debug("next buf@"__stringify(__LINE__)": ",
1526 DUMP_PREFIX_ADDRESS, 16, 4, next_buf, *next_buflen,
1531 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE);
1536 static int ahash_finup_first(struct ahash_request *req)
1538 return ahash_digest(req);
1541 static int ahash_init(struct ahash_request *req)
1543 struct caam_hash_state *state = ahash_request_ctx(req);
1545 state->update = ahash_update_first;
1546 state->finup = ahash_finup_first;
1547 state->final = ahash_final_no_ctx;
1550 state->ctx_dma_len = 0;
1551 state->current_buf = 0;
1553 state->buflen_0 = 0;
1554 state->buflen_1 = 0;
1559 static int ahash_update(struct ahash_request *req)
1561 struct caam_hash_state *state = ahash_request_ctx(req);
1563 return state->update(req);
1566 static int ahash_finup(struct ahash_request *req)
1568 struct caam_hash_state *state = ahash_request_ctx(req);
1570 return state->finup(req);
1573 static int ahash_final(struct ahash_request *req)
1575 struct caam_hash_state *state = ahash_request_ctx(req);
1577 return state->final(req);
1580 static int ahash_export(struct ahash_request *req, void *out)
1582 struct caam_hash_state *state = ahash_request_ctx(req);
1583 struct caam_export_state *export = out;
1587 if (state->current_buf) {
1589 len = state->buflen_1;
1592 len = state->buflen_0;
1595 memcpy(export->buf, buf, len);
1596 memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx));
1597 export->buflen = len;
1598 export->update = state->update;
1599 export->final = state->final;
1600 export->finup = state->finup;
1605 static int ahash_import(struct ahash_request *req, const void *in)
1607 struct caam_hash_state *state = ahash_request_ctx(req);
1608 const struct caam_export_state *export = in;
1610 memset(state, 0, sizeof(*state));
1611 memcpy(state->buf_0, export->buf, export->buflen);
1612 memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx));
1613 state->buflen_0 = export->buflen;
1614 state->update = export->update;
1615 state->final = export->final;
1616 state->finup = export->finup;
1621 struct caam_hash_template {
1622 char name[CRYPTO_MAX_ALG_NAME];
1623 char driver_name[CRYPTO_MAX_ALG_NAME];
1624 char hmac_name[CRYPTO_MAX_ALG_NAME];
1625 char hmac_driver_name[CRYPTO_MAX_ALG_NAME];
1626 unsigned int blocksize;
1627 struct ahash_alg template_ahash;
1631 /* ahash descriptors */
1632 static struct caam_hash_template driver_hash[] = {
1635 .driver_name = "sha1-caam",
1636 .hmac_name = "hmac(sha1)",
1637 .hmac_driver_name = "hmac-sha1-caam",
1638 .blocksize = SHA1_BLOCK_SIZE,
1641 .update = ahash_update,
1642 .final = ahash_final,
1643 .finup = ahash_finup,
1644 .digest = ahash_digest,
1645 .export = ahash_export,
1646 .import = ahash_import,
1647 .setkey = ahash_setkey,
1649 .digestsize = SHA1_DIGEST_SIZE,
1650 .statesize = sizeof(struct caam_export_state),
1653 .alg_type = OP_ALG_ALGSEL_SHA1,
1656 .driver_name = "sha224-caam",
1657 .hmac_name = "hmac(sha224)",
1658 .hmac_driver_name = "hmac-sha224-caam",
1659 .blocksize = SHA224_BLOCK_SIZE,
1662 .update = ahash_update,
1663 .final = ahash_final,
1664 .finup = ahash_finup,
1665 .digest = ahash_digest,
1666 .export = ahash_export,
1667 .import = ahash_import,
1668 .setkey = ahash_setkey,
1670 .digestsize = SHA224_DIGEST_SIZE,
1671 .statesize = sizeof(struct caam_export_state),
1674 .alg_type = OP_ALG_ALGSEL_SHA224,
1677 .driver_name = "sha256-caam",
1678 .hmac_name = "hmac(sha256)",
1679 .hmac_driver_name = "hmac-sha256-caam",
1680 .blocksize = SHA256_BLOCK_SIZE,
1683 .update = ahash_update,
1684 .final = ahash_final,
1685 .finup = ahash_finup,
1686 .digest = ahash_digest,
1687 .export = ahash_export,
1688 .import = ahash_import,
1689 .setkey = ahash_setkey,
1691 .digestsize = SHA256_DIGEST_SIZE,
1692 .statesize = sizeof(struct caam_export_state),
1695 .alg_type = OP_ALG_ALGSEL_SHA256,
1698 .driver_name = "sha384-caam",
1699 .hmac_name = "hmac(sha384)",
1700 .hmac_driver_name = "hmac-sha384-caam",
1701 .blocksize = SHA384_BLOCK_SIZE,
1704 .update = ahash_update,
1705 .final = ahash_final,
1706 .finup = ahash_finup,
1707 .digest = ahash_digest,
1708 .export = ahash_export,
1709 .import = ahash_import,
1710 .setkey = ahash_setkey,
1712 .digestsize = SHA384_DIGEST_SIZE,
1713 .statesize = sizeof(struct caam_export_state),
1716 .alg_type = OP_ALG_ALGSEL_SHA384,
1719 .driver_name = "sha512-caam",
1720 .hmac_name = "hmac(sha512)",
1721 .hmac_driver_name = "hmac-sha512-caam",
1722 .blocksize = SHA512_BLOCK_SIZE,
1725 .update = ahash_update,
1726 .final = ahash_final,
1727 .finup = ahash_finup,
1728 .digest = ahash_digest,
1729 .export = ahash_export,
1730 .import = ahash_import,
1731 .setkey = ahash_setkey,
1733 .digestsize = SHA512_DIGEST_SIZE,
1734 .statesize = sizeof(struct caam_export_state),
1737 .alg_type = OP_ALG_ALGSEL_SHA512,
1740 .driver_name = "md5-caam",
1741 .hmac_name = "hmac(md5)",
1742 .hmac_driver_name = "hmac-md5-caam",
1743 .blocksize = MD5_BLOCK_WORDS * 4,
1746 .update = ahash_update,
1747 .final = ahash_final,
1748 .finup = ahash_finup,
1749 .digest = ahash_digest,
1750 .export = ahash_export,
1751 .import = ahash_import,
1752 .setkey = ahash_setkey,
1754 .digestsize = MD5_DIGEST_SIZE,
1755 .statesize = sizeof(struct caam_export_state),
1758 .alg_type = OP_ALG_ALGSEL_MD5,
1760 .hmac_name = "xcbc(aes)",
1761 .hmac_driver_name = "xcbc-aes-caam",
1762 .blocksize = AES_BLOCK_SIZE,
1765 .update = ahash_update,
1766 .final = ahash_final,
1767 .finup = ahash_finup,
1768 .digest = ahash_digest,
1769 .export = ahash_export,
1770 .import = ahash_import,
1771 .setkey = axcbc_setkey,
1773 .digestsize = AES_BLOCK_SIZE,
1774 .statesize = sizeof(struct caam_export_state),
1777 .alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XCBC_MAC,
1779 .hmac_name = "cmac(aes)",
1780 .hmac_driver_name = "cmac-aes-caam",
1781 .blocksize = AES_BLOCK_SIZE,
1784 .update = ahash_update,
1785 .final = ahash_final,
1786 .finup = ahash_finup,
1787 .digest = ahash_digest,
1788 .export = ahash_export,
1789 .import = ahash_import,
1790 .setkey = acmac_setkey,
1792 .digestsize = AES_BLOCK_SIZE,
1793 .statesize = sizeof(struct caam_export_state),
1796 .alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CMAC,
1800 struct caam_hash_alg {
1801 struct list_head entry;
1803 struct ahash_alg ahash_alg;
1806 static int caam_hash_cra_init(struct crypto_tfm *tfm)
1808 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
1809 struct crypto_alg *base = tfm->__crt_alg;
1810 struct hash_alg_common *halg =
1811 container_of(base, struct hash_alg_common, base);
1812 struct ahash_alg *alg =
1813 container_of(halg, struct ahash_alg, halg);
1814 struct caam_hash_alg *caam_hash =
1815 container_of(alg, struct caam_hash_alg, ahash_alg);
1816 struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
1817 /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */
1818 static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE,
1819 HASH_MSG_LEN + SHA1_DIGEST_SIZE,
1821 HASH_MSG_LEN + SHA256_DIGEST_SIZE,
1823 HASH_MSG_LEN + SHA512_DIGEST_SIZE };
1824 dma_addr_t dma_addr;
1825 struct caam_drv_private *priv;
1828 * Get a Job ring from Job Ring driver to ensure in-order
1829 * crypto request processing per tfm
1831 ctx->jrdev = caam_jr_alloc();
1832 if (IS_ERR(ctx->jrdev)) {
1833 pr_err("Job Ring Device allocation for transform failed\n");
1834 return PTR_ERR(ctx->jrdev);
1837 priv = dev_get_drvdata(ctx->jrdev->parent);
1839 if (is_xcbc_aes(caam_hash->alg_type)) {
1840 ctx->dir = DMA_TO_DEVICE;
1841 ctx->key_dir = DMA_BIDIRECTIONAL;
1842 ctx->adata.algtype = OP_TYPE_CLASS1_ALG | caam_hash->alg_type;
1844 } else if (is_cmac_aes(caam_hash->alg_type)) {
1845 ctx->dir = DMA_TO_DEVICE;
1846 ctx->key_dir = DMA_NONE;
1847 ctx->adata.algtype = OP_TYPE_CLASS1_ALG | caam_hash->alg_type;
1850 if (priv->era >= 6) {
1851 ctx->dir = DMA_BIDIRECTIONAL;
1852 ctx->key_dir = alg->setkey ? DMA_TO_DEVICE : DMA_NONE;
1854 ctx->dir = DMA_TO_DEVICE;
1855 ctx->key_dir = DMA_NONE;
1857 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type;
1858 ctx->ctx_len = runninglen[(ctx->adata.algtype &
1859 OP_ALG_ALGSEL_SUBMASK) >>
1860 OP_ALG_ALGSEL_SHIFT];
1863 if (ctx->key_dir != DMA_NONE) {
1864 ctx->adata.key_dma = dma_map_single_attrs(ctx->jrdev, ctx->key,
1865 ARRAY_SIZE(ctx->key),
1867 DMA_ATTR_SKIP_CPU_SYNC);
1868 if (dma_mapping_error(ctx->jrdev, ctx->adata.key_dma)) {
1869 dev_err(ctx->jrdev, "unable to map key\n");
1870 caam_jr_free(ctx->jrdev);
1875 dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_update,
1876 offsetof(struct caam_hash_ctx, key),
1877 ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
1878 if (dma_mapping_error(ctx->jrdev, dma_addr)) {
1879 dev_err(ctx->jrdev, "unable to map shared descriptors\n");
1881 if (ctx->key_dir != DMA_NONE)
1882 dma_unmap_single_attrs(ctx->jrdev, ctx->adata.key_dma,
1883 ARRAY_SIZE(ctx->key),
1885 DMA_ATTR_SKIP_CPU_SYNC);
1887 caam_jr_free(ctx->jrdev);
1891 ctx->sh_desc_update_dma = dma_addr;
1892 ctx->sh_desc_update_first_dma = dma_addr +
1893 offsetof(struct caam_hash_ctx,
1894 sh_desc_update_first);
1895 ctx->sh_desc_fin_dma = dma_addr + offsetof(struct caam_hash_ctx,
1897 ctx->sh_desc_digest_dma = dma_addr + offsetof(struct caam_hash_ctx,
1900 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1901 sizeof(struct caam_hash_state));
1904 * For keyed hash algorithms shared descriptors
1905 * will be created later in setkey() callback
1907 return alg->setkey ? 0 : ahash_set_sh_desc(ahash);
1910 static void caam_hash_cra_exit(struct crypto_tfm *tfm)
1912 struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
1914 dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_update_dma,
1915 offsetof(struct caam_hash_ctx, key),
1916 ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
1917 if (ctx->key_dir != DMA_NONE)
1918 dma_unmap_single_attrs(ctx->jrdev, ctx->adata.key_dma,
1919 ARRAY_SIZE(ctx->key), ctx->key_dir,
1920 DMA_ATTR_SKIP_CPU_SYNC);
1921 caam_jr_free(ctx->jrdev);
1924 void caam_algapi_hash_exit(void)
1926 struct caam_hash_alg *t_alg, *n;
1928 if (!hash_list.next)
1931 list_for_each_entry_safe(t_alg, n, &hash_list, entry) {
1932 crypto_unregister_ahash(&t_alg->ahash_alg);
1933 list_del(&t_alg->entry);
1938 static struct caam_hash_alg *
1939 caam_hash_alloc(struct caam_hash_template *template,
1942 struct caam_hash_alg *t_alg;
1943 struct ahash_alg *halg;
1944 struct crypto_alg *alg;
1946 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
1948 pr_err("failed to allocate t_alg\n");
1949 return ERR_PTR(-ENOMEM);
1952 t_alg->ahash_alg = template->template_ahash;
1953 halg = &t_alg->ahash_alg;
1954 alg = &halg->halg.base;
1957 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
1958 template->hmac_name);
1959 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
1960 template->hmac_driver_name);
1962 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
1964 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
1965 template->driver_name);
1966 t_alg->ahash_alg.setkey = NULL;
1968 alg->cra_module = THIS_MODULE;
1969 alg->cra_init = caam_hash_cra_init;
1970 alg->cra_exit = caam_hash_cra_exit;
1971 alg->cra_ctxsize = sizeof(struct caam_hash_ctx);
1972 alg->cra_priority = CAAM_CRA_PRIORITY;
1973 alg->cra_blocksize = template->blocksize;
1974 alg->cra_alignmask = 0;
1975 alg->cra_flags = CRYPTO_ALG_ASYNC;
1977 t_alg->alg_type = template->alg_type;
1982 int caam_algapi_hash_init(struct device *ctrldev)
1985 struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
1986 unsigned int md_limit = SHA512_DIGEST_SIZE;
1987 u32 md_inst, md_vid;
1990 * Register crypto algorithms the device supports. First, identify
1991 * presence and attributes of MD block.
1993 if (priv->era < 10) {
1994 md_vid = (rd_reg32(&priv->ctrl->perfmon.cha_id_ls) &
1995 CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
1996 md_inst = (rd_reg32(&priv->ctrl->perfmon.cha_num_ls) &
1997 CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
1999 u32 mdha = rd_reg32(&priv->ctrl->vreg.mdha);
2001 md_vid = (mdha & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT;
2002 md_inst = mdha & CHA_VER_NUM_MASK;
2006 * Skip registration of any hashing algorithms if MD block
2012 /* Limit digest size based on LP256 */
2013 if (md_vid == CHA_VER_VID_MD_LP256)
2014 md_limit = SHA256_DIGEST_SIZE;
2016 INIT_LIST_HEAD(&hash_list);
2018 /* register crypto algorithms the device supports */
2019 for (i = 0; i < ARRAY_SIZE(driver_hash); i++) {
2020 struct caam_hash_alg *t_alg;
2021 struct caam_hash_template *alg = driver_hash + i;
2023 /* If MD size is not supported by device, skip registration */
2024 if (is_mdha(alg->alg_type) &&
2025 alg->template_ahash.halg.digestsize > md_limit)
2028 /* register hmac version */
2029 t_alg = caam_hash_alloc(alg, true);
2030 if (IS_ERR(t_alg)) {
2031 err = PTR_ERR(t_alg);
2032 pr_warn("%s alg allocation failed\n",
2033 alg->hmac_driver_name);
2037 err = crypto_register_ahash(&t_alg->ahash_alg);
2039 pr_warn("%s alg registration failed: %d\n",
2040 t_alg->ahash_alg.halg.base.cra_driver_name,
2044 list_add_tail(&t_alg->entry, &hash_list);
2046 if ((alg->alg_type & OP_ALG_ALGSEL_MASK) == OP_ALG_ALGSEL_AES)
2049 /* register unkeyed version */
2050 t_alg = caam_hash_alloc(alg, false);
2051 if (IS_ERR(t_alg)) {
2052 err = PTR_ERR(t_alg);
2053 pr_warn("%s alg allocation failed\n", alg->driver_name);
2057 err = crypto_register_ahash(&t_alg->ahash_alg);
2059 pr_warn("%s alg registration failed: %d\n",
2060 t_alg->ahash_alg.halg.base.cra_driver_name,
2064 list_add_tail(&t_alg->entry, &hash_list);