drivers/crypto/caam/caamalg_qi2.c

   1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
   2 /*
   3  * Copyright 2015-2016 Freescale Semiconductor Inc.
   4  * Copyright 2017-2019 NXP
   5  */
   6
   7 #include "compat.h"
   8 #include "regs.h"
   9 #include "caamalg_qi2.h"
  10 #include "dpseci_cmd.h"
  11 #include "desc_constr.h"
  12 #include "error.h"
  13 #include "sg_sw_sec4.h"
  14 #include "sg_sw_qm2.h"
  15 #include "key_gen.h"
  16 #include "caamalg_desc.h"
  17 #include "caamhash_desc.h"
  18 #include "dpseci-debugfs.h"
  19 #include <linux/fsl/mc.h>
  20 #include <soc/fsl/dpaa2-io.h>
  21 #include <soc/fsl/dpaa2-fd.h>
  22
  23 #define CAAM_CRA_PRIORITY       2000
  24
  25 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */
  26 #define CAAM_MAX_KEY_SIZE       (AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE + \
  27                                  SHA512_DIGEST_SIZE * 2)
  28
  29 /*
  30  * This is a a cache of buffers, from which the users of CAAM QI driver
  31  * can allocate short buffers. It's speedier than doing kmalloc on the hotpath.
  32  * NOTE: A more elegant solution would be to have some headroom in the frames
  33  *       being processed. This can be added by the dpaa2-eth driver. This would
  34  *       pose a problem for userspace application processing which cannot
  35  *       know of this limitation. So for now, this will work.
  36  * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here
  37  */
  38 static struct kmem_cache *qi_cache;
  39
  40 struct caam_alg_entry {
  41         struct device *dev;
  42         int class1_alg_type;
  43         int class2_alg_type;
  44         bool rfc3686;
  45         bool geniv;
  46         bool nodkp;
  47 };
  48
  49 struct caam_aead_alg {
  50         struct aead_alg aead;
  51         struct caam_alg_entry caam;
  52         bool registered;
  53 };
  54
  55 struct caam_skcipher_alg {
  56         struct skcipher_alg skcipher;
  57         struct caam_alg_entry caam;
  58         bool registered;
  59 };
  60
  61 /**
  62  * caam_ctx - per-session context
  63  * @flc: Flow Contexts array
  64  * @key:  [authentication key], encryption key
  65  * @flc_dma: I/O virtual addresses of the Flow Contexts
  66  * @key_dma: I/O virtual address of the key
  67  * @dir: DMA direction for mapping key and Flow Contexts
  68  * @dev: dpseci device
  69  * @adata: authentication algorithm details
  70  * @cdata: encryption algorithm details
  71  * @authsize: authentication tag (a.k.a. ICV / MAC) size
  72  */
  73 struct caam_ctx {
  74         struct caam_flc flc[NUM_OP];
  75         u8 key[CAAM_MAX_KEY_SIZE];
  76         dma_addr_t flc_dma[NUM_OP];
  77         dma_addr_t key_dma;
  78         enum dma_data_direction dir;
  79         struct device *dev;
  80         struct alginfo adata;
  81         struct alginfo cdata;
  82         unsigned int authsize;
  83 };
  84
  85 static void *dpaa2_caam_iova_to_virt(struct dpaa2_caam_priv *priv,
  86                                      dma_addr_t iova_addr)
  87 {
  88         phys_addr_t phys_addr;
  89
  90         phys_addr = priv->domain ? iommu_iova_to_phys(priv->domain, iova_addr) :
  91                                    iova_addr;
  92
  93         return phys_to_virt(phys_addr);
  94 }
  95
  96 /*
  97  * qi_cache_zalloc - Allocate buffers from CAAM-QI cache
  98  *
  99  * Allocate data on the hotpath. Instead of using kzalloc, one can use the
 100  * services of the CAAM QI memory cache (backed by kmem_cache). The buffers
 101  * will have a size of CAAM_QI_MEMCACHE_SIZE, which should be sufficient for
 102  * hosting 16 SG entries.
 103  *
 104  * @flags - flags that would be used for the equivalent kmalloc(..) call
 105  *
 106  * Returns a pointer to a retrieved buffer on success or NULL on failure.
 107  */
 108 static inline void *qi_cache_zalloc(gfp_t flags)
 109 {
 110         return kmem_cache_zalloc(qi_cache, flags);
 111 }
 112
 113 /*
 114  * qi_cache_free - Frees buffers allocated from CAAM-QI cache
 115  *
 116  * @obj - buffer previously allocated by qi_cache_zalloc
 117  *
 118  * No checking is being done, the call is a passthrough call to
 119  * kmem_cache_free(...)
 120  */
 121 static inline void qi_cache_free(void *obj)
 122 {
 123         kmem_cache_free(qi_cache, obj);
 124 }
 125
 126 static struct caam_request *to_caam_req(struct crypto_async_request *areq)
 127 {
 128         switch (crypto_tfm_alg_type(areq->tfm)) {
 129         case CRYPTO_ALG_TYPE_SKCIPHER:
 130                 return skcipher_request_ctx(skcipher_request_cast(areq));
 131         case CRYPTO_ALG_TYPE_AEAD:
 132                 return aead_request_ctx(container_of(areq, struct aead_request,
 133                                                      base));
 134         case CRYPTO_ALG_TYPE_AHASH:
 135                 return ahash_request_ctx(ahash_request_cast(areq));
 136         default:
 137                 return ERR_PTR(-EINVAL);
 138         }
 139 }
 140
 141 static void caam_unmap(struct device *dev, struct scatterlist *src,
 142                        struct scatterlist *dst, int src_nents,
 143                        int dst_nents, dma_addr_t iv_dma, int ivsize,
 144                        enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma,
 145                        int qm_sg_bytes)
 146 {
 147         if (dst != src) {
 148                 if (src_nents)
 149                         dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
 150                 if (dst_nents)
 151                         dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
 152         } else {
 153                 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
 154         }
 155
 156         if (iv_dma)
 157                 dma_unmap_single(dev, iv_dma, ivsize, iv_dir);
 158
 159         if (qm_sg_bytes)
 160                 dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE);
 161 }
 162
 163 static int aead_set_sh_desc(struct crypto_aead *aead)
 164 {
 165         struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
 166                                                  typeof(*alg), aead);
 167         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 168         unsigned int ivsize = crypto_aead_ivsize(aead);
 169         struct device *dev = ctx->dev;
 170         struct dpaa2_caam_priv *priv = dev_get_drvdata(dev);
 171         struct caam_flc *flc;
 172         u32 *desc;
 173         u32 ctx1_iv_off = 0;
 174         u32 *nonce = NULL;
 175         unsigned int data_len[2];
 176         u32 inl_mask;
 177         const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
 178                                OP_ALG_AAI_CTR_MOD128);
 179         const bool is_rfc3686 = alg->caam.rfc3686;
 180
 181         if (!ctx->cdata.keylen || !ctx->authsize)
 182                 return 0;
 183
 184         /*
 185          * AES-CTR needs to load IV in CONTEXT1 reg
 186          * at an offset of 128bits (16bytes)
 187          * CONTEXT1[255:128] = IV
 188          */
 189         if (ctr_mode)
 190                 ctx1_iv_off = 16;
 191
 192         /*
 193          * RFC3686 specific:
 194          *      CONTEXT1[255:128] = {NONCE, IV, COUNTER}
 195          */
 196         if (is_rfc3686) {
 197                 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
 198                 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad +
 199                                 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE);
 200         }
 201
 202         /*
 203          * In case |user key| > |derived key|, using DKP<imm,imm> would result
 204          * in invalid opcodes (last bytes of user key) in the resulting
 205          * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key
 206          * addresses are needed.
 207          */
 208         ctx->adata.key_virt = ctx->key;
 209         ctx->adata.key_dma = ctx->key_dma;
 210
 211         ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
 212         ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
 213
 214         data_len[0] = ctx->adata.keylen_pad;
 215         data_len[1] = ctx->cdata.keylen;
 216
 217         /* aead_encrypt shared descriptor */
 218         if (desc_inline_query((alg->caam.geniv ? DESC_QI_AEAD_GIVENC_LEN :
 219                                                  DESC_QI_AEAD_ENC_LEN) +
 220                               (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
 221                               DESC_JOB_IO_LEN, data_len, &inl_mask,
 222                               ARRAY_SIZE(data_len)) < 0)
 223                 return -EINVAL;
 224
 225         ctx->adata.key_inline = !!(inl_mask & 1);
 226         ctx->cdata.key_inline = !!(inl_mask & 2);
 227
 228         flc = &ctx->flc[ENCRYPT];
 229         desc = flc->sh_desc;
 230
 231         if (alg->caam.geniv)
 232                 cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata,
 233                                           ivsize, ctx->authsize, is_rfc3686,
 234                                           nonce, ctx1_iv_off, true,
 235                                           priv->sec_attr.era);
 236         else
 237                 cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata,
 238                                        ivsize, ctx->authsize, is_rfc3686, nonce,
 239                                        ctx1_iv_off, true, priv->sec_attr.era);
 240
 241         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 242         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 243                                    sizeof(flc->flc) + desc_bytes(desc),
 244                                    ctx->dir);
 245
 246         /* aead_decrypt shared descriptor */
 247         if (desc_inline_query(DESC_QI_AEAD_DEC_LEN +
 248                               (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
 249                               DESC_JOB_IO_LEN, data_len, &inl_mask,
 250                               ARRAY_SIZE(data_len)) < 0)
 251                 return -EINVAL;
 252
 253         ctx->adata.key_inline = !!(inl_mask & 1);
 254         ctx->cdata.key_inline = !!(inl_mask & 2);
 255
 256         flc = &ctx->flc[DECRYPT];
 257         desc = flc->sh_desc;
 258         cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata,
 259                                ivsize, ctx->authsize, alg->caam.geniv,
 260                                is_rfc3686, nonce, ctx1_iv_off, true,
 261                                priv->sec_attr.era);
 262         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 263         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 264                                    sizeof(flc->flc) + desc_bytes(desc),
 265                                    ctx->dir);
 266
 267         return 0;
 268 }
 269
 270 static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
 271 {
 272         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 273
 274         ctx->authsize = authsize;
 275         aead_set_sh_desc(authenc);
 276
 277         return 0;
 278 }
 279
 280 static int aead_setkey(struct crypto_aead *aead, const u8 *key,
 281                        unsigned int keylen)
 282 {
 283         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 284         struct device *dev = ctx->dev;
 285         struct crypto_authenc_keys keys;
 286
 287         if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
 288                 goto badkey;
 289
 290         dev_dbg(dev, "keylen %d enckeylen %d authkeylen %d\n",
 291                 keys.authkeylen + keys.enckeylen, keys.enckeylen,
 292                 keys.authkeylen);
 293         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 294                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 295
 296         ctx->adata.keylen = keys.authkeylen;
 297         ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
 298                                               OP_ALG_ALGSEL_MASK);
 299
 300         if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE)
 301                 goto badkey;
 302
 303         memcpy(ctx->key, keys.authkey, keys.authkeylen);
 304         memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
 305         dma_sync_single_for_device(dev, ctx->key_dma, ctx->adata.keylen_pad +
 306                                    keys.enckeylen, ctx->dir);
 307         print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ",
 308                              DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
 309                              ctx->adata.keylen_pad + keys.enckeylen, 1);
 310
 311         ctx->cdata.keylen = keys.enckeylen;
 312
 313         memzero_explicit(&keys, sizeof(keys));
 314         return aead_set_sh_desc(aead);
 315 badkey:
 316         crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
 317         memzero_explicit(&keys, sizeof(keys));
 318         return -EINVAL;
 319 }
 320
 321 static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key,
 322                             unsigned int keylen)
 323 {
 324         struct crypto_authenc_keys keys;
 325         int err;
 326
 327         err = crypto_authenc_extractkeys(&keys, key, keylen);
 328         if (unlikely(err))
 329                 goto badkey;
 330
 331         err = -EINVAL;
 332         if (keys.enckeylen != DES3_EDE_KEY_SIZE)
 333                 goto badkey;
 334
 335         err = crypto_des3_ede_verify_key(crypto_aead_tfm(aead), keys.enckey) ?:
 336               aead_setkey(aead, key, keylen);
 337
 338 out:
 339         memzero_explicit(&keys, sizeof(keys));
 340         return err;
 341
 342 badkey:
 343         crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
 344         goto out;
 345 }
 346
 347 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
 348                                            bool encrypt)
 349 {
 350         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 351         struct caam_request *req_ctx = aead_request_ctx(req);
 352         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
 353         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
 354         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 355         struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
 356                                                  typeof(*alg), aead);
 357         struct device *dev = ctx->dev;
 358         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
 359                       GFP_KERNEL : GFP_ATOMIC;
 360         int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
 361         int src_len, dst_len = 0;
 362         struct aead_edesc *edesc;
 363         dma_addr_t qm_sg_dma, iv_dma = 0;
 364         int ivsize = 0;
 365         unsigned int authsize = ctx->authsize;
 366         int qm_sg_index = 0, qm_sg_nents = 0, qm_sg_bytes;
 367         int in_len, out_len;
 368         struct dpaa2_sg_entry *sg_table;
 369
 370         /* allocate space for base edesc, link tables and IV */
 371         edesc = qi_cache_zalloc(GFP_DMA | flags);
 372         if (unlikely(!edesc)) {
 373                 dev_err(dev, "could not allocate extended descriptor\n");
 374                 return ERR_PTR(-ENOMEM);
 375         }
 376
 377         if (unlikely(req->dst != req->src)) {
 378                 src_len = req->assoclen + req->cryptlen;
 379                 dst_len = src_len + (encrypt ? authsize : (-authsize));
 380
 381                 src_nents = sg_nents_for_len(req->src, src_len);
 382                 if (unlikely(src_nents < 0)) {
 383                         dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
 384                                 src_len);
 385                         qi_cache_free(edesc);
 386                         return ERR_PTR(src_nents);
 387                 }
 388
 389                 dst_nents = sg_nents_for_len(req->dst, dst_len);
 390                 if (unlikely(dst_nents < 0)) {
 391                         dev_err(dev, "Insufficient bytes (%d) in dst S/G\n",
 392                                 dst_len);
 393                         qi_cache_free(edesc);
 394                         return ERR_PTR(dst_nents);
 395                 }
 396
 397                 if (src_nents) {
 398                         mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
 399                                                       DMA_TO_DEVICE);
 400                         if (unlikely(!mapped_src_nents)) {
 401                                 dev_err(dev, "unable to map source\n");
 402                                 qi_cache_free(edesc);
 403                                 return ERR_PTR(-ENOMEM);
 404                         }
 405                 } else {
 406                         mapped_src_nents = 0;
 407                 }
 408
 409                 if (dst_nents) {
 410                         mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents,
 411                                                       DMA_FROM_DEVICE);
 412                         if (unlikely(!mapped_dst_nents)) {
 413                                 dev_err(dev, "unable to map destination\n");
 414                                 dma_unmap_sg(dev, req->src, src_nents,
 415                                              DMA_TO_DEVICE);
 416                                 qi_cache_free(edesc);
 417                                 return ERR_PTR(-ENOMEM);
 418                         }
 419                 } else {
 420                         mapped_dst_nents = 0;
 421                 }
 422         } else {
 423                 src_len = req->assoclen + req->cryptlen +
 424                           (encrypt ? authsize : 0);
 425
 426                 src_nents = sg_nents_for_len(req->src, src_len);
 427                 if (unlikely(src_nents < 0)) {
 428                         dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
 429                                 src_len);
 430                         qi_cache_free(edesc);
 431                         return ERR_PTR(src_nents);
 432                 }
 433
 434                 mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
 435                                               DMA_BIDIRECTIONAL);
 436                 if (unlikely(!mapped_src_nents)) {
 437                         dev_err(dev, "unable to map source\n");
 438                         qi_cache_free(edesc);
 439                         return ERR_PTR(-ENOMEM);
 440                 }
 441         }
 442
 443         if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv)
 444                 ivsize = crypto_aead_ivsize(aead);
 445
 446         /*
 447          * Create S/G table: req->assoclen, [IV,] req->src [, req->dst].
 448          * Input is not contiguous.
 449          * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
 450          * the end of the table by allocating more S/G entries. Logic:
 451          * if (src != dst && output S/G)
 452          *      pad output S/G, if needed
 453          * else if (src == dst && S/G)
 454          *      overlapping S/Gs; pad one of them
 455          * else if (input S/G) ...
 456          *      pad input S/G, if needed
 457          */
 458         qm_sg_nents = 1 + !!ivsize + mapped_src_nents;
 459         if (mapped_dst_nents > 1)
 460                 qm_sg_nents += pad_sg_nents(mapped_dst_nents);
 461         else if ((req->src == req->dst) && (mapped_src_nents > 1))
 462                 qm_sg_nents = max(pad_sg_nents(qm_sg_nents),
 463                                   1 + !!ivsize +
 464                                   pad_sg_nents(mapped_src_nents));
 465         else
 466                 qm_sg_nents = pad_sg_nents(qm_sg_nents);
 467
 468         sg_table = &edesc->sgt[0];
 469         qm_sg_bytes = qm_sg_nents * sizeof(*sg_table);
 470         if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize >
 471                      CAAM_QI_MEMCACHE_SIZE)) {
 472                 dev_err(dev, "No space for %d S/G entries and/or %dB IV\n",
 473                         qm_sg_nents, ivsize);
 474                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
 475                            0, DMA_NONE, 0, 0);
 476                 qi_cache_free(edesc);
 477                 return ERR_PTR(-ENOMEM);
 478         }
 479
 480         if (ivsize) {
 481                 u8 *iv = (u8 *)(sg_table + qm_sg_nents);
 482
 483                 /* Make sure IV is located in a DMAable area */
 484                 memcpy(iv, req->iv, ivsize);
 485
 486                 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
 487                 if (dma_mapping_error(dev, iv_dma)) {
 488                         dev_err(dev, "unable to map IV\n");
 489                         caam_unmap(dev, req->src, req->dst, src_nents,
 490                                    dst_nents, 0, 0, DMA_NONE, 0, 0);
 491                         qi_cache_free(edesc);
 492                         return ERR_PTR(-ENOMEM);
 493                 }
 494         }
 495
 496         edesc->src_nents = src_nents;
 497         edesc->dst_nents = dst_nents;
 498         edesc->iv_dma = iv_dma;
 499
 500         if ((alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK) ==
 501             OP_ALG_ALGSEL_CHACHA20 && ivsize != CHACHAPOLY_IV_SIZE)
 502                 /*
 503                  * The associated data comes already with the IV but we need
 504                  * to skip it when we authenticate or encrypt...
 505                  */
 506                 edesc->assoclen = cpu_to_caam32(req->assoclen - ivsize);
 507         else
 508                 edesc->assoclen = cpu_to_caam32(req->assoclen);
 509         edesc->assoclen_dma = dma_map_single(dev, &edesc->assoclen, 4,
 510                                              DMA_TO_DEVICE);
 511         if (dma_mapping_error(dev, edesc->assoclen_dma)) {
 512                 dev_err(dev, "unable to map assoclen\n");
 513                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
 514                            iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
 515                 qi_cache_free(edesc);
 516                 return ERR_PTR(-ENOMEM);
 517         }
 518
 519         dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0);
 520         qm_sg_index++;
 521         if (ivsize) {
 522                 dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0);
 523                 qm_sg_index++;
 524         }
 525         sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0);
 526         qm_sg_index += mapped_src_nents;
 527
 528         if (mapped_dst_nents > 1)
 529                 sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0);
 530
 531         qm_sg_dma = dma_map_single(dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE);
 532         if (dma_mapping_error(dev, qm_sg_dma)) {
 533                 dev_err(dev, "unable to map S/G table\n");
 534                 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
 535                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
 536                            iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
 537                 qi_cache_free(edesc);
 538                 return ERR_PTR(-ENOMEM);
 539         }
 540
 541         edesc->qm_sg_dma = qm_sg_dma;
 542         edesc->qm_sg_bytes = qm_sg_bytes;
 543
 544         out_len = req->assoclen + req->cryptlen +
 545                   (encrypt ? ctx->authsize : (-ctx->authsize));
 546         in_len = 4 + ivsize + req->assoclen + req->cryptlen;
 547
 548         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
 549         dpaa2_fl_set_final(in_fle, true);
 550         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
 551         dpaa2_fl_set_addr(in_fle, qm_sg_dma);
 552         dpaa2_fl_set_len(in_fle, in_len);
 553
 554         if (req->dst == req->src) {
 555                 if (mapped_src_nents == 1) {
 556                         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
 557                         dpaa2_fl_set_addr(out_fle, sg_dma_address(req->src));
 558                 } else {
 559                         dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
 560                         dpaa2_fl_set_addr(out_fle, qm_sg_dma +
 561                                           (1 + !!ivsize) * sizeof(*sg_table));
 562                 }
 563         } else if (!mapped_dst_nents) {
 564                 /*
 565                  * crypto engine requires the output entry to be present when
 566                  * "frame list" FD is used.
 567                  * Since engine does not support FMT=2'b11 (unused entry type),
 568                  * leaving out_fle zeroized is the best option.
 569                  */
 570                 goto skip_out_fle;
 571         } else if (mapped_dst_nents == 1) {
 572                 dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
 573                 dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst));
 574         } else {
 575                 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
 576                 dpaa2_fl_set_addr(out_fle, qm_sg_dma + qm_sg_index *
 577                                   sizeof(*sg_table));
 578         }
 579
 580         dpaa2_fl_set_len(out_fle, out_len);
 581
 582 skip_out_fle:
 583         return edesc;
 584 }
 585
 586 static int chachapoly_set_sh_desc(struct crypto_aead *aead)
 587 {
 588         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 589         unsigned int ivsize = crypto_aead_ivsize(aead);
 590         struct device *dev = ctx->dev;
 591         struct caam_flc *flc;
 592         u32 *desc;
 593
 594         if (!ctx->cdata.keylen || !ctx->authsize)
 595                 return 0;
 596
 597         flc = &ctx->flc[ENCRYPT];
 598         desc = flc->sh_desc;
 599         cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize,
 600                                ctx->authsize, true, true);
 601         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 602         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 603                                    sizeof(flc->flc) + desc_bytes(desc),
 604                                    ctx->dir);
 605
 606         flc = &ctx->flc[DECRYPT];
 607         desc = flc->sh_desc;
 608         cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize,
 609                                ctx->authsize, false, true);
 610         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 611         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 612                                    sizeof(flc->flc) + desc_bytes(desc),
 613                                    ctx->dir);
 614
 615         return 0;
 616 }
 617
 618 static int chachapoly_setauthsize(struct crypto_aead *aead,
 619                                   unsigned int authsize)
 620 {
 621         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 622
 623         if (authsize != POLY1305_DIGEST_SIZE)
 624                 return -EINVAL;
 625
 626         ctx->authsize = authsize;
 627         return chachapoly_set_sh_desc(aead);
 628 }
 629
 630 static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key,
 631                              unsigned int keylen)
 632 {
 633         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 634         unsigned int ivsize = crypto_aead_ivsize(aead);
 635         unsigned int saltlen = CHACHAPOLY_IV_SIZE - ivsize;
 636
 637         if (keylen != CHACHA_KEY_SIZE + saltlen) {
 638                 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
 639                 return -EINVAL;
 640         }
 641
 642         ctx->cdata.key_virt = key;
 643         ctx->cdata.keylen = keylen - saltlen;
 644
 645         return chachapoly_set_sh_desc(aead);
 646 }
 647
 648 static int gcm_set_sh_desc(struct crypto_aead *aead)
 649 {
 650         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 651         struct device *dev = ctx->dev;
 652         unsigned int ivsize = crypto_aead_ivsize(aead);
 653         struct caam_flc *flc;
 654         u32 *desc;
 655         int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
 656                         ctx->cdata.keylen;
 657
 658         if (!ctx->cdata.keylen || !ctx->authsize)
 659                 return 0;
 660
 661         /*
 662          * AES GCM encrypt shared descriptor
 663          * Job Descriptor and Shared Descriptor
 664          * must fit into the 64-word Descriptor h/w Buffer
 665          */
 666         if (rem_bytes >= DESC_QI_GCM_ENC_LEN) {
 667                 ctx->cdata.key_inline = true;
 668                 ctx->cdata.key_virt = ctx->key;
 669         } else {
 670                 ctx->cdata.key_inline = false;
 671                 ctx->cdata.key_dma = ctx->key_dma;
 672         }
 673
 674         flc = &ctx->flc[ENCRYPT];
 675         desc = flc->sh_desc;
 676         cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, true);
 677         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 678         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 679                                    sizeof(flc->flc) + desc_bytes(desc),
 680                                    ctx->dir);
 681
 682         /*
 683          * Job Descriptor and Shared Descriptors
 684          * must all fit into the 64-word Descriptor h/w Buffer
 685          */
 686         if (rem_bytes >= DESC_QI_GCM_DEC_LEN) {
 687                 ctx->cdata.key_inline = true;
 688                 ctx->cdata.key_virt = ctx->key;
 689         } else {
 690                 ctx->cdata.key_inline = false;
 691                 ctx->cdata.key_dma = ctx->key_dma;
 692         }
 693
 694         flc = &ctx->flc[DECRYPT];
 695         desc = flc->sh_desc;
 696         cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, true);
 697         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 698         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 699                                    sizeof(flc->flc) + desc_bytes(desc),
 700                                    ctx->dir);
 701
 702         return 0;
 703 }
 704
 705 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
 706 {
 707         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 708         int err;
 709
 710         err = crypto_gcm_check_authsize(authsize);
 711         if (err)
 712                 return err;
 713
 714         ctx->authsize = authsize;
 715         gcm_set_sh_desc(authenc);
 716
 717         return 0;
 718 }
 719
 720 static int gcm_setkey(struct crypto_aead *aead,
 721                       const u8 *key, unsigned int keylen)
 722 {
 723         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 724         struct device *dev = ctx->dev;
 725         int ret;
 726
 727         ret = aes_check_keylen(keylen);
 728         if (ret) {
 729                 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
 730                 return ret;
 731         }
 732         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 733                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 734
 735         memcpy(ctx->key, key, keylen);
 736         dma_sync_single_for_device(dev, ctx->key_dma, keylen, ctx->dir);
 737         ctx->cdata.keylen = keylen;
 738
 739         return gcm_set_sh_desc(aead);
 740 }
 741
 742 static int rfc4106_set_sh_desc(struct crypto_aead *aead)
 743 {
 744         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 745         struct device *dev = ctx->dev;
 746         unsigned int ivsize = crypto_aead_ivsize(aead);
 747         struct caam_flc *flc;
 748         u32 *desc;
 749         int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
 750                         ctx->cdata.keylen;
 751
 752         if (!ctx->cdata.keylen || !ctx->authsize)
 753                 return 0;
 754
 755         ctx->cdata.key_virt = ctx->key;
 756
 757         /*
 758          * RFC4106 encrypt shared descriptor
 759          * Job Descriptor and Shared Descriptor
 760          * must fit into the 64-word Descriptor h/w Buffer
 761          */
 762         if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) {
 763                 ctx->cdata.key_inline = true;
 764         } else {
 765                 ctx->cdata.key_inline = false;
 766                 ctx->cdata.key_dma = ctx->key_dma;
 767         }
 768
 769         flc = &ctx->flc[ENCRYPT];
 770         desc = flc->sh_desc;
 771         cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
 772                                   true);
 773         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 774         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 775                                    sizeof(flc->flc) + desc_bytes(desc),
 776                                    ctx->dir);
 777
 778         /*
 779          * Job Descriptor and Shared Descriptors
 780          * must all fit into the 64-word Descriptor h/w Buffer
 781          */
 782         if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) {
 783                 ctx->cdata.key_inline = true;
 784         } else {
 785                 ctx->cdata.key_inline = false;
 786                 ctx->cdata.key_dma = ctx->key_dma;
 787         }
 788
 789         flc = &ctx->flc[DECRYPT];
 790         desc = flc->sh_desc;
 791         cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
 792                                   true);
 793         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 794         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 795                                    sizeof(flc->flc) + desc_bytes(desc),
 796                                    ctx->dir);
 797
 798         return 0;
 799 }
 800
 801 static int rfc4106_setauthsize(struct crypto_aead *authenc,
 802                                unsigned int authsize)
 803 {
 804         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 805         int err;
 806
 807         err = crypto_rfc4106_check_authsize(authsize);
 808         if (err)
 809                 return err;
 810
 811         ctx->authsize = authsize;
 812         rfc4106_set_sh_desc(authenc);
 813
 814         return 0;
 815 }
 816
 817 static int rfc4106_setkey(struct crypto_aead *aead,
 818                           const u8 *key, unsigned int keylen)
 819 {
 820         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 821         struct device *dev = ctx->dev;
 822         int ret;
 823
 824         ret = aes_check_keylen(keylen - 4);
 825         if (ret) {
 826                 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
 827                 return ret;
 828         }
 829
 830         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 831                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 832
 833         memcpy(ctx->key, key, keylen);
 834         /*
 835          * The last four bytes of the key material are used as the salt value
 836          * in the nonce. Update the AES key length.
 837          */
 838         ctx->cdata.keylen = keylen - 4;
 839         dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen,
 840                                    ctx->dir);
 841
 842         return rfc4106_set_sh_desc(aead);
 843 }
 844
 845 static int rfc4543_set_sh_desc(struct crypto_aead *aead)
 846 {
 847         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 848         struct device *dev = ctx->dev;
 849         unsigned int ivsize = crypto_aead_ivsize(aead);
 850         struct caam_flc *flc;
 851         u32 *desc;
 852         int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
 853                         ctx->cdata.keylen;
 854
 855         if (!ctx->cdata.keylen || !ctx->authsize)
 856                 return 0;
 857
 858         ctx->cdata.key_virt = ctx->key;
 859
 860         /*
 861          * RFC4543 encrypt shared descriptor
 862          * Job Descriptor and Shared Descriptor
 863          * must fit into the 64-word Descriptor h/w Buffer
 864          */
 865         if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) {
 866                 ctx->cdata.key_inline = true;
 867         } else {
 868                 ctx->cdata.key_inline = false;
 869                 ctx->cdata.key_dma = ctx->key_dma;
 870         }
 871
 872         flc = &ctx->flc[ENCRYPT];
 873         desc = flc->sh_desc;
 874         cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
 875                                   true);
 876         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 877         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 878                                    sizeof(flc->flc) + desc_bytes(desc),
 879                                    ctx->dir);
 880
 881         /*
 882          * Job Descriptor and Shared Descriptors
 883          * must all fit into the 64-word Descriptor h/w Buffer
 884          */
 885         if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) {
 886                 ctx->cdata.key_inline = true;
 887         } else {
 888                 ctx->cdata.key_inline = false;
 889                 ctx->cdata.key_dma = ctx->key_dma;
 890         }
 891
 892         flc = &ctx->flc[DECRYPT];
 893         desc = flc->sh_desc;
 894         cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
 895                                   true);
 896         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 897         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 898                                    sizeof(flc->flc) + desc_bytes(desc),
 899                                    ctx->dir);
 900
 901         return 0;
 902 }
 903
 904 static int rfc4543_setauthsize(struct crypto_aead *authenc,
 905                                unsigned int authsize)
 906 {
 907         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 908
 909         if (authsize != 16)
 910                 return -EINVAL;
 911
 912         ctx->authsize = authsize;
 913         rfc4543_set_sh_desc(authenc);
 914
 915         return 0;
 916 }
 917
 918 static int rfc4543_setkey(struct crypto_aead *aead,
 919                           const u8 *key, unsigned int keylen)
 920 {
 921         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 922         struct device *dev = ctx->dev;
 923         int ret;
 924
 925         ret = aes_check_keylen(keylen - 4);
 926         if (ret) {
 927                 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
 928                 return ret;
 929         }
 930
 931         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 932                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 933
 934         memcpy(ctx->key, key, keylen);
 935         /*
 936          * The last four bytes of the key material are used as the salt value
 937          * in the nonce. Update the AES key length.
 938          */
 939         ctx->cdata.keylen = keylen - 4;
 940         dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen,
 941                                    ctx->dir);
 942
 943         return rfc4543_set_sh_desc(aead);
 944 }
 945
 946 static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
 947                            unsigned int keylen, const u32 ctx1_iv_off)
 948 {
 949         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
 950         struct caam_skcipher_alg *alg =
 951                 container_of(crypto_skcipher_alg(skcipher),
 952                              struct caam_skcipher_alg, skcipher);
 953         struct device *dev = ctx->dev;
 954         struct caam_flc *flc;
 955         unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
 956         u32 *desc;
 957         const bool is_rfc3686 = alg->caam.rfc3686;
 958
 959         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 960                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 961
 962         ctx->cdata.keylen = keylen;
 963         ctx->cdata.key_virt = key;
 964         ctx->cdata.key_inline = true;
 965
 966         /* skcipher_encrypt shared descriptor */
 967         flc = &ctx->flc[ENCRYPT];
 968         desc = flc->sh_desc;
 969         cnstr_shdsc_skcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686,
 970                                    ctx1_iv_off);
 971         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 972         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 973                                    sizeof(flc->flc) + desc_bytes(desc),
 974                                    ctx->dir);
 975
 976         /* skcipher_decrypt shared descriptor */
 977         flc = &ctx->flc[DECRYPT];
 978         desc = flc->sh_desc;
 979         cnstr_shdsc_skcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686,
 980                                    ctx1_iv_off);
 981         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 982         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 983                                    sizeof(flc->flc) + desc_bytes(desc),
 984                                    ctx->dir);
 985
 986         return 0;
 987 }
 988
 989 static int aes_skcipher_setkey(struct crypto_skcipher *skcipher,
 990                                const u8 *key, unsigned int keylen)
 991 {
 992         int err;
 993
 994         err = aes_check_keylen(keylen);
 995         if (err) {
 996                 crypto_skcipher_set_flags(skcipher,
 997                                           CRYPTO_TFM_RES_BAD_KEY_LEN);
 998                 return err;
 999         }
1000
1001         return skcipher_setkey(skcipher, key, keylen, 0);
1002 }
1003
1004 static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher,
1005                                    const u8 *key, unsigned int keylen)
1006 {
1007         u32 ctx1_iv_off;
1008         int err;
1009
1010         /*
1011          * RFC3686 specific:
1012          *      | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
1013          *      | *key = {KEY, NONCE}
1014          */
1015         ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
1016         keylen -= CTR_RFC3686_NONCE_SIZE;
1017
1018         err = aes_check_keylen(keylen);
1019         if (err) {
1020                 crypto_skcipher_set_flags(skcipher,
1021                                           CRYPTO_TFM_RES_BAD_KEY_LEN);
1022                 return err;
1023         }
1024
1025         return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
1026 }
1027
1028 static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher,
1029                                const u8 *key, unsigned int keylen)
1030 {
1031         u32 ctx1_iv_off;
1032         int err;
1033
1034         /*
1035          * AES-CTR needs to load IV in CONTEXT1 reg
1036          * at an offset of 128bits (16bytes)
1037          * CONTEXT1[255:128] = IV
1038          */
1039         ctx1_iv_off = 16;
1040
1041         err = aes_check_keylen(keylen);
1042         if (err) {
1043                 crypto_skcipher_set_flags(skcipher,
1044                                           CRYPTO_TFM_RES_BAD_KEY_LEN);
1045                 return err;
1046         }
1047
1048         return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
1049 }
1050
1051 static int chacha20_skcipher_setkey(struct crypto_skcipher *skcipher,
1052                                     const u8 *key, unsigned int keylen)
1053 {
1054         if (keylen != CHACHA_KEY_SIZE) {
1055                 crypto_skcipher_set_flags(skcipher,
1056                                           CRYPTO_TFM_RES_BAD_KEY_LEN);
1057                 return -EINVAL;
1058         }
1059
1060         return skcipher_setkey(skcipher, key, keylen, 0);
1061 }
1062
1063 static int des_skcipher_setkey(struct crypto_skcipher *skcipher,
1064                                const u8 *key, unsigned int keylen)
1065 {
1066         return verify_skcipher_des_key(skcipher, key) ?:
1067                skcipher_setkey(skcipher, key, keylen, 0);
1068 }
1069
1070 static int des3_skcipher_setkey(struct crypto_skcipher *skcipher,
1071                                 const u8 *key, unsigned int keylen)
1072 {
1073         return verify_skcipher_des3_key(skcipher, key) ?:
1074                skcipher_setkey(skcipher, key, keylen, 0);
1075 }
1076
1077 static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
1078                                unsigned int keylen)
1079 {
1080         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1081         struct device *dev = ctx->dev;
1082         struct caam_flc *flc;
1083         u32 *desc;
1084
1085         if (keylen != 2 * AES_MIN_KEY_SIZE  && keylen != 2 * AES_MAX_KEY_SIZE) {
1086                 dev_err(dev, "key size mismatch\n");
1087                 crypto_skcipher_set_flags(skcipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
1088                 return -EINVAL;
1089         }
1090
1091         ctx->cdata.keylen = keylen;
1092         ctx->cdata.key_virt = key;
1093         ctx->cdata.key_inline = true;
1094
1095         /* xts_skcipher_encrypt shared descriptor */
1096         flc = &ctx->flc[ENCRYPT];
1097         desc = flc->sh_desc;
1098         cnstr_shdsc_xts_skcipher_encap(desc, &ctx->cdata);
1099         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
1100         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
1101                                    sizeof(flc->flc) + desc_bytes(desc),
1102                                    ctx->dir);
1103
1104         /* xts_skcipher_decrypt shared descriptor */
1105         flc = &ctx->flc[DECRYPT];
1106         desc = flc->sh_desc;
1107         cnstr_shdsc_xts_skcipher_decap(desc, &ctx->cdata);
1108         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
1109         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
1110                                    sizeof(flc->flc) + desc_bytes(desc),
1111                                    ctx->dir);
1112
1113         return 0;
1114 }
1115
1116 static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req)
1117 {
1118         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1119         struct caam_request *req_ctx = skcipher_request_ctx(req);
1120         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
1121         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
1122         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1123         struct device *dev = ctx->dev;
1124         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1125                        GFP_KERNEL : GFP_ATOMIC;
1126         int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
1127         struct skcipher_edesc *edesc;
1128         dma_addr_t iv_dma;
1129         u8 *iv;
1130         int ivsize = crypto_skcipher_ivsize(skcipher);
1131         int dst_sg_idx, qm_sg_ents, qm_sg_bytes;
1132         struct dpaa2_sg_entry *sg_table;
1133
1134         src_nents = sg_nents_for_len(req->src, req->cryptlen);
1135         if (unlikely(src_nents < 0)) {
1136                 dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
1137                         req->cryptlen);
1138                 return ERR_PTR(src_nents);
1139         }
1140
1141         if (unlikely(req->dst != req->src)) {
1142                 dst_nents = sg_nents_for_len(req->dst, req->cryptlen);
1143                 if (unlikely(dst_nents < 0)) {
1144                         dev_err(dev, "Insufficient bytes (%d) in dst S/G\n",
1145                                 req->cryptlen);
1146                         return ERR_PTR(dst_nents);
1147                 }
1148
1149                 mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
1150                                               DMA_TO_DEVICE);
1151                 if (unlikely(!mapped_src_nents)) {
1152                         dev_err(dev, "unable to map source\n");
1153                         return ERR_PTR(-ENOMEM);
1154                 }
1155
1156                 mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents,
1157                                               DMA_FROM_DEVICE);
1158                 if (unlikely(!mapped_dst_nents)) {
1159                         dev_err(dev, "unable to map destination\n");
1160                         dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE);
1161                         return ERR_PTR(-ENOMEM);
1162                 }
1163         } else {
1164                 mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
1165                                               DMA_BIDIRECTIONAL);
1166                 if (unlikely(!mapped_src_nents)) {
1167                         dev_err(dev, "unable to map source\n");
1168                         return ERR_PTR(-ENOMEM);
1169                 }
1170         }
1171
1172         qm_sg_ents = 1 + mapped_src_nents;
1173         dst_sg_idx = qm_sg_ents;
1174
1175         /*
1176          * Input, output HW S/G tables: [IV, src][dst, IV]
1177          * IV entries point to the same buffer
1178          * If src == dst, S/G entries are reused (S/G tables overlap)
1179          *
1180          * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
1181          * the end of the table by allocating more S/G entries.
1182          */
1183         if (req->src != req->dst)
1184                 qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1);
1185         else
1186                 qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents);
1187
1188         qm_sg_bytes = qm_sg_ents * sizeof(struct dpaa2_sg_entry);
1189         if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes +
1190                      ivsize > CAAM_QI_MEMCACHE_SIZE)) {
1191                 dev_err(dev, "No space for %d S/G entries and/or %dB IV\n",
1192                         qm_sg_ents, ivsize);
1193                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
1194                            0, DMA_NONE, 0, 0);
1195                 return ERR_PTR(-ENOMEM);
1196         }
1197
1198         /* allocate space for base edesc, link tables and IV */
1199         edesc = qi_cache_zalloc(GFP_DMA | flags);
1200         if (unlikely(!edesc)) {
1201                 dev_err(dev, "could not allocate extended descriptor\n");
1202                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
1203                            0, DMA_NONE, 0, 0);
1204                 return ERR_PTR(-ENOMEM);
1205         }
1206
1207         /* Make sure IV is located in a DMAable area */
1208         sg_table = &edesc->sgt[0];
1209         iv = (u8 *)(sg_table + qm_sg_ents);
1210         memcpy(iv, req->iv, ivsize);
1211
1212         iv_dma = dma_map_single(dev, iv, ivsize, DMA_BIDIRECTIONAL);
1213         if (dma_mapping_error(dev, iv_dma)) {
1214                 dev_err(dev, "unable to map IV\n");
1215                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
1216                            0, DMA_NONE, 0, 0);
1217                 qi_cache_free(edesc);
1218                 return ERR_PTR(-ENOMEM);
1219         }
1220
1221         edesc->src_nents = src_nents;
1222         edesc->dst_nents = dst_nents;
1223         edesc->iv_dma = iv_dma;
1224         edesc->qm_sg_bytes = qm_sg_bytes;
1225
1226         dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0);
1227         sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0);
1228
1229         if (req->src != req->dst)
1230                 sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0);
1231
1232         dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma,
1233                          ivsize, 0);
1234
1235         edesc->qm_sg_dma = dma_map_single(dev, sg_table, edesc->qm_sg_bytes,
1236                                           DMA_TO_DEVICE);
1237         if (dma_mapping_error(dev, edesc->qm_sg_dma)) {
1238                 dev_err(dev, "unable to map S/G table\n");
1239                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
1240                            iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0);
1241                 qi_cache_free(edesc);
1242                 return ERR_PTR(-ENOMEM);
1243         }
1244
1245         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
1246         dpaa2_fl_set_final(in_fle, true);
1247         dpaa2_fl_set_len(in_fle, req->cryptlen + ivsize);
1248         dpaa2_fl_set_len(out_fle, req->cryptlen + ivsize);
1249
1250         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
1251         dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
1252
1253         dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
1254
1255         if (req->src == req->dst)
1256                 dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma +
1257                                   sizeof(*sg_table));
1258         else
1259                 dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + dst_sg_idx *
1260                                   sizeof(*sg_table));
1261
1262         return edesc;
1263 }
1264
1265 static void aead_unmap(struct device *dev, struct aead_edesc *edesc,
1266                        struct aead_request *req)
1267 {
1268         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1269         int ivsize = crypto_aead_ivsize(aead);
1270
1271         caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
1272                    edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma,
1273                    edesc->qm_sg_bytes);
1274         dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
1275 }
1276
1277 static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc,
1278                            struct skcipher_request *req)
1279 {
1280         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1281         int ivsize = crypto_skcipher_ivsize(skcipher);
1282
1283         caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
1284                    edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma,
1285                    edesc->qm_sg_bytes);
1286 }
1287
1288 static void aead_encrypt_done(void *cbk_ctx, u32 status)
1289 {
1290         struct crypto_async_request *areq = cbk_ctx;
1291         struct aead_request *req = container_of(areq, struct aead_request,
1292                                                 base);
1293         struct caam_request *req_ctx = to_caam_req(areq);
1294         struct aead_edesc *edesc = req_ctx->edesc;
1295         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1296         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1297         int ecode = 0;
1298
1299         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1300
1301         if (unlikely(status))
1302                 ecode = caam_qi2_strstatus(ctx->dev, status);
1303
1304         aead_unmap(ctx->dev, edesc, req);
1305         qi_cache_free(edesc);
1306         aead_request_complete(req, ecode);
1307 }
1308
1309 static void aead_decrypt_done(void *cbk_ctx, u32 status)
1310 {
1311         struct crypto_async_request *areq = cbk_ctx;
1312         struct aead_request *req = container_of(areq, struct aead_request,
1313                                                 base);
1314         struct caam_request *req_ctx = to_caam_req(areq);
1315         struct aead_edesc *edesc = req_ctx->edesc;
1316         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1317         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1318         int ecode = 0;
1319
1320         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1321
1322         if (unlikely(status))
1323                 ecode = caam_qi2_strstatus(ctx->dev, status);
1324
1325         aead_unmap(ctx->dev, edesc, req);
1326         qi_cache_free(edesc);
1327         aead_request_complete(req, ecode);
1328 }
1329
1330 static int aead_encrypt(struct aead_request *req)
1331 {
1332         struct aead_edesc *edesc;
1333         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1334         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1335         struct caam_request *caam_req = aead_request_ctx(req);
1336         int ret;
1337
1338         /* allocate extended descriptor */
1339         edesc = aead_edesc_alloc(req, true);
1340         if (IS_ERR(edesc))
1341                 return PTR_ERR(edesc);
1342
1343         caam_req->flc = &ctx->flc[ENCRYPT];
1344         caam_req->flc_dma = ctx->flc_dma[ENCRYPT];
1345         caam_req->cbk = aead_encrypt_done;
1346         caam_req->ctx = &req->base;
1347         caam_req->edesc = edesc;
1348         ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1349         if (ret != -EINPROGRESS &&
1350             !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1351                 aead_unmap(ctx->dev, edesc, req);
1352                 qi_cache_free(edesc);
1353         }
1354
1355         return ret;
1356 }
1357
1358 static int aead_decrypt(struct aead_request *req)
1359 {
1360         struct aead_edesc *edesc;
1361         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1362         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1363         struct caam_request *caam_req = aead_request_ctx(req);
1364         int ret;
1365
1366         /* allocate extended descriptor */
1367         edesc = aead_edesc_alloc(req, false);
1368         if (IS_ERR(edesc))
1369                 return PTR_ERR(edesc);
1370
1371         caam_req->flc = &ctx->flc[DECRYPT];
1372         caam_req->flc_dma = ctx->flc_dma[DECRYPT];
1373         caam_req->cbk = aead_decrypt_done;
1374         caam_req->ctx = &req->base;
1375         caam_req->edesc = edesc;
1376         ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1377         if (ret != -EINPROGRESS &&
1378             !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1379                 aead_unmap(ctx->dev, edesc, req);
1380                 qi_cache_free(edesc);
1381         }
1382
1383         return ret;
1384 }
1385
1386 static int ipsec_gcm_encrypt(struct aead_request *req)
1387 {
1388         return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_encrypt(req);
1389 }
1390
1391 static int ipsec_gcm_decrypt(struct aead_request *req)
1392 {
1393         return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_decrypt(req);
1394 }
1395
1396 static void skcipher_encrypt_done(void *cbk_ctx, u32 status)
1397 {
1398         struct crypto_async_request *areq = cbk_ctx;
1399         struct skcipher_request *req = skcipher_request_cast(areq);
1400         struct caam_request *req_ctx = to_caam_req(areq);
1401         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1402         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1403         struct skcipher_edesc *edesc = req_ctx->edesc;
1404         int ecode = 0;
1405         int ivsize = crypto_skcipher_ivsize(skcipher);
1406
1407         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1408
1409         if (unlikely(status))
1410                 ecode = caam_qi2_strstatus(ctx->dev, status);
1411
1412         print_hex_dump_debug("dstiv  @" __stringify(__LINE__)": ",
1413                              DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
1414                              edesc->src_nents > 1 ? 100 : ivsize, 1);
1415         caam_dump_sg("dst    @" __stringify(__LINE__)": ",
1416                      DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
1417                      edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
1418
1419         skcipher_unmap(ctx->dev, edesc, req);
1420
1421         /*
1422          * The crypto API expects us to set the IV (req->iv) to the last
1423          * ciphertext block (CBC mode) or last counter (CTR mode).
1424          * This is used e.g. by the CTS mode.
1425          */
1426         if (!ecode)
1427                 memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes,
1428                        ivsize);
1429
1430         qi_cache_free(edesc);
1431         skcipher_request_complete(req, ecode);
1432 }
1433
1434 static void skcipher_decrypt_done(void *cbk_ctx, u32 status)
1435 {
1436         struct crypto_async_request *areq = cbk_ctx;
1437         struct skcipher_request *req = skcipher_request_cast(areq);
1438         struct caam_request *req_ctx = to_caam_req(areq);
1439         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1440         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1441         struct skcipher_edesc *edesc = req_ctx->edesc;
1442         int ecode = 0;
1443         int ivsize = crypto_skcipher_ivsize(skcipher);
1444
1445         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1446
1447         if (unlikely(status))
1448                 ecode = caam_qi2_strstatus(ctx->dev, status);
1449
1450         print_hex_dump_debug("dstiv  @" __stringify(__LINE__)": ",
1451                              DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
1452                              edesc->src_nents > 1 ? 100 : ivsize, 1);
1453         caam_dump_sg("dst    @" __stringify(__LINE__)": ",
1454                      DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
1455                      edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
1456
1457         skcipher_unmap(ctx->dev, edesc, req);
1458
1459         /*
1460          * The crypto API expects us to set the IV (req->iv) to the last
1461          * ciphertext block (CBC mode) or last counter (CTR mode).
1462          * This is used e.g. by the CTS mode.
1463          */
1464         if (!ecode)
1465                 memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes,
1466                        ivsize);
1467
1468         qi_cache_free(edesc);
1469         skcipher_request_complete(req, ecode);
1470 }
1471
1472 static int skcipher_encrypt(struct skcipher_request *req)
1473 {
1474         struct skcipher_edesc *edesc;
1475         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1476         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1477         struct caam_request *caam_req = skcipher_request_ctx(req);
1478         int ret;
1479
1480         if (!req->cryptlen)
1481                 return 0;
1482
1483         /* allocate extended descriptor */
1484         edesc = skcipher_edesc_alloc(req);
1485         if (IS_ERR(edesc))
1486                 return PTR_ERR(edesc);
1487
1488         caam_req->flc = &ctx->flc[ENCRYPT];
1489         caam_req->flc_dma = ctx->flc_dma[ENCRYPT];
1490         caam_req->cbk = skcipher_encrypt_done;
1491         caam_req->ctx = &req->base;
1492         caam_req->edesc = edesc;
1493         ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1494         if (ret != -EINPROGRESS &&
1495             !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1496                 skcipher_unmap(ctx->dev, edesc, req);
1497                 qi_cache_free(edesc);
1498         }
1499
1500         return ret;
1501 }
1502
1503 static int skcipher_decrypt(struct skcipher_request *req)
1504 {
1505         struct skcipher_edesc *edesc;
1506         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1507         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1508         struct caam_request *caam_req = skcipher_request_ctx(req);
1509         int ret;
1510
1511         if (!req->cryptlen)
1512                 return 0;
1513         /* allocate extended descriptor */
1514         edesc = skcipher_edesc_alloc(req);
1515         if (IS_ERR(edesc))
1516                 return PTR_ERR(edesc);
1517
1518         caam_req->flc = &ctx->flc[DECRYPT];
1519         caam_req->flc_dma = ctx->flc_dma[DECRYPT];
1520         caam_req->cbk = skcipher_decrypt_done;
1521         caam_req->ctx = &req->base;
1522         caam_req->edesc = edesc;
1523         ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1524         if (ret != -EINPROGRESS &&
1525             !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1526                 skcipher_unmap(ctx->dev, edesc, req);
1527                 qi_cache_free(edesc);
1528         }
1529
1530         return ret;
1531 }
1532
1533 static int caam_cra_init(struct caam_ctx *ctx, struct caam_alg_entry *caam,
1534                          bool uses_dkp)
1535 {
1536         dma_addr_t dma_addr;
1537         int i;
1538
1539         /* copy descriptor header template value */
1540         ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
1541         ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
1542
1543         ctx->dev = caam->dev;
1544         ctx->dir = uses_dkp ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
1545
1546         dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc,
1547                                         offsetof(struct caam_ctx, flc_dma),
1548                                         ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
1549         if (dma_mapping_error(ctx->dev, dma_addr)) {
1550                 dev_err(ctx->dev, "unable to map key, shared descriptors\n");
1551                 return -ENOMEM;
1552         }
1553
1554         for (i = 0; i < NUM_OP; i++)
1555                 ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]);
1556         ctx->key_dma = dma_addr + NUM_OP * sizeof(ctx->flc[0]);
1557
1558         return 0;
1559 }
1560
1561 static int caam_cra_init_skcipher(struct crypto_skcipher *tfm)
1562 {
1563         struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
1564         struct caam_skcipher_alg *caam_alg =
1565                 container_of(alg, typeof(*caam_alg), skcipher);
1566
1567         crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_request));
1568         return caam_cra_init(crypto_skcipher_ctx(tfm), &caam_alg->caam, false);
1569 }
1570
1571 static int caam_cra_init_aead(struct crypto_aead *tfm)
1572 {
1573         struct aead_alg *alg = crypto_aead_alg(tfm);
1574         struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg),
1575                                                       aead);
1576
1577         crypto_aead_set_reqsize(tfm, sizeof(struct caam_request));
1578         return caam_cra_init(crypto_aead_ctx(tfm), &caam_alg->caam,
1579                              !caam_alg->caam.nodkp);
1580 }
1581
1582 static void caam_exit_common(struct caam_ctx *ctx)
1583 {
1584         dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0],
1585                                offsetof(struct caam_ctx, flc_dma), ctx->dir,
1586                                DMA_ATTR_SKIP_CPU_SYNC);
1587 }
1588
1589 static void caam_cra_exit(struct crypto_skcipher *tfm)
1590 {
1591         caam_exit_common(crypto_skcipher_ctx(tfm));
1592 }
1593
1594 static void caam_cra_exit_aead(struct crypto_aead *tfm)
1595 {
1596         caam_exit_common(crypto_aead_ctx(tfm));
1597 }
1598
1599 static struct caam_skcipher_alg driver_algs[] = {
1600         {
1601                 .skcipher = {
1602                         .base = {
1603                                 .cra_name = "cbc(aes)",
1604                                 .cra_driver_name = "cbc-aes-caam-qi2",
1605                                 .cra_blocksize = AES_BLOCK_SIZE,
1606                         },
1607                         .setkey = aes_skcipher_setkey,
1608                         .encrypt = skcipher_encrypt,
1609                         .decrypt = skcipher_decrypt,
1610                         .min_keysize = AES_MIN_KEY_SIZE,
1611                         .max_keysize = AES_MAX_KEY_SIZE,
1612                         .ivsize = AES_BLOCK_SIZE,
1613                 },
1614                 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1615         },
1616         {
1617                 .skcipher = {
1618                         .base = {
1619                                 .cra_name = "cbc(des3_ede)",
1620                                 .cra_driver_name = "cbc-3des-caam-qi2",
1621                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1622                         },
1623                         .setkey = des3_skcipher_setkey,
1624                         .encrypt = skcipher_encrypt,
1625                         .decrypt = skcipher_decrypt,
1626                         .min_keysize = DES3_EDE_KEY_SIZE,
1627                         .max_keysize = DES3_EDE_KEY_SIZE,
1628                         .ivsize = DES3_EDE_BLOCK_SIZE,
1629                 },
1630                 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1631         },
1632         {
1633                 .skcipher = {
1634                         .base = {
1635                                 .cra_name = "cbc(des)",
1636                                 .cra_driver_name = "cbc-des-caam-qi2",
1637                                 .cra_blocksize = DES_BLOCK_SIZE,
1638                         },
1639                         .setkey = des_skcipher_setkey,
1640                         .encrypt = skcipher_encrypt,
1641                         .decrypt = skcipher_decrypt,
1642                         .min_keysize = DES_KEY_SIZE,
1643                         .max_keysize = DES_KEY_SIZE,
1644                         .ivsize = DES_BLOCK_SIZE,
1645                 },
1646                 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
1647         },
1648         {
1649                 .skcipher = {
1650                         .base = {
1651                                 .cra_name = "ctr(aes)",
1652                                 .cra_driver_name = "ctr-aes-caam-qi2",
1653                                 .cra_blocksize = 1,
1654                         },
1655                         .setkey = ctr_skcipher_setkey,
1656                         .encrypt = skcipher_encrypt,
1657                         .decrypt = skcipher_decrypt,
1658                         .min_keysize = AES_MIN_KEY_SIZE,
1659                         .max_keysize = AES_MAX_KEY_SIZE,
1660                         .ivsize = AES_BLOCK_SIZE,
1661                         .chunksize = AES_BLOCK_SIZE,
1662                 },
1663                 .caam.class1_alg_type = OP_ALG_ALGSEL_AES |
1664                                         OP_ALG_AAI_CTR_MOD128,
1665         },
1666         {
1667                 .skcipher = {
1668                         .base = {
1669                                 .cra_name = "rfc3686(ctr(aes))",
1670                                 .cra_driver_name = "rfc3686-ctr-aes-caam-qi2",
1671                                 .cra_blocksize = 1,
1672                         },
1673                         .setkey = rfc3686_skcipher_setkey,
1674                         .encrypt = skcipher_encrypt,
1675                         .decrypt = skcipher_decrypt,
1676                         .min_keysize = AES_MIN_KEY_SIZE +
1677                                        CTR_RFC3686_NONCE_SIZE,
1678                         .max_keysize = AES_MAX_KEY_SIZE +
1679                                        CTR_RFC3686_NONCE_SIZE,
1680                         .ivsize = CTR_RFC3686_IV_SIZE,
1681                         .chunksize = AES_BLOCK_SIZE,
1682                 },
1683                 .caam = {
1684                         .class1_alg_type = OP_ALG_ALGSEL_AES |
1685                                            OP_ALG_AAI_CTR_MOD128,
1686                         .rfc3686 = true,
1687                 },
1688         },
1689         {
1690                 .skcipher = {
1691                         .base = {
1692                                 .cra_name = "xts(aes)",
1693                                 .cra_driver_name = "xts-aes-caam-qi2",
1694                                 .cra_blocksize = AES_BLOCK_SIZE,
1695                         },
1696                         .setkey = xts_skcipher_setkey,
1697                         .encrypt = skcipher_encrypt,
1698                         .decrypt = skcipher_decrypt,
1699                         .min_keysize = 2 * AES_MIN_KEY_SIZE,
1700                         .max_keysize = 2 * AES_MAX_KEY_SIZE,
1701                         .ivsize = AES_BLOCK_SIZE,
1702                 },
1703                 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS,
1704         },
1705         {
1706                 .skcipher = {
1707                         .base = {
1708                                 .cra_name = "chacha20",
1709                                 .cra_driver_name = "chacha20-caam-qi2",
1710                                 .cra_blocksize = 1,
1711                         },
1712                         .setkey = chacha20_skcipher_setkey,
1713                         .encrypt = skcipher_encrypt,
1714                         .decrypt = skcipher_decrypt,
1715                         .min_keysize = CHACHA_KEY_SIZE,
1716                         .max_keysize = CHACHA_KEY_SIZE,
1717                         .ivsize = CHACHA_IV_SIZE,
1718                 },
1719                 .caam.class1_alg_type = OP_ALG_ALGSEL_CHACHA20,
1720         },
1721 };
1722
1723 static struct caam_aead_alg driver_aeads[] = {
1724         {
1725                 .aead = {
1726                         .base = {
1727                                 .cra_name = "rfc4106(gcm(aes))",
1728                                 .cra_driver_name = "rfc4106-gcm-aes-caam-qi2",
1729                                 .cra_blocksize = 1,
1730                         },
1731                         .setkey = rfc4106_setkey,
1732                         .setauthsize = rfc4106_setauthsize,
1733                         .encrypt = ipsec_gcm_encrypt,
1734                         .decrypt = ipsec_gcm_decrypt,
1735                         .ivsize = 8,
1736                         .maxauthsize = AES_BLOCK_SIZE,
1737                 },
1738                 .caam = {
1739                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1740                         .nodkp = true,
1741                 },
1742         },
1743         {
1744                 .aead = {
1745                         .base = {
1746                                 .cra_name = "rfc4543(gcm(aes))",
1747                                 .cra_driver_name = "rfc4543-gcm-aes-caam-qi2",
1748                                 .cra_blocksize = 1,
1749                         },
1750                         .setkey = rfc4543_setkey,
1751                         .setauthsize = rfc4543_setauthsize,
1752                         .encrypt = ipsec_gcm_encrypt,
1753                         .decrypt = ipsec_gcm_decrypt,
1754                         .ivsize = 8,
1755                         .maxauthsize = AES_BLOCK_SIZE,
1756                 },
1757                 .caam = {
1758                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1759                         .nodkp = true,
1760                 },
1761         },
1762         /* Galois Counter Mode */
1763         {
1764                 .aead = {
1765                         .base = {
1766                                 .cra_name = "gcm(aes)",
1767                                 .cra_driver_name = "gcm-aes-caam-qi2",
1768                                 .cra_blocksize = 1,
1769                         },
1770                         .setkey = gcm_setkey,
1771                         .setauthsize = gcm_setauthsize,
1772                         .encrypt = aead_encrypt,
1773                         .decrypt = aead_decrypt,
1774                         .ivsize = 12,
1775                         .maxauthsize = AES_BLOCK_SIZE,
1776                 },
1777                 .caam = {
1778                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1779                         .nodkp = true,
1780                 }
1781         },
1782         /* single-pass ipsec_esp descriptor */
1783         {
1784                 .aead = {
1785                         .base = {
1786                                 .cra_name = "authenc(hmac(md5),cbc(aes))",
1787                                 .cra_driver_name = "authenc-hmac-md5-"
1788                                                    "cbc-aes-caam-qi2",
1789                                 .cra_blocksize = AES_BLOCK_SIZE,
1790                         },
1791                         .setkey = aead_setkey,
1792                         .setauthsize = aead_setauthsize,
1793                         .encrypt = aead_encrypt,
1794                         .decrypt = aead_decrypt,
1795                         .ivsize = AES_BLOCK_SIZE,
1796                         .maxauthsize = MD5_DIGEST_SIZE,
1797                 },
1798                 .caam = {
1799                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1800                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
1801                                            OP_ALG_AAI_HMAC_PRECOMP,
1802                 }
1803         },
1804         {
1805                 .aead = {
1806                         .base = {
1807                                 .cra_name = "echainiv(authenc(hmac(md5),"
1808                                             "cbc(aes)))",
1809                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
1810                                                    "cbc-aes-caam-qi2",
1811                                 .cra_blocksize = AES_BLOCK_SIZE,
1812                         },
1813                         .setkey = aead_setkey,
1814                         .setauthsize = aead_setauthsize,
1815                         .encrypt = aead_encrypt,
1816                         .decrypt = aead_decrypt,
1817                         .ivsize = AES_BLOCK_SIZE,
1818                         .maxauthsize = MD5_DIGEST_SIZE,
1819                 },
1820                 .caam = {
1821                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1822                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
1823                                            OP_ALG_AAI_HMAC_PRECOMP,
1824                         .geniv = true,
1825                 }
1826         },
1827         {
1828                 .aead = {
1829                         .base = {
1830                                 .cra_name = "authenc(hmac(sha1),cbc(aes))",
1831                                 .cra_driver_name = "authenc-hmac-sha1-"
1832                                                    "cbc-aes-caam-qi2",
1833                                 .cra_blocksize = AES_BLOCK_SIZE,
1834                         },
1835                         .setkey = aead_setkey,
1836                         .setauthsize = aead_setauthsize,
1837                         .encrypt = aead_encrypt,
1838                         .decrypt = aead_decrypt,
1839                         .ivsize = AES_BLOCK_SIZE,
1840                         .maxauthsize = SHA1_DIGEST_SIZE,
1841                 },
1842                 .caam = {
1843                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1844                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1845                                            OP_ALG_AAI_HMAC_PRECOMP,
1846                 }
1847         },
1848         {
1849                 .aead = {
1850                         .base = {
1851                                 .cra_name = "echainiv(authenc(hmac(sha1),"
1852                                             "cbc(aes)))",
1853                                 .cra_driver_name = "echainiv-authenc-"
1854                                                    "hmac-sha1-cbc-aes-caam-qi2",
1855                                 .cra_blocksize = AES_BLOCK_SIZE,
1856                         },
1857                         .setkey = aead_setkey,
1858                         .setauthsize = aead_setauthsize,
1859                         .encrypt = aead_encrypt,
1860                         .decrypt = aead_decrypt,
1861                         .ivsize = AES_BLOCK_SIZE,
1862                         .maxauthsize = SHA1_DIGEST_SIZE,
1863                 },
1864                 .caam = {
1865                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1866                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1867                                            OP_ALG_AAI_HMAC_PRECOMP,
1868                         .geniv = true,
1869                 },
1870         },
1871         {
1872                 .aead = {
1873                         .base = {
1874                                 .cra_name = "authenc(hmac(sha224),cbc(aes))",
1875                                 .cra_driver_name = "authenc-hmac-sha224-"
1876                                                    "cbc-aes-caam-qi2",
1877                                 .cra_blocksize = AES_BLOCK_SIZE,
1878                         },
1879                         .setkey = aead_setkey,
1880                         .setauthsize = aead_setauthsize,
1881                         .encrypt = aead_encrypt,
1882                         .decrypt = aead_decrypt,
1883                         .ivsize = AES_BLOCK_SIZE,
1884                         .maxauthsize = SHA224_DIGEST_SIZE,
1885                 },
1886                 .caam = {
1887                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1888                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1889                                            OP_ALG_AAI_HMAC_PRECOMP,
1890                 }
1891         },
1892         {
1893                 .aead = {
1894                         .base = {
1895                                 .cra_name = "echainiv(authenc(hmac(sha224),"
1896                                             "cbc(aes)))",
1897                                 .cra_driver_name = "echainiv-authenc-"
1898                                                    "hmac-sha224-cbc-aes-caam-qi2",
1899                                 .cra_blocksize = AES_BLOCK_SIZE,
1900                         },
1901                         .setkey = aead_setkey,
1902                         .setauthsize = aead_setauthsize,
1903                         .encrypt = aead_encrypt,
1904                         .decrypt = aead_decrypt,
1905                         .ivsize = AES_BLOCK_SIZE,
1906                         .maxauthsize = SHA224_DIGEST_SIZE,
1907                 },
1908                 .caam = {
1909                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1910                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1911                                            OP_ALG_AAI_HMAC_PRECOMP,
1912                         .geniv = true,
1913                 }
1914         },
1915         {
1916                 .aead = {
1917                         .base = {
1918                                 .cra_name = "authenc(hmac(sha256),cbc(aes))",
1919                                 .cra_driver_name = "authenc-hmac-sha256-"
1920                                                    "cbc-aes-caam-qi2",
1921                                 .cra_blocksize = AES_BLOCK_SIZE,
1922                         },
1923                         .setkey = aead_setkey,
1924                         .setauthsize = aead_setauthsize,
1925                         .encrypt = aead_encrypt,
1926                         .decrypt = aead_decrypt,
1927                         .ivsize = AES_BLOCK_SIZE,
1928                         .maxauthsize = SHA256_DIGEST_SIZE,
1929                 },
1930                 .caam = {
1931                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1932                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
1933                                            OP_ALG_AAI_HMAC_PRECOMP,
1934                 }
1935         },
1936         {
1937                 .aead = {
1938                         .base = {
1939                                 .cra_name = "echainiv(authenc(hmac(sha256),"
1940                                             "cbc(aes)))",
1941                                 .cra_driver_name = "echainiv-authenc-"
1942                                                    "hmac-sha256-cbc-aes-"
1943                                                    "caam-qi2",
1944                                 .cra_blocksize = AES_BLOCK_SIZE,
1945                         },
1946                         .setkey = aead_setkey,
1947                         .setauthsize = aead_setauthsize,
1948                         .encrypt = aead_encrypt,
1949                         .decrypt = aead_decrypt,
1950                         .ivsize = AES_BLOCK_SIZE,
1951                         .maxauthsize = SHA256_DIGEST_SIZE,
1952                 },
1953                 .caam = {
1954                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1955                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
1956                                            OP_ALG_AAI_HMAC_PRECOMP,
1957                         .geniv = true,
1958                 }
1959         },
1960         {
1961                 .aead = {
1962                         .base = {
1963                                 .cra_name = "authenc(hmac(sha384),cbc(aes))",
1964                                 .cra_driver_name = "authenc-hmac-sha384-"
1965                                                    "cbc-aes-caam-qi2",
1966                                 .cra_blocksize = AES_BLOCK_SIZE,
1967                         },
1968                         .setkey = aead_setkey,
1969                         .setauthsize = aead_setauthsize,
1970                         .encrypt = aead_encrypt,
1971                         .decrypt = aead_decrypt,
1972                         .ivsize = AES_BLOCK_SIZE,
1973                         .maxauthsize = SHA384_DIGEST_SIZE,
1974                 },
1975                 .caam = {
1976                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1977                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
1978                                            OP_ALG_AAI_HMAC_PRECOMP,
1979                 }
1980         },
1981         {
1982                 .aead = {
1983                         .base = {
1984                                 .cra_name = "echainiv(authenc(hmac(sha384),"
1985                                             "cbc(aes)))",
1986                                 .cra_driver_name = "echainiv-authenc-"
1987                                                    "hmac-sha384-cbc-aes-"
1988                                                    "caam-qi2",
1989                                 .cra_blocksize = AES_BLOCK_SIZE,
1990                         },
1991                         .setkey = aead_setkey,
1992                         .setauthsize = aead_setauthsize,
1993                         .encrypt = aead_encrypt,
1994                         .decrypt = aead_decrypt,
1995                         .ivsize = AES_BLOCK_SIZE,
1996                         .maxauthsize = SHA384_DIGEST_SIZE,
1997                 },
1998                 .caam = {
1999                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2000                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2001                                            OP_ALG_AAI_HMAC_PRECOMP,
2002                         .geniv = true,
2003                 }
2004         },
2005         {
2006                 .aead = {
2007                         .base = {
2008                                 .cra_name = "authenc(hmac(sha512),cbc(aes))",
2009                                 .cra_driver_name = "authenc-hmac-sha512-"
2010                                                    "cbc-aes-caam-qi2",
2011                                 .cra_blocksize = AES_BLOCK_SIZE,
2012                         },
2013                         .setkey = aead_setkey,
2014                         .setauthsize = aead_setauthsize,
2015                         .encrypt = aead_encrypt,
2016                         .decrypt = aead_decrypt,
2017                         .ivsize = AES_BLOCK_SIZE,
2018                         .maxauthsize = SHA512_DIGEST_SIZE,
2019                 },
2020                 .caam = {
2021                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2022                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2023                                            OP_ALG_AAI_HMAC_PRECOMP,
2024                 }
2025         },
2026         {
2027                 .aead = {
2028                         .base = {
2029                                 .cra_name = "echainiv(authenc(hmac(sha512),"
2030                                             "cbc(aes)))",
2031                                 .cra_driver_name = "echainiv-authenc-"
2032                                                    "hmac-sha512-cbc-aes-"
2033                                                    "caam-qi2",
2034                                 .cra_blocksize = AES_BLOCK_SIZE,
2035                         },
2036                         .setkey = aead_setkey,
2037                         .setauthsize = aead_setauthsize,
2038                         .encrypt = aead_encrypt,
2039                         .decrypt = aead_decrypt,
2040                         .ivsize = AES_BLOCK_SIZE,
2041                         .maxauthsize = SHA512_DIGEST_SIZE,
2042                 },
2043                 .caam = {
2044                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2045                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2046                                            OP_ALG_AAI_HMAC_PRECOMP,
2047                         .geniv = true,
2048                 }
2049         },
2050         {
2051                 .aead = {
2052                         .base = {
2053                                 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2054                                 .cra_driver_name = "authenc-hmac-md5-"
2055                                                    "cbc-des3_ede-caam-qi2",
2056                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2057                         },
2058                         .setkey = des3_aead_setkey,
2059                         .setauthsize = aead_setauthsize,
2060                         .encrypt = aead_encrypt,
2061                         .decrypt = aead_decrypt,
2062                         .ivsize = DES3_EDE_BLOCK_SIZE,
2063                         .maxauthsize = MD5_DIGEST_SIZE,
2064                 },
2065                 .caam = {
2066                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2067                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2068                                            OP_ALG_AAI_HMAC_PRECOMP,
2069                 }
2070         },
2071         {
2072                 .aead = {
2073                         .base = {
2074                                 .cra_name = "echainiv(authenc(hmac(md5),"
2075                                             "cbc(des3_ede)))",
2076                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
2077                                                    "cbc-des3_ede-caam-qi2",
2078                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2079                         },
2080                         .setkey = des3_aead_setkey,
2081                         .setauthsize = aead_setauthsize,
2082                         .encrypt = aead_encrypt,
2083                         .decrypt = aead_decrypt,
2084                         .ivsize = DES3_EDE_BLOCK_SIZE,
2085                         .maxauthsize = MD5_DIGEST_SIZE,
2086                 },
2087                 .caam = {
2088                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2089                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2090                                            OP_ALG_AAI_HMAC_PRECOMP,
2091                         .geniv = true,
2092                 }
2093         },
2094         {
2095                 .aead = {
2096                         .base = {
2097                                 .cra_name = "authenc(hmac(sha1),"
2098                                             "cbc(des3_ede))",
2099                                 .cra_driver_name = "authenc-hmac-sha1-"
2100                                                    "cbc-des3_ede-caam-qi2",
2101                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2102                         },
2103                         .setkey = des3_aead_setkey,
2104                         .setauthsize = aead_setauthsize,
2105                         .encrypt = aead_encrypt,
2106                         .decrypt = aead_decrypt,
2107                         .ivsize = DES3_EDE_BLOCK_SIZE,
2108                         .maxauthsize = SHA1_DIGEST_SIZE,
2109                 },
2110                 .caam = {
2111                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2112                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2113                                            OP_ALG_AAI_HMAC_PRECOMP,
2114                 },
2115         },
2116         {
2117                 .aead = {
2118                         .base = {
2119                                 .cra_name = "echainiv(authenc(hmac(sha1),"
2120                                             "cbc(des3_ede)))",
2121                                 .cra_driver_name = "echainiv-authenc-"
2122                                                    "hmac-sha1-"
2123                                                    "cbc-des3_ede-caam-qi2",
2124                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2125                         },
2126                         .setkey = des3_aead_setkey,
2127                         .setauthsize = aead_setauthsize,
2128                         .encrypt = aead_encrypt,
2129                         .decrypt = aead_decrypt,
2130                         .ivsize = DES3_EDE_BLOCK_SIZE,
2131                         .maxauthsize = SHA1_DIGEST_SIZE,
2132                 },
2133                 .caam = {
2134                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2135                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2136                                            OP_ALG_AAI_HMAC_PRECOMP,
2137                         .geniv = true,
2138                 }
2139         },
2140         {
2141                 .aead = {
2142                         .base = {
2143                                 .cra_name = "authenc(hmac(sha224),"
2144                                             "cbc(des3_ede))",
2145                                 .cra_driver_name = "authenc-hmac-sha224-"
2146                                                    "cbc-des3_ede-caam-qi2",
2147                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2148                         },
2149                         .setkey = des3_aead_setkey,
2150                         .setauthsize = aead_setauthsize,
2151                         .encrypt = aead_encrypt,
2152                         .decrypt = aead_decrypt,
2153                         .ivsize = DES3_EDE_BLOCK_SIZE,
2154                         .maxauthsize = SHA224_DIGEST_SIZE,
2155                 },
2156                 .caam = {
2157                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2158                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2159                                            OP_ALG_AAI_HMAC_PRECOMP,
2160                 },
2161         },
2162         {
2163                 .aead = {
2164                         .base = {
2165                                 .cra_name = "echainiv(authenc(hmac(sha224),"
2166                                             "cbc(des3_ede)))",
2167                                 .cra_driver_name = "echainiv-authenc-"
2168                                                    "hmac-sha224-"
2169                                                    "cbc-des3_ede-caam-qi2",
2170                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2171                         },
2172                         .setkey = des3_aead_setkey,
2173                         .setauthsize = aead_setauthsize,
2174                         .encrypt = aead_encrypt,
2175                         .decrypt = aead_decrypt,
2176                         .ivsize = DES3_EDE_BLOCK_SIZE,
2177                         .maxauthsize = SHA224_DIGEST_SIZE,
2178                 },
2179                 .caam = {
2180                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2181                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2182                                            OP_ALG_AAI_HMAC_PRECOMP,
2183                         .geniv = true,
2184                 }
2185         },
2186         {
2187                 .aead = {
2188                         .base = {
2189                                 .cra_name = "authenc(hmac(sha256),"
2190                                             "cbc(des3_ede))",
2191                                 .cra_driver_name = "authenc-hmac-sha256-"
2192                                                    "cbc-des3_ede-caam-qi2",
2193                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2194                         },
2195                         .setkey = des3_aead_setkey,
2196                         .setauthsize = aead_setauthsize,
2197                         .encrypt = aead_encrypt,
2198                         .decrypt = aead_decrypt,
2199                         .ivsize = DES3_EDE_BLOCK_SIZE,
2200                         .maxauthsize = SHA256_DIGEST_SIZE,
2201                 },
2202                 .caam = {
2203                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2204                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2205                                            OP_ALG_AAI_HMAC_PRECOMP,
2206                 },
2207         },
2208         {
2209                 .aead = {
2210                         .base = {
2211                                 .cra_name = "echainiv(authenc(hmac(sha256),"
2212                                             "cbc(des3_ede)))",
2213                                 .cra_driver_name = "echainiv-authenc-"
2214                                                    "hmac-sha256-"
2215                                                    "cbc-des3_ede-caam-qi2",
2216                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2217                         },
2218                         .setkey = des3_aead_setkey,
2219                         .setauthsize = aead_setauthsize,
2220                         .encrypt = aead_encrypt,
2221                         .decrypt = aead_decrypt,
2222                         .ivsize = DES3_EDE_BLOCK_SIZE,
2223                         .maxauthsize = SHA256_DIGEST_SIZE,
2224                 },
2225                 .caam = {
2226                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2227                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2228                                            OP_ALG_AAI_HMAC_PRECOMP,
2229                         .geniv = true,
2230                 }
2231         },
2232         {
2233                 .aead = {
2234                         .base = {
2235                                 .cra_name = "authenc(hmac(sha384),"
2236                                             "cbc(des3_ede))",
2237                                 .cra_driver_name = "authenc-hmac-sha384-"
2238                                                    "cbc-des3_ede-caam-qi2",
2239                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2240                         },
2241                         .setkey = des3_aead_setkey,
2242                         .setauthsize = aead_setauthsize,
2243                         .encrypt = aead_encrypt,
2244                         .decrypt = aead_decrypt,
2245                         .ivsize = DES3_EDE_BLOCK_SIZE,
2246                         .maxauthsize = SHA384_DIGEST_SIZE,
2247                 },
2248                 .caam = {
2249                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2250                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2251                                            OP_ALG_AAI_HMAC_PRECOMP,
2252                 },
2253         },
2254         {
2255                 .aead = {
2256                         .base = {
2257                                 .cra_name = "echainiv(authenc(hmac(sha384),"
2258                                             "cbc(des3_ede)))",
2259                                 .cra_driver_name = "echainiv-authenc-"
2260                                                    "hmac-sha384-"
2261                                                    "cbc-des3_ede-caam-qi2",
2262                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2263                         },
2264                         .setkey = des3_aead_setkey,
2265                         .setauthsize = aead_setauthsize,
2266                         .encrypt = aead_encrypt,
2267                         .decrypt = aead_decrypt,
2268                         .ivsize = DES3_EDE_BLOCK_SIZE,
2269                         .maxauthsize = SHA384_DIGEST_SIZE,
2270                 },
2271                 .caam = {
2272                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2273                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2274                                            OP_ALG_AAI_HMAC_PRECOMP,
2275                         .geniv = true,
2276                 }
2277         },
2278         {
2279                 .aead = {
2280                         .base = {
2281                                 .cra_name = "authenc(hmac(sha512),"
2282                                             "cbc(des3_ede))",
2283                                 .cra_driver_name = "authenc-hmac-sha512-"
2284                                                    "cbc-des3_ede-caam-qi2",
2285                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2286                         },
2287                         .setkey = des3_aead_setkey,
2288                         .setauthsize = aead_setauthsize,
2289                         .encrypt = aead_encrypt,
2290                         .decrypt = aead_decrypt,
2291                         .ivsize = DES3_EDE_BLOCK_SIZE,
2292                         .maxauthsize = SHA512_DIGEST_SIZE,
2293                 },
2294                 .caam = {
2295                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2296                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2297                                            OP_ALG_AAI_HMAC_PRECOMP,
2298                 },
2299         },
2300         {
2301                 .aead = {
2302                         .base = {
2303                                 .cra_name = "echainiv(authenc(hmac(sha512),"
2304                                             "cbc(des3_ede)))",
2305                                 .cra_driver_name = "echainiv-authenc-"
2306                                                    "hmac-sha512-"
2307                                                    "cbc-des3_ede-caam-qi2",
2308                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2309                         },
2310                         .setkey = des3_aead_setkey,
2311                         .setauthsize = aead_setauthsize,
2312                         .encrypt = aead_encrypt,
2313                         .decrypt = aead_decrypt,
2314                         .ivsize = DES3_EDE_BLOCK_SIZE,
2315                         .maxauthsize = SHA512_DIGEST_SIZE,
2316                 },
2317                 .caam = {
2318                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2319                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2320                                            OP_ALG_AAI_HMAC_PRECOMP,
2321                         .geniv = true,
2322                 }
2323         },
2324         {
2325                 .aead = {
2326                         .base = {
2327                                 .cra_name = "authenc(hmac(md5),cbc(des))",
2328                                 .cra_driver_name = "authenc-hmac-md5-"
2329                                                    "cbc-des-caam-qi2",
2330                                 .cra_blocksize = DES_BLOCK_SIZE,
2331                         },
2332                         .setkey = aead_setkey,
2333                         .setauthsize = aead_setauthsize,
2334                         .encrypt = aead_encrypt,
2335                         .decrypt = aead_decrypt,
2336                         .ivsize = DES_BLOCK_SIZE,
2337                         .maxauthsize = MD5_DIGEST_SIZE,
2338                 },
2339                 .caam = {
2340                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2341                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2342                                            OP_ALG_AAI_HMAC_PRECOMP,
2343                 },
2344         },
2345         {
2346                 .aead = {
2347                         .base = {
2348                                 .cra_name = "echainiv(authenc(hmac(md5),"
2349                                             "cbc(des)))",
2350                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
2351                                                    "cbc-des-caam-qi2",
2352                                 .cra_blocksize = DES_BLOCK_SIZE,
2353                         },
2354                         .setkey = aead_setkey,
2355                         .setauthsize = aead_setauthsize,
2356                         .encrypt = aead_encrypt,
2357                         .decrypt = aead_decrypt,
2358                         .ivsize = DES_BLOCK_SIZE,
2359                         .maxauthsize = MD5_DIGEST_SIZE,
2360                 },
2361                 .caam = {
2362                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2363                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2364                                            OP_ALG_AAI_HMAC_PRECOMP,
2365                         .geniv = true,
2366                 }
2367         },
2368         {
2369                 .aead = {
2370                         .base = {
2371                                 .cra_name = "authenc(hmac(sha1),cbc(des))",
2372                                 .cra_driver_name = "authenc-hmac-sha1-"
2373                                                    "cbc-des-caam-qi2",
2374                                 .cra_blocksize = DES_BLOCK_SIZE,
2375                         },
2376                         .setkey = aead_setkey,
2377                         .setauthsize = aead_setauthsize,
2378                         .encrypt = aead_encrypt,
2379                         .decrypt = aead_decrypt,
2380                         .ivsize = DES_BLOCK_SIZE,
2381                         .maxauthsize = SHA1_DIGEST_SIZE,
2382                 },
2383                 .caam = {
2384                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2385                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2386                                            OP_ALG_AAI_HMAC_PRECOMP,
2387                 },
2388         },
2389         {
2390                 .aead = {
2391                         .base = {
2392                                 .cra_name = "echainiv(authenc(hmac(sha1),"
2393                                             "cbc(des)))",
2394                                 .cra_driver_name = "echainiv-authenc-"
2395                                                    "hmac-sha1-cbc-des-caam-qi2",
2396                                 .cra_blocksize = DES_BLOCK_SIZE,
2397                         },
2398                         .setkey = aead_setkey,
2399                         .setauthsize = aead_setauthsize,
2400                         .encrypt = aead_encrypt,
2401                         .decrypt = aead_decrypt,
2402                         .ivsize = DES_BLOCK_SIZE,
2403                         .maxauthsize = SHA1_DIGEST_SIZE,
2404                 },
2405                 .caam = {
2406                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2407                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2408                                            OP_ALG_AAI_HMAC_PRECOMP,
2409                         .geniv = true,
2410                 }
2411         },
2412         {
2413                 .aead = {
2414                         .base = {
2415                                 .cra_name = "authenc(hmac(sha224),cbc(des))",
2416                                 .cra_driver_name = "authenc-hmac-sha224-"
2417                                                    "cbc-des-caam-qi2",
2418                                 .cra_blocksize = DES_BLOCK_SIZE,
2419                         },
2420                         .setkey = aead_setkey,
2421                         .setauthsize = aead_setauthsize,
2422                         .encrypt = aead_encrypt,
2423                         .decrypt = aead_decrypt,
2424                         .ivsize = DES_BLOCK_SIZE,
2425                         .maxauthsize = SHA224_DIGEST_SIZE,
2426                 },
2427                 .caam = {
2428                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2429                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2430                                            OP_ALG_AAI_HMAC_PRECOMP,
2431                 },
2432         },
2433         {
2434                 .aead = {
2435                         .base = {
2436                                 .cra_name = "echainiv(authenc(hmac(sha224),"
2437                                             "cbc(des)))",
2438                                 .cra_driver_name = "echainiv-authenc-"
2439                                                    "hmac-sha224-cbc-des-"
2440                                                    "caam-qi2",
2441                                 .cra_blocksize = DES_BLOCK_SIZE,
2442                         },
2443                         .setkey = aead_setkey,
2444                         .setauthsize = aead_setauthsize,
2445                         .encrypt = aead_encrypt,
2446                         .decrypt = aead_decrypt,
2447                         .ivsize = DES_BLOCK_SIZE,
2448                         .maxauthsize = SHA224_DIGEST_SIZE,
2449                 },
2450                 .caam = {
2451                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2452                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2453                                            OP_ALG_AAI_HMAC_PRECOMP,
2454                         .geniv = true,
2455                 }
2456         },
2457         {
2458                 .aead = {
2459                         .base = {
2460                                 .cra_name = "authenc(hmac(sha256),cbc(des))",
2461                                 .cra_driver_name = "authenc-hmac-sha256-"
2462                                                    "cbc-des-caam-qi2",
2463                                 .cra_blocksize = DES_BLOCK_SIZE,
2464                         },
2465                         .setkey = aead_setkey,
2466                         .setauthsize = aead_setauthsize,
2467                         .encrypt = aead_encrypt,
2468                         .decrypt = aead_decrypt,
2469                         .ivsize = DES_BLOCK_SIZE,
2470                         .maxauthsize = SHA256_DIGEST_SIZE,
2471                 },
2472                 .caam = {
2473                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2474                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2475                                            OP_ALG_AAI_HMAC_PRECOMP,
2476                 },
2477         },
2478         {
2479                 .aead = {
2480                         .base = {
2481                                 .cra_name = "echainiv(authenc(hmac(sha256),"
2482                                             "cbc(des)))",
2483                                 .cra_driver_name = "echainiv-authenc-"
2484                                                    "hmac-sha256-cbc-desi-"
2485                                                    "caam-qi2",
2486                                 .cra_blocksize = DES_BLOCK_SIZE,
2487                         },
2488                         .setkey = aead_setkey,
2489                         .setauthsize = aead_setauthsize,
2490                         .encrypt = aead_encrypt,
2491                         .decrypt = aead_decrypt,
2492                         .ivsize = DES_BLOCK_SIZE,
2493                         .maxauthsize = SHA256_DIGEST_SIZE,
2494                 },
2495                 .caam = {
2496                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2497                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2498                                            OP_ALG_AAI_HMAC_PRECOMP,
2499                         .geniv = true,
2500                 },
2501         },
2502         {
2503                 .aead = {
2504                         .base = {
2505                                 .cra_name = "authenc(hmac(sha384),cbc(des))",
2506                                 .cra_driver_name = "authenc-hmac-sha384-"
2507                                                    "cbc-des-caam-qi2",
2508                                 .cra_blocksize = DES_BLOCK_SIZE,
2509                         },
2510                         .setkey = aead_setkey,
2511                         .setauthsize = aead_setauthsize,
2512                         .encrypt = aead_encrypt,
2513                         .decrypt = aead_decrypt,
2514                         .ivsize = DES_BLOCK_SIZE,
2515                         .maxauthsize = SHA384_DIGEST_SIZE,
2516                 },
2517                 .caam = {
2518                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2519                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2520                                            OP_ALG_AAI_HMAC_PRECOMP,
2521                 },
2522         },
2523         {
2524                 .aead = {
2525                         .base = {
2526                                 .cra_name = "echainiv(authenc(hmac(sha384),"
2527                                             "cbc(des)))",
2528                                 .cra_driver_name = "echainiv-authenc-"
2529                                                    "hmac-sha384-cbc-des-"
2530                                                    "caam-qi2",
2531                                 .cra_blocksize = DES_BLOCK_SIZE,
2532                         },
2533                         .setkey = aead_setkey,
2534                         .setauthsize = aead_setauthsize,
2535                         .encrypt = aead_encrypt,
2536                         .decrypt = aead_decrypt,
2537                         .ivsize = DES_BLOCK_SIZE,
2538                         .maxauthsize = SHA384_DIGEST_SIZE,
2539                 },
2540                 .caam = {
2541                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2542                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2543                                            OP_ALG_AAI_HMAC_PRECOMP,
2544                         .geniv = true,
2545                 }
2546         },
2547         {
2548                 .aead = {
2549                         .base = {
2550                                 .cra_name = "authenc(hmac(sha512),cbc(des))",
2551                                 .cra_driver_name = "authenc-hmac-sha512-"
2552                                                    "cbc-des-caam-qi2",
2553                                 .cra_blocksize = DES_BLOCK_SIZE,
2554                         },
2555                         .setkey = aead_setkey,
2556                         .setauthsize = aead_setauthsize,
2557                         .encrypt = aead_encrypt,
2558                         .decrypt = aead_decrypt,
2559                         .ivsize = DES_BLOCK_SIZE,
2560                         .maxauthsize = SHA512_DIGEST_SIZE,
2561                 },
2562                 .caam = {
2563                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2564                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2565                                            OP_ALG_AAI_HMAC_PRECOMP,
2566                 }
2567         },
2568         {
2569                 .aead = {
2570                         .base = {
2571                                 .cra_name = "echainiv(authenc(hmac(sha512),"
2572                                             "cbc(des)))",
2573                                 .cra_driver_name = "echainiv-authenc-"
2574                                                    "hmac-sha512-cbc-des-"
2575                                                    "caam-qi2",
2576                                 .cra_blocksize = DES_BLOCK_SIZE,
2577                         },
2578                         .setkey = aead_setkey,
2579                         .setauthsize = aead_setauthsize,
2580                         .encrypt = aead_encrypt,
2581                         .decrypt = aead_decrypt,
2582                         .ivsize = DES_BLOCK_SIZE,
2583                         .maxauthsize = SHA512_DIGEST_SIZE,
2584                 },
2585                 .caam = {
2586                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2587                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2588                                            OP_ALG_AAI_HMAC_PRECOMP,
2589                         .geniv = true,
2590                 }
2591         },
2592         {
2593                 .aead = {
2594                         .base = {
2595                                 .cra_name = "authenc(hmac(md5),"
2596                                             "rfc3686(ctr(aes)))",
2597                                 .cra_driver_name = "authenc-hmac-md5-"
2598                                                    "rfc3686-ctr-aes-caam-qi2",
2599                                 .cra_blocksize = 1,
2600                         },
2601                         .setkey = aead_setkey,
2602                         .setauthsize = aead_setauthsize,
2603                         .encrypt = aead_encrypt,
2604                         .decrypt = aead_decrypt,
2605                         .ivsize = CTR_RFC3686_IV_SIZE,
2606                         .maxauthsize = MD5_DIGEST_SIZE,
2607                 },
2608                 .caam = {
2609                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2610                                            OP_ALG_AAI_CTR_MOD128,
2611                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2612                                            OP_ALG_AAI_HMAC_PRECOMP,
2613                         .rfc3686 = true,
2614                 },
2615         },
2616         {
2617                 .aead = {
2618                         .base = {
2619                                 .cra_name = "seqiv(authenc("
2620                                             "hmac(md5),rfc3686(ctr(aes))))",
2621                                 .cra_driver_name = "seqiv-authenc-hmac-md5-"
2622                                                    "rfc3686-ctr-aes-caam-qi2",
2623                                 .cra_blocksize = 1,
2624                         },
2625                         .setkey = aead_setkey,
2626                         .setauthsize = aead_setauthsize,
2627                         .encrypt = aead_encrypt,
2628                         .decrypt = aead_decrypt,
2629                         .ivsize = CTR_RFC3686_IV_SIZE,
2630                         .maxauthsize = MD5_DIGEST_SIZE,
2631                 },
2632                 .caam = {
2633                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2634                                            OP_ALG_AAI_CTR_MOD128,
2635                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2636                                            OP_ALG_AAI_HMAC_PRECOMP,
2637                         .rfc3686 = true,
2638                         .geniv = true,
2639                 },
2640         },
2641         {
2642                 .aead = {
2643                         .base = {
2644                                 .cra_name = "authenc(hmac(sha1),"
2645                                             "rfc3686(ctr(aes)))",
2646                                 .cra_driver_name = "authenc-hmac-sha1-"
2647                                                    "rfc3686-ctr-aes-caam-qi2",
2648                                 .cra_blocksize = 1,
2649                         },
2650                         .setkey = aead_setkey,
2651                         .setauthsize = aead_setauthsize,
2652                         .encrypt = aead_encrypt,
2653                         .decrypt = aead_decrypt,
2654                         .ivsize = CTR_RFC3686_IV_SIZE,
2655                         .maxauthsize = SHA1_DIGEST_SIZE,
2656                 },
2657                 .caam = {
2658                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2659                                            OP_ALG_AAI_CTR_MOD128,
2660                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2661                                            OP_ALG_AAI_HMAC_PRECOMP,
2662                         .rfc3686 = true,
2663                 },
2664         },
2665         {
2666                 .aead = {
2667                         .base = {
2668                                 .cra_name = "seqiv(authenc("
2669                                             "hmac(sha1),rfc3686(ctr(aes))))",
2670                                 .cra_driver_name = "seqiv-authenc-hmac-sha1-"
2671                                                    "rfc3686-ctr-aes-caam-qi2",
2672                                 .cra_blocksize = 1,
2673                         },
2674                         .setkey = aead_setkey,
2675                         .setauthsize = aead_setauthsize,
2676                         .encrypt = aead_encrypt,
2677                         .decrypt = aead_decrypt,
2678                         .ivsize = CTR_RFC3686_IV_SIZE,
2679                         .maxauthsize = SHA1_DIGEST_SIZE,
2680                 },
2681                 .caam = {
2682                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2683                                            OP_ALG_AAI_CTR_MOD128,
2684                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2685                                            OP_ALG_AAI_HMAC_PRECOMP,
2686                         .rfc3686 = true,
2687                         .geniv = true,
2688                 },
2689         },
2690         {
2691                 .aead = {
2692                         .base = {
2693                                 .cra_name = "authenc(hmac(sha224),"
2694                                             "rfc3686(ctr(aes)))",
2695                                 .cra_driver_name = "authenc-hmac-sha224-"
2696                                                    "rfc3686-ctr-aes-caam-qi2",
2697                                 .cra_blocksize = 1,
2698                         },
2699                         .setkey = aead_setkey,
2700                         .setauthsize = aead_setauthsize,
2701                         .encrypt = aead_encrypt,
2702                         .decrypt = aead_decrypt,
2703                         .ivsize = CTR_RFC3686_IV_SIZE,
2704                         .maxauthsize = SHA224_DIGEST_SIZE,
2705                 },
2706                 .caam = {
2707                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2708                                            OP_ALG_AAI_CTR_MOD128,
2709                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2710                                            OP_ALG_AAI_HMAC_PRECOMP,
2711                         .rfc3686 = true,
2712                 },
2713         },
2714         {
2715                 .aead = {
2716                         .base = {
2717                                 .cra_name = "seqiv(authenc("
2718                                             "hmac(sha224),rfc3686(ctr(aes))))",
2719                                 .cra_driver_name = "seqiv-authenc-hmac-sha224-"
2720                                                    "rfc3686-ctr-aes-caam-qi2",
2721                                 .cra_blocksize = 1,
2722                         },
2723                         .setkey = aead_setkey,
2724                         .setauthsize = aead_setauthsize,
2725                         .encrypt = aead_encrypt,
2726                         .decrypt = aead_decrypt,
2727                         .ivsize = CTR_RFC3686_IV_SIZE,
2728                         .maxauthsize = SHA224_DIGEST_SIZE,
2729                 },
2730                 .caam = {
2731                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2732                                            OP_ALG_AAI_CTR_MOD128,
2733                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2734                                            OP_ALG_AAI_HMAC_PRECOMP,
2735                         .rfc3686 = true,
2736                         .geniv = true,
2737                 },
2738         },
2739         {
2740                 .aead = {
2741                         .base = {
2742                                 .cra_name = "authenc(hmac(sha256),"
2743                                             "rfc3686(ctr(aes)))",
2744                                 .cra_driver_name = "authenc-hmac-sha256-"
2745                                                    "rfc3686-ctr-aes-caam-qi2",
2746                                 .cra_blocksize = 1,
2747                         },
2748                         .setkey = aead_setkey,
2749                         .setauthsize = aead_setauthsize,
2750                         .encrypt = aead_encrypt,
2751                         .decrypt = aead_decrypt,
2752                         .ivsize = CTR_RFC3686_IV_SIZE,
2753                         .maxauthsize = SHA256_DIGEST_SIZE,
2754                 },
2755                 .caam = {
2756                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2757                                            OP_ALG_AAI_CTR_MOD128,
2758                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2759                                            OP_ALG_AAI_HMAC_PRECOMP,
2760                         .rfc3686 = true,
2761                 },
2762         },
2763         {
2764                 .aead = {
2765                         .base = {
2766                                 .cra_name = "seqiv(authenc(hmac(sha256),"
2767                                             "rfc3686(ctr(aes))))",
2768                                 .cra_driver_name = "seqiv-authenc-hmac-sha256-"
2769                                                    "rfc3686-ctr-aes-caam-qi2",
2770                                 .cra_blocksize = 1,
2771                         },
2772                         .setkey = aead_setkey,
2773                         .setauthsize = aead_setauthsize,
2774                         .encrypt = aead_encrypt,
2775                         .decrypt = aead_decrypt,
2776                         .ivsize = CTR_RFC3686_IV_SIZE,
2777                         .maxauthsize = SHA256_DIGEST_SIZE,
2778                 },
2779                 .caam = {
2780                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2781                                            OP_ALG_AAI_CTR_MOD128,
2782                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2783                                            OP_ALG_AAI_HMAC_PRECOMP,
2784                         .rfc3686 = true,
2785                         .geniv = true,
2786                 },
2787         },
2788         {
2789                 .aead = {
2790                         .base = {
2791                                 .cra_name = "authenc(hmac(sha384),"
2792                                             "rfc3686(ctr(aes)))",
2793                                 .cra_driver_name = "authenc-hmac-sha384-"
2794                                                    "rfc3686-ctr-aes-caam-qi2",
2795                                 .cra_blocksize = 1,
2796                         },
2797                         .setkey = aead_setkey,
2798                         .setauthsize = aead_setauthsize,
2799                         .encrypt = aead_encrypt,
2800                         .decrypt = aead_decrypt,
2801                         .ivsize = CTR_RFC3686_IV_SIZE,
2802                         .maxauthsize = SHA384_DIGEST_SIZE,
2803                 },
2804                 .caam = {
2805                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2806                                            OP_ALG_AAI_CTR_MOD128,
2807                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2808                                            OP_ALG_AAI_HMAC_PRECOMP,
2809                         .rfc3686 = true,
2810                 },
2811         },
2812         {
2813                 .aead = {
2814                         .base = {
2815                                 .cra_name = "seqiv(authenc(hmac(sha384),"
2816                                             "rfc3686(ctr(aes))))",
2817                                 .cra_driver_name = "seqiv-authenc-hmac-sha384-"
2818                                                    "rfc3686-ctr-aes-caam-qi2",
2819                                 .cra_blocksize = 1,
2820                         },
2821                         .setkey = aead_setkey,
2822                         .setauthsize = aead_setauthsize,
2823                         .encrypt = aead_encrypt,
2824                         .decrypt = aead_decrypt,
2825                         .ivsize = CTR_RFC3686_IV_SIZE,
2826                         .maxauthsize = SHA384_DIGEST_SIZE,
2827                 },
2828                 .caam = {
2829                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2830                                            OP_ALG_AAI_CTR_MOD128,
2831                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2832                                            OP_ALG_AAI_HMAC_PRECOMP,
2833                         .rfc3686 = true,
2834                         .geniv = true,
2835                 },
2836         },
2837         {
2838                 .aead = {
2839                         .base = {
2840                                 .cra_name = "rfc7539(chacha20,poly1305)",
2841                                 .cra_driver_name = "rfc7539-chacha20-poly1305-"
2842                                                    "caam-qi2",
2843                                 .cra_blocksize = 1,
2844                         },
2845                         .setkey = chachapoly_setkey,
2846                         .setauthsize = chachapoly_setauthsize,
2847                         .encrypt = aead_encrypt,
2848                         .decrypt = aead_decrypt,
2849                         .ivsize = CHACHAPOLY_IV_SIZE,
2850                         .maxauthsize = POLY1305_DIGEST_SIZE,
2851                 },
2852                 .caam = {
2853                         .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
2854                                            OP_ALG_AAI_AEAD,
2855                         .class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
2856                                            OP_ALG_AAI_AEAD,
2857                         .nodkp = true,
2858                 },
2859         },
2860         {
2861                 .aead = {
2862                         .base = {
2863                                 .cra_name = "rfc7539esp(chacha20,poly1305)",
2864                                 .cra_driver_name = "rfc7539esp-chacha20-"
2865                                                    "poly1305-caam-qi2",
2866                                 .cra_blocksize = 1,
2867                         },
2868                         .setkey = chachapoly_setkey,
2869                         .setauthsize = chachapoly_setauthsize,
2870                         .encrypt = aead_encrypt,
2871                         .decrypt = aead_decrypt,
2872                         .ivsize = 8,
2873                         .maxauthsize = POLY1305_DIGEST_SIZE,
2874                 },
2875                 .caam = {
2876                         .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
2877                                            OP_ALG_AAI_AEAD,
2878                         .class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
2879                                            OP_ALG_AAI_AEAD,
2880                         .nodkp = true,
2881                 },
2882         },
2883         {
2884                 .aead = {
2885                         .base = {
2886                                 .cra_name = "authenc(hmac(sha512),"
2887                                             "rfc3686(ctr(aes)))",
2888                                 .cra_driver_name = "authenc-hmac-sha512-"
2889                                                    "rfc3686-ctr-aes-caam-qi2",
2890                                 .cra_blocksize = 1,
2891                         },
2892                         .setkey = aead_setkey,
2893                         .setauthsize = aead_setauthsize,
2894                         .encrypt = aead_encrypt,
2895                         .decrypt = aead_decrypt,
2896                         .ivsize = CTR_RFC3686_IV_SIZE,
2897                         .maxauthsize = SHA512_DIGEST_SIZE,
2898                 },
2899                 .caam = {
2900                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2901                                            OP_ALG_AAI_CTR_MOD128,
2902                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2903                                            OP_ALG_AAI_HMAC_PRECOMP,
2904                         .rfc3686 = true,
2905                 },
2906         },
2907         {
2908                 .aead = {
2909                         .base = {
2910                                 .cra_name = "seqiv(authenc(hmac(sha512),"
2911                                             "rfc3686(ctr(aes))))",
2912                                 .cra_driver_name = "seqiv-authenc-hmac-sha512-"
2913                                                    "rfc3686-ctr-aes-caam-qi2",
2914                                 .cra_blocksize = 1,
2915                         },
2916                         .setkey = aead_setkey,
2917                         .setauthsize = aead_setauthsize,
2918                         .encrypt = aead_encrypt,
2919                         .decrypt = aead_decrypt,
2920                         .ivsize = CTR_RFC3686_IV_SIZE,
2921                         .maxauthsize = SHA512_DIGEST_SIZE,
2922                 },
2923                 .caam = {
2924                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2925                                            OP_ALG_AAI_CTR_MOD128,
2926                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2927                                            OP_ALG_AAI_HMAC_PRECOMP,
2928                         .rfc3686 = true,
2929                         .geniv = true,
2930                 },
2931         },
2932 };
2933
2934 static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg)
2935 {
2936         struct skcipher_alg *alg = &t_alg->skcipher;
2937
2938         alg->base.cra_module = THIS_MODULE;
2939         alg->base.cra_priority = CAAM_CRA_PRIORITY;
2940         alg->base.cra_ctxsize = sizeof(struct caam_ctx);
2941         alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
2942
2943         alg->init = caam_cra_init_skcipher;
2944         alg->exit = caam_cra_exit;
2945 }
2946
2947 static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
2948 {
2949         struct aead_alg *alg = &t_alg->aead;
2950
2951         alg->base.cra_module = THIS_MODULE;
2952         alg->base.cra_priority = CAAM_CRA_PRIORITY;
2953         alg->base.cra_ctxsize = sizeof(struct caam_ctx);
2954         alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
2955
2956         alg->init = caam_cra_init_aead;
2957         alg->exit = caam_cra_exit_aead;
2958 }
2959
2960 /* max hash key is max split key size */
2961 #define CAAM_MAX_HASH_KEY_SIZE          (SHA512_DIGEST_SIZE * 2)
2962
2963 #define CAAM_MAX_HASH_BLOCK_SIZE        SHA512_BLOCK_SIZE
2964
2965 /* caam context sizes for hashes: running digest + 8 */
2966 #define HASH_MSG_LEN                    8
2967 #define MAX_CTX_LEN                     (HASH_MSG_LEN + SHA512_DIGEST_SIZE)
2968
2969 enum hash_optype {
2970         UPDATE = 0,
2971         UPDATE_FIRST,
2972         FINALIZE,
2973         DIGEST,
2974         HASH_NUM_OP
2975 };
2976
2977 /**
2978  * caam_hash_ctx - ahash per-session context
2979  * @flc: Flow Contexts array
2980  * @key: authentication key
2981  * @flc_dma: I/O virtual addresses of the Flow Contexts
2982  * @dev: dpseci device
2983  * @ctx_len: size of Context Register
2984  * @adata: hashing algorithm details
2985  */
2986 struct caam_hash_ctx {
2987         struct caam_flc flc[HASH_NUM_OP];
2988         u8 key[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
2989         dma_addr_t flc_dma[HASH_NUM_OP];
2990         struct device *dev;
2991         int ctx_len;
2992         struct alginfo adata;
2993 };
2994
2995 /* ahash state */
2996 struct caam_hash_state {
2997         struct caam_request caam_req;
2998         dma_addr_t buf_dma;
2999         dma_addr_t ctx_dma;
3000         int ctx_dma_len;
3001         u8 buf_0[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
3002         int buflen_0;
3003         u8 buf_1[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
3004         int buflen_1;
3005         u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned;
3006         int (*update)(struct ahash_request *req);
3007         int (*final)(struct ahash_request *req);
3008         int (*finup)(struct ahash_request *req);
3009         int current_buf;
3010 };
3011
3012 struct caam_export_state {
3013         u8 buf[CAAM_MAX_HASH_BLOCK_SIZE];
3014         u8 caam_ctx[MAX_CTX_LEN];
3015         int buflen;
3016         int (*update)(struct ahash_request *req);
3017         int (*final)(struct ahash_request *req);
3018         int (*finup)(struct ahash_request *req);
3019 };
3020
3021 static inline void switch_buf(struct caam_hash_state *state)
3022 {
3023         state->current_buf ^= 1;
3024 }
3025
3026 static inline u8 *current_buf(struct caam_hash_state *state)
3027 {
3028         return state->current_buf ? state->buf_1 : state->buf_0;
3029 }
3030
3031 static inline u8 *alt_buf(struct caam_hash_state *state)
3032 {
3033         return state->current_buf ? state->buf_0 : state->buf_1;
3034 }
3035
3036 static inline int *current_buflen(struct caam_hash_state *state)
3037 {
3038         return state->current_buf ? &state->buflen_1 : &state->buflen_0;
3039 }
3040
3041 static inline int *alt_buflen(struct caam_hash_state *state)
3042 {
3043         return state->current_buf ? &state->buflen_0 : &state->buflen_1;
3044 }
3045
3046 /* Map current buffer in state (if length > 0) and put it in link table */
3047 static inline int buf_map_to_qm_sg(struct device *dev,
3048                                    struct dpaa2_sg_entry *qm_sg,
3049                                    struct caam_hash_state *state)
3050 {
3051         int buflen = *current_buflen(state);
3052
3053         if (!buflen)
3054                 return 0;
3055
3056         state->buf_dma = dma_map_single(dev, current_buf(state), buflen,
3057                                         DMA_TO_DEVICE);
3058         if (dma_mapping_error(dev, state->buf_dma)) {
3059                 dev_err(dev, "unable to map buf\n");
3060                 state->buf_dma = 0;
3061                 return -ENOMEM;
3062         }
3063
3064         dma_to_qm_sg_one(qm_sg, state->buf_dma, buflen, 0);
3065
3066         return 0;
3067 }
3068
3069 /* Map state->caam_ctx, and add it to link table */
3070 static inline int ctx_map_to_qm_sg(struct device *dev,
3071                                    struct caam_hash_state *state, int ctx_len,
3072                                    struct dpaa2_sg_entry *qm_sg, u32 flag)
3073 {
3074         state->ctx_dma_len = ctx_len;
3075         state->ctx_dma = dma_map_single(dev, state->caam_ctx, ctx_len, flag);
3076         if (dma_mapping_error(dev, state->ctx_dma)) {
3077                 dev_err(dev, "unable to map ctx\n");
3078                 state->ctx_dma = 0;
3079                 return -ENOMEM;
3080         }
3081
3082         dma_to_qm_sg_one(qm_sg, state->ctx_dma, ctx_len, 0);
3083
3084         return 0;
3085 }
3086
3087 static int ahash_set_sh_desc(struct crypto_ahash *ahash)
3088 {
3089         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3090         int digestsize = crypto_ahash_digestsize(ahash);
3091         struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev);
3092         struct caam_flc *flc;
3093         u32 *desc;
3094
3095         /* ahash_update shared descriptor */
3096         flc = &ctx->flc[UPDATE];
3097         desc = flc->sh_desc;
3098         cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_UPDATE, ctx->ctx_len,
3099                           ctx->ctx_len, true, priv->sec_attr.era);
3100         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3101         dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE],
3102                                    desc_bytes(desc), DMA_BIDIRECTIONAL);
3103         print_hex_dump_debug("ahash update shdesc@" __stringify(__LINE__)": ",
3104                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3105                              1);
3106
3107         /* ahash_update_first shared descriptor */
3108         flc = &ctx->flc[UPDATE_FIRST];
3109         desc = flc->sh_desc;
3110         cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len,
3111                           ctx->ctx_len, false, priv->sec_attr.era);
3112         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3113         dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE_FIRST],
3114                                    desc_bytes(desc), DMA_BIDIRECTIONAL);
3115         print_hex_dump_debug("ahash update first shdesc@" __stringify(__LINE__)": ",
3116                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3117                              1);
3118
3119         /* ahash_final shared descriptor */
3120         flc = &ctx->flc[FINALIZE];
3121         desc = flc->sh_desc;
3122         cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, digestsize,
3123                           ctx->ctx_len, true, priv->sec_attr.era);
3124         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3125         dma_sync_single_for_device(ctx->dev, ctx->flc_dma[FINALIZE],
3126                                    desc_bytes(desc), DMA_BIDIRECTIONAL);
3127         print_hex_dump_debug("ahash final shdesc@" __stringify(__LINE__)": ",
3128                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3129                              1);
3130
3131         /* ahash_digest shared descriptor */
3132         flc = &ctx->flc[DIGEST];
3133         desc = flc->sh_desc;
3134         cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, digestsize,
3135                           ctx->ctx_len, false, priv->sec_attr.era);
3136         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3137         dma_sync_single_for_device(ctx->dev, ctx->flc_dma[DIGEST],
3138                                    desc_bytes(desc), DMA_BIDIRECTIONAL);
3139         print_hex_dump_debug("ahash digest shdesc@" __stringify(__LINE__)": ",
3140                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3141                              1);
3142
3143         return 0;
3144 }
3145
3146 struct split_key_sh_result {
3147         struct completion completion;
3148         int err;
3149         struct device *dev;
3150 };
3151
3152 static void split_key_sh_done(void *cbk_ctx, u32 err)
3153 {
3154         struct split_key_sh_result *res = cbk_ctx;
3155
3156         dev_dbg(res->dev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
3157
3158         res->err = err ? caam_qi2_strstatus(res->dev, err) : 0;
3159         complete(&res->completion);
3160 }
3161
3162 /* Digest hash size if it is too large */
3163 static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key,
3164                            u32 digestsize)
3165 {
3166         struct caam_request *req_ctx;
3167         u32 *desc;
3168         struct split_key_sh_result result;
3169         dma_addr_t key_dma;
3170         struct caam_flc *flc;
3171         dma_addr_t flc_dma;
3172         int ret = -ENOMEM;
3173         struct dpaa2_fl_entry *in_fle, *out_fle;
3174
3175         req_ctx = kzalloc(sizeof(*req_ctx), GFP_KERNEL | GFP_DMA);
3176         if (!req_ctx)
3177                 return -ENOMEM;
3178
3179         in_fle = &req_ctx->fd_flt[1];
3180         out_fle = &req_ctx->fd_flt[0];
3181
3182         flc = kzalloc(sizeof(*flc), GFP_KERNEL | GFP_DMA);
3183         if (!flc)
3184                 goto err_flc;
3185
3186         key_dma = dma_map_single(ctx->dev, key, *keylen, DMA_BIDIRECTIONAL);
3187         if (dma_mapping_error(ctx->dev, key_dma)) {
3188                 dev_err(ctx->dev, "unable to map key memory\n");
3189                 goto err_key_dma;
3190         }
3191
3192         desc = flc->sh_desc;
3193
3194         init_sh_desc(desc, 0);
3195
3196         /* descriptor to perform unkeyed hash on key_in */
3197         append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT |
3198                          OP_ALG_AS_INITFINAL);
3199         append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 |
3200                              FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG);
3201         append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
3202                          LDST_SRCDST_BYTE_CONTEXT);
3203
3204         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3205         flc_dma = dma_map_single(ctx->dev, flc, sizeof(flc->flc) +
3206                                  desc_bytes(desc), DMA_TO_DEVICE);
3207         if (dma_mapping_error(ctx->dev, flc_dma)) {
3208                 dev_err(ctx->dev, "unable to map shared descriptor\n");
3209                 goto err_flc_dma;
3210         }
3211
3212         dpaa2_fl_set_final(in_fle, true);
3213         dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
3214         dpaa2_fl_set_addr(in_fle, key_dma);
3215         dpaa2_fl_set_len(in_fle, *keylen);
3216         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3217         dpaa2_fl_set_addr(out_fle, key_dma);
3218         dpaa2_fl_set_len(out_fle, digestsize);
3219
3220         print_hex_dump_debug("key_in@" __stringify(__LINE__)": ",
3221                              DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1);
3222         print_hex_dump_debug("shdesc@" __stringify(__LINE__)": ",
3223                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3224                              1);
3225
3226         result.err = 0;
3227         init_completion(&result.completion);
3228         result.dev = ctx->dev;
3229
3230         req_ctx->flc = flc;
3231         req_ctx->flc_dma = flc_dma;
3232         req_ctx->cbk = split_key_sh_done;
3233         req_ctx->ctx = &result;
3234
3235         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3236         if (ret == -EINPROGRESS) {
3237                 /* in progress */
3238                 wait_for_completion(&result.completion);
3239                 ret = result.err;
3240                 print_hex_dump_debug("digested key@" __stringify(__LINE__)": ",
3241                                      DUMP_PREFIX_ADDRESS, 16, 4, key,
3242                                      digestsize, 1);
3243         }
3244
3245         dma_unmap_single(ctx->dev, flc_dma, sizeof(flc->flc) + desc_bytes(desc),
3246                          DMA_TO_DEVICE);
3247 err_flc_dma:
3248         dma_unmap_single(ctx->dev, key_dma, *keylen, DMA_BIDIRECTIONAL);
3249 err_key_dma:
3250         kfree(flc);
3251 err_flc:
3252         kfree(req_ctx);
3253
3254         *keylen = digestsize;
3255
3256         return ret;
3257 }
3258
3259 static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key,
3260                         unsigned int keylen)
3261 {
3262         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3263         unsigned int blocksize = crypto_tfm_alg_blocksize(&ahash->base);
3264         unsigned int digestsize = crypto_ahash_digestsize(ahash);
3265         int ret;
3266         u8 *hashed_key = NULL;
3267
3268         dev_dbg(ctx->dev, "keylen %d blocksize %d\n", keylen, blocksize);
3269
3270         if (keylen > blocksize) {
3271                 hashed_key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);
3272                 if (!hashed_key)
3273                         return -ENOMEM;
3274                 ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize);
3275                 if (ret)
3276                         goto bad_free_key;
3277                 key = hashed_key;
3278         }
3279
3280         ctx->adata.keylen = keylen;
3281         ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
3282                                               OP_ALG_ALGSEL_MASK);
3283         if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE)
3284                 goto bad_free_key;
3285
3286         ctx->adata.key_virt = key;
3287         ctx->adata.key_inline = true;
3288
3289         /*
3290          * In case |user key| > |derived key|, using DKP<imm,imm> would result
3291          * in invalid opcodes (last bytes of user key) in the resulting
3292          * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key
3293          * addresses are needed.
3294          */
3295         if (keylen > ctx->adata.keylen_pad) {
3296                 memcpy(ctx->key, key, keylen);
3297                 dma_sync_single_for_device(ctx->dev, ctx->adata.key_dma,
3298                                            ctx->adata.keylen_pad,
3299                                            DMA_TO_DEVICE);
3300         }
3301
3302         ret = ahash_set_sh_desc(ahash);
3303         kfree(hashed_key);
3304         return ret;
3305 bad_free_key:
3306         kfree(hashed_key);
3307         crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN);
3308         return -EINVAL;
3309 }
3310
3311 static inline void ahash_unmap(struct device *dev, struct ahash_edesc *edesc,
3312                                struct ahash_request *req)
3313 {
3314         struct caam_hash_state *state = ahash_request_ctx(req);
3315
3316         if (edesc->src_nents)
3317                 dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE);
3318
3319         if (edesc->qm_sg_bytes)
3320                 dma_unmap_single(dev, edesc->qm_sg_dma, edesc->qm_sg_bytes,
3321                                  DMA_TO_DEVICE);
3322
3323         if (state->buf_dma) {
3324                 dma_unmap_single(dev, state->buf_dma, *current_buflen(state),
3325                                  DMA_TO_DEVICE);
3326                 state->buf_dma = 0;
3327         }
3328 }
3329
3330 static inline void ahash_unmap_ctx(struct device *dev,
3331                                    struct ahash_edesc *edesc,
3332                                    struct ahash_request *req, u32 flag)
3333 {
3334         struct caam_hash_state *state = ahash_request_ctx(req);
3335
3336         if (state->ctx_dma) {
3337                 dma_unmap_single(dev, state->ctx_dma, state->ctx_dma_len, flag);
3338                 state->ctx_dma = 0;
3339         }
3340         ahash_unmap(dev, edesc, req);
3341 }
3342
3343 static void ahash_done(void *cbk_ctx, u32 status)
3344 {
3345         struct crypto_async_request *areq = cbk_ctx;
3346         struct ahash_request *req = ahash_request_cast(areq);
3347         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3348         struct caam_hash_state *state = ahash_request_ctx(req);
3349         struct ahash_edesc *edesc = state->caam_req.edesc;
3350         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3351         int digestsize = crypto_ahash_digestsize(ahash);
3352         int ecode = 0;
3353
3354         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3355
3356         if (unlikely(status))
3357                 ecode = caam_qi2_strstatus(ctx->dev, status);
3358
3359         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3360         memcpy(req->result, state->caam_ctx, digestsize);
3361         qi_cache_free(edesc);
3362
3363         print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3364                              DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3365                              ctx->ctx_len, 1);
3366
3367         req->base.complete(&req->base, ecode);
3368 }
3369
3370 static void ahash_done_bi(void *cbk_ctx, u32 status)
3371 {
3372         struct crypto_async_request *areq = cbk_ctx;
3373         struct ahash_request *req = ahash_request_cast(areq);
3374         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3375         struct caam_hash_state *state = ahash_request_ctx(req);
3376         struct ahash_edesc *edesc = state->caam_req.edesc;
3377         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3378         int ecode = 0;
3379
3380         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3381
3382         if (unlikely(status))
3383                 ecode = caam_qi2_strstatus(ctx->dev, status);
3384
3385         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3386         switch_buf(state);
3387         qi_cache_free(edesc);
3388
3389         print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3390                              DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3391                              ctx->ctx_len, 1);
3392         if (req->result)
3393                 print_hex_dump_debug("result@" __stringify(__LINE__)": ",
3394                                      DUMP_PREFIX_ADDRESS, 16, 4, req->result,
3395                                      crypto_ahash_digestsize(ahash), 1);
3396
3397         req->base.complete(&req->base, ecode);
3398 }
3399
3400 static void ahash_done_ctx_src(void *cbk_ctx, u32 status)
3401 {
3402         struct crypto_async_request *areq = cbk_ctx;
3403         struct ahash_request *req = ahash_request_cast(areq);
3404         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3405         struct caam_hash_state *state = ahash_request_ctx(req);
3406         struct ahash_edesc *edesc = state->caam_req.edesc;
3407         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3408         int digestsize = crypto_ahash_digestsize(ahash);
3409         int ecode = 0;
3410
3411         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3412
3413         if (unlikely(status))
3414                 ecode = caam_qi2_strstatus(ctx->dev, status);
3415
3416         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3417         memcpy(req->result, state->caam_ctx, digestsize);
3418         qi_cache_free(edesc);
3419
3420         print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3421                              DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3422                              ctx->ctx_len, 1);
3423
3424         req->base.complete(&req->base, ecode);
3425 }
3426
3427 static void ahash_done_ctx_dst(void *cbk_ctx, u32 status)
3428 {
3429         struct crypto_async_request *areq = cbk_ctx;
3430         struct ahash_request *req = ahash_request_cast(areq);
3431         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3432         struct caam_hash_state *state = ahash_request_ctx(req);
3433         struct ahash_edesc *edesc = state->caam_req.edesc;
3434         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3435         int ecode = 0;
3436
3437         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3438
3439         if (unlikely(status))
3440                 ecode = caam_qi2_strstatus(ctx->dev, status);
3441
3442         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3443         switch_buf(state);
3444         qi_cache_free(edesc);
3445
3446         print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3447                              DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3448                              ctx->ctx_len, 1);
3449         if (req->result)
3450                 print_hex_dump_debug("result@" __stringify(__LINE__)": ",
3451                                      DUMP_PREFIX_ADDRESS, 16, 4, req->result,
3452                                      crypto_ahash_digestsize(ahash), 1);
3453
3454         req->base.complete(&req->base, ecode);
3455 }
3456
3457 static int ahash_update_ctx(struct ahash_request *req)
3458 {
3459         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3460         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3461         struct caam_hash_state *state = ahash_request_ctx(req);
3462         struct caam_request *req_ctx = &state->caam_req;
3463         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3464         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3465         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3466                       GFP_KERNEL : GFP_ATOMIC;
3467         u8 *buf = current_buf(state);
3468         int *buflen = current_buflen(state);
3469         u8 *next_buf = alt_buf(state);
3470         int *next_buflen = alt_buflen(state), last_buflen;
3471         int in_len = *buflen + req->nbytes, to_hash;
3472         int src_nents, mapped_nents, qm_sg_bytes, qm_sg_src_index;
3473         struct ahash_edesc *edesc;
3474         int ret = 0;
3475
3476         last_buflen = *next_buflen;
3477         *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
3478         to_hash = in_len - *next_buflen;
3479
3480         if (to_hash) {
3481                 struct dpaa2_sg_entry *sg_table;
3482                 int src_len = req->nbytes - *next_buflen;
3483
3484                 src_nents = sg_nents_for_len(req->src, src_len);
3485                 if (src_nents < 0) {
3486                         dev_err(ctx->dev, "Invalid number of src SG.\n");
3487                         return src_nents;
3488                 }
3489
3490                 if (src_nents) {
3491                         mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
3492                                                   DMA_TO_DEVICE);
3493                         if (!mapped_nents) {
3494                                 dev_err(ctx->dev, "unable to DMA map source\n");
3495                                 return -ENOMEM;
3496                         }
3497                 } else {
3498                         mapped_nents = 0;
3499                 }
3500
3501                 /* allocate space for base edesc and link tables */
3502                 edesc = qi_cache_zalloc(GFP_DMA | flags);
3503                 if (!edesc) {
3504                         dma_unmap_sg(ctx->dev, req->src, src_nents,
3505                                      DMA_TO_DEVICE);
3506                         return -ENOMEM;
3507                 }
3508
3509                 edesc->src_nents = src_nents;
3510                 qm_sg_src_index = 1 + (*buflen ? 1 : 0);
3511                 qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) *
3512                               sizeof(*sg_table);
3513                 sg_table = &edesc->sgt[0];
3514
3515                 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
3516                                        DMA_BIDIRECTIONAL);
3517                 if (ret)
3518                         goto unmap_ctx;
3519
3520                 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
3521                 if (ret)
3522                         goto unmap_ctx;
3523
3524                 if (mapped_nents) {
3525                         sg_to_qm_sg_last(req->src, src_len,
3526                                          sg_table + qm_sg_src_index, 0);
3527                         if (*next_buflen)
3528                                 scatterwalk_map_and_copy(next_buf, req->src,
3529                                                          to_hash - *buflen,
3530                                                          *next_buflen, 0);
3531                 } else {
3532                         dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1,
3533                                            true);
3534                 }
3535
3536                 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
3537                                                   qm_sg_bytes, DMA_TO_DEVICE);
3538                 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3539                         dev_err(ctx->dev, "unable to map S/G table\n");
3540                         ret = -ENOMEM;
3541                         goto unmap_ctx;
3542                 }
3543                 edesc->qm_sg_bytes = qm_sg_bytes;
3544
3545                 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3546                 dpaa2_fl_set_final(in_fle, true);
3547                 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3548                 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3549                 dpaa2_fl_set_len(in_fle, ctx->ctx_len + to_hash);
3550                 dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3551                 dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3552                 dpaa2_fl_set_len(out_fle, ctx->ctx_len);
3553
3554                 req_ctx->flc = &ctx->flc[UPDATE];
3555                 req_ctx->flc_dma = ctx->flc_dma[UPDATE];
3556                 req_ctx->cbk = ahash_done_bi;
3557                 req_ctx->ctx = &req->base;
3558                 req_ctx->edesc = edesc;
3559
3560                 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3561                 if (ret != -EINPROGRESS &&
3562                     !(ret == -EBUSY &&
3563                       req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3564                         goto unmap_ctx;
3565         } else if (*next_buflen) {
3566                 scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
3567                                          req->nbytes, 0);
3568                 *buflen = *next_buflen;
3569                 *next_buflen = last_buflen;
3570         }
3571
3572         print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
3573                              DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
3574         print_hex_dump_debug("next buf@" __stringify(__LINE__)": ",
3575                              DUMP_PREFIX_ADDRESS, 16, 4, next_buf, *next_buflen,
3576                              1);
3577
3578         return ret;
3579 unmap_ctx:
3580         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3581         qi_cache_free(edesc);
3582         return ret;
3583 }
3584
3585 static int ahash_final_ctx(struct ahash_request *req)
3586 {
3587         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3588         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3589         struct caam_hash_state *state = ahash_request_ctx(req);
3590         struct caam_request *req_ctx = &state->caam_req;
3591         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3592         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3593         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3594                       GFP_KERNEL : GFP_ATOMIC;
3595         int buflen = *current_buflen(state);
3596         int qm_sg_bytes;
3597         int digestsize = crypto_ahash_digestsize(ahash);
3598         struct ahash_edesc *edesc;
3599         struct dpaa2_sg_entry *sg_table;
3600         int ret;
3601
3602         /* allocate space for base edesc and link tables */
3603         edesc = qi_cache_zalloc(GFP_DMA | flags);
3604         if (!edesc)
3605                 return -ENOMEM;
3606
3607         qm_sg_bytes = pad_sg_nents(1 + (buflen ? 1 : 0)) * sizeof(*sg_table);
3608         sg_table = &edesc->sgt[0];
3609
3610         ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
3611                                DMA_BIDIRECTIONAL);
3612         if (ret)
3613                 goto unmap_ctx;
3614
3615         ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
3616         if (ret)
3617                 goto unmap_ctx;
3618
3619         dpaa2_sg_set_final(sg_table + (buflen ? 1 : 0), true);
3620
3621         edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
3622                                           DMA_TO_DEVICE);
3623         if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3624                 dev_err(ctx->dev, "unable to map S/G table\n");
3625                 ret = -ENOMEM;
3626                 goto unmap_ctx;
3627         }
3628         edesc->qm_sg_bytes = qm_sg_bytes;
3629
3630         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3631         dpaa2_fl_set_final(in_fle, true);
3632         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3633         dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3634         dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen);
3635         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3636         dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3637         dpaa2_fl_set_len(out_fle, digestsize);
3638
3639         req_ctx->flc = &ctx->flc[FINALIZE];
3640         req_ctx->flc_dma = ctx->flc_dma[FINALIZE];
3641         req_ctx->cbk = ahash_done_ctx_src;
3642         req_ctx->ctx = &req->base;
3643         req_ctx->edesc = edesc;
3644
3645         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3646         if (ret == -EINPROGRESS ||
3647             (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3648                 return ret;
3649
3650 unmap_ctx:
3651         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3652         qi_cache_free(edesc);
3653         return ret;
3654 }
3655
3656 static int ahash_finup_ctx(struct ahash_request *req)
3657 {
3658         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3659         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3660         struct caam_hash_state *state = ahash_request_ctx(req);
3661         struct caam_request *req_ctx = &state->caam_req;
3662         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3663         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3664         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3665                       GFP_KERNEL : GFP_ATOMIC;
3666         int buflen = *current_buflen(state);
3667         int qm_sg_bytes, qm_sg_src_index;
3668         int src_nents, mapped_nents;
3669         int digestsize = crypto_ahash_digestsize(ahash);
3670         struct ahash_edesc *edesc;
3671         struct dpaa2_sg_entry *sg_table;
3672         int ret;
3673
3674         src_nents = sg_nents_for_len(req->src, req->nbytes);
3675         if (src_nents < 0) {
3676                 dev_err(ctx->dev, "Invalid number of src SG.\n");
3677                 return src_nents;
3678         }
3679
3680         if (src_nents) {
3681                 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
3682                                           DMA_TO_DEVICE);
3683                 if (!mapped_nents) {
3684                         dev_err(ctx->dev, "unable to DMA map source\n");
3685                         return -ENOMEM;
3686                 }
3687         } else {
3688                 mapped_nents = 0;
3689         }
3690
3691         /* allocate space for base edesc and link tables */
3692         edesc = qi_cache_zalloc(GFP_DMA | flags);
3693         if (!edesc) {
3694                 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
3695                 return -ENOMEM;
3696         }
3697
3698         edesc->src_nents = src_nents;
3699         qm_sg_src_index = 1 + (buflen ? 1 : 0);
3700         qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) *
3701                       sizeof(*sg_table);
3702         sg_table = &edesc->sgt[0];
3703
3704         ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
3705                                DMA_BIDIRECTIONAL);
3706         if (ret)
3707                 goto unmap_ctx;
3708
3709         ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
3710         if (ret)
3711                 goto unmap_ctx;
3712
3713         sg_to_qm_sg_last(req->src, req->nbytes, sg_table + qm_sg_src_index, 0);
3714
3715         edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
3716                                           DMA_TO_DEVICE);
3717         if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3718                 dev_err(ctx->dev, "unable to map S/G table\n");
3719                 ret = -ENOMEM;
3720                 goto unmap_ctx;
3721         }
3722         edesc->qm_sg_bytes = qm_sg_bytes;
3723
3724         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3725         dpaa2_fl_set_final(in_fle, true);
3726         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3727         dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3728         dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen + req->nbytes);
3729         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3730         dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3731         dpaa2_fl_set_len(out_fle, digestsize);
3732
3733         req_ctx->flc = &ctx->flc[FINALIZE];
3734         req_ctx->flc_dma = ctx->flc_dma[FINALIZE];
3735         req_ctx->cbk = ahash_done_ctx_src;
3736         req_ctx->ctx = &req->base;
3737         req_ctx->edesc = edesc;
3738
3739         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3740         if (ret == -EINPROGRESS ||
3741             (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3742                 return ret;
3743
3744 unmap_ctx:
3745         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3746         qi_cache_free(edesc);
3747         return ret;
3748 }
3749
3750 static int ahash_digest(struct ahash_request *req)
3751 {
3752         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3753         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3754         struct caam_hash_state *state = ahash_request_ctx(req);
3755         struct caam_request *req_ctx = &state->caam_req;
3756         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3757         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3758         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3759                       GFP_KERNEL : GFP_ATOMIC;
3760         int digestsize = crypto_ahash_digestsize(ahash);
3761         int src_nents, mapped_nents;
3762         struct ahash_edesc *edesc;
3763         int ret = -ENOMEM;
3764
3765         state->buf_dma = 0;
3766
3767         src_nents = sg_nents_for_len(req->src, req->nbytes);
3768         if (src_nents < 0) {
3769                 dev_err(ctx->dev, "Invalid number of src SG.\n");
3770                 return src_nents;
3771         }
3772
3773         if (src_nents) {
3774                 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
3775                                           DMA_TO_DEVICE);
3776                 if (!mapped_nents) {
3777                         dev_err(ctx->dev, "unable to map source for DMA\n");
3778                         return ret;
3779                 }
3780         } else {
3781                 mapped_nents = 0;
3782         }
3783
3784         /* allocate space for base edesc and link tables */
3785         edesc = qi_cache_zalloc(GFP_DMA | flags);
3786         if (!edesc) {
3787                 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
3788                 return ret;
3789         }
3790
3791         edesc->src_nents = src_nents;
3792         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3793
3794         if (mapped_nents > 1) {
3795                 int qm_sg_bytes;
3796                 struct dpaa2_sg_entry *sg_table = &edesc->sgt[0];
3797
3798                 qm_sg_bytes = pad_sg_nents(mapped_nents) * sizeof(*sg_table);
3799                 sg_to_qm_sg_last(req->src, req->nbytes, sg_table, 0);
3800                 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
3801                                                   qm_sg_bytes, DMA_TO_DEVICE);
3802                 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3803                         dev_err(ctx->dev, "unable to map S/G table\n");
3804                         goto unmap;
3805                 }
3806                 edesc->qm_sg_bytes = qm_sg_bytes;
3807                 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3808                 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3809         } else {
3810                 dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
3811                 dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src));
3812         }
3813
3814         state->ctx_dma_len = digestsize;
3815         state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize,
3816                                         DMA_FROM_DEVICE);
3817         if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
3818                 dev_err(ctx->dev, "unable to map ctx\n");
3819                 state->ctx_dma = 0;
3820                 goto unmap;
3821         }
3822
3823         dpaa2_fl_set_final(in_fle, true);
3824         dpaa2_fl_set_len(in_fle, req->nbytes);
3825         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3826         dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3827         dpaa2_fl_set_len(out_fle, digestsize);
3828
3829         req_ctx->flc = &ctx->flc[DIGEST];
3830         req_ctx->flc_dma = ctx->flc_dma[DIGEST];
3831         req_ctx->cbk = ahash_done;
3832         req_ctx->ctx = &req->base;
3833         req_ctx->edesc = edesc;
3834         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3835         if (ret == -EINPROGRESS ||
3836             (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3837                 return ret;
3838
3839 unmap:
3840         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3841         qi_cache_free(edesc);
3842         return ret;
3843 }
3844
3845 static int ahash_final_no_ctx(struct ahash_request *req)
3846 {
3847         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3848         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3849         struct caam_hash_state *state = ahash_request_ctx(req);
3850         struct caam_request *req_ctx = &state->caam_req;
3851         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3852         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3853         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3854                       GFP_KERNEL : GFP_ATOMIC;
3855         u8 *buf = current_buf(state);
3856         int buflen = *current_buflen(state);
3857         int digestsize = crypto_ahash_digestsize(ahash);
3858         struct ahash_edesc *edesc;
3859         int ret = -ENOMEM;
3860
3861         /* allocate space for base edesc and link tables */
3862         edesc = qi_cache_zalloc(GFP_DMA | flags);
3863         if (!edesc)
3864                 return ret;
3865
3866         if (buflen) {
3867                 state->buf_dma = dma_map_single(ctx->dev, buf, buflen,
3868                                                 DMA_TO_DEVICE);
3869                 if (dma_mapping_error(ctx->dev, state->buf_dma)) {
3870                         dev_err(ctx->dev, "unable to map src\n");
3871                         goto unmap;
3872                 }
3873         }
3874
3875         state->ctx_dma_len = digestsize;
3876         state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize,
3877                                         DMA_FROM_DEVICE);
3878         if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
3879                 dev_err(ctx->dev, "unable to map ctx\n");
3880                 state->ctx_dma = 0;
3881                 goto unmap;
3882         }
3883
3884         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3885         dpaa2_fl_set_final(in_fle, true);
3886         /*
3887          * crypto engine requires the input entry to be present when
3888          * "frame list" FD is used.
3889          * Since engine does not support FMT=2'b11 (unused entry type), leaving
3890          * in_fle zeroized (except for "Final" flag) is the best option.
3891          */
3892         if (buflen) {
3893                 dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
3894                 dpaa2_fl_set_addr(in_fle, state->buf_dma);
3895                 dpaa2_fl_set_len(in_fle, buflen);
3896         }
3897         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3898         dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3899         dpaa2_fl_set_len(out_fle, digestsize);
3900
3901         req_ctx->flc = &ctx->flc[DIGEST];
3902         req_ctx->flc_dma = ctx->flc_dma[DIGEST];
3903         req_ctx->cbk = ahash_done;
3904         req_ctx->ctx = &req->base;
3905         req_ctx->edesc = edesc;
3906
3907         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3908         if (ret == -EINPROGRESS ||
3909             (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3910                 return ret;
3911
3912 unmap:
3913         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3914         qi_cache_free(edesc);
3915         return ret;
3916 }
3917
3918 static int ahash_update_no_ctx(struct ahash_request *req)
3919 {
3920         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3921         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3922         struct caam_hash_state *state = ahash_request_ctx(req);
3923         struct caam_request *req_ctx = &state->caam_req;
3924         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3925         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3926         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3927                       GFP_KERNEL : GFP_ATOMIC;
3928         u8 *buf = current_buf(state);
3929         int *buflen = current_buflen(state);
3930         u8 *next_buf = alt_buf(state);
3931         int *next_buflen = alt_buflen(state);
3932         int in_len = *buflen + req->nbytes, to_hash;
3933         int qm_sg_bytes, src_nents, mapped_nents;
3934         struct ahash_edesc *edesc;
3935         int ret = 0;
3936
3937         *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
3938         to_hash = in_len - *next_buflen;
3939
3940         if (to_hash) {
3941                 struct dpaa2_sg_entry *sg_table;
3942                 int src_len = req->nbytes - *next_buflen;
3943
3944                 src_nents = sg_nents_for_len(req->src, src_len);
3945                 if (src_nents < 0) {
3946                         dev_err(ctx->dev, "Invalid number of src SG.\n");
3947                         return src_nents;
3948                 }
3949
3950                 if (src_nents) {
3951                         mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
3952                                                   DMA_TO_DEVICE);
3953                         if (!mapped_nents) {
3954                                 dev_err(ctx->dev, "unable to DMA map source\n");
3955                                 return -ENOMEM;
3956                         }
3957                 } else {
3958                         mapped_nents = 0;
3959                 }
3960
3961                 /* allocate space for base edesc and link tables */
3962                 edesc = qi_cache_zalloc(GFP_DMA | flags);
3963                 if (!edesc) {
3964                         dma_unmap_sg(ctx->dev, req->src, src_nents,
3965                                      DMA_TO_DEVICE);
3966                         return -ENOMEM;
3967                 }
3968
3969                 edesc->src_nents = src_nents;
3970                 qm_sg_bytes = pad_sg_nents(1 + mapped_nents) *
3971                               sizeof(*sg_table);
3972                 sg_table = &edesc->sgt[0];
3973
3974                 ret = buf_map_to_qm_sg(ctx->dev, sg_table, state);
3975                 if (ret)
3976                         goto unmap_ctx;
3977
3978                 sg_to_qm_sg_last(req->src, src_len, sg_table + 1, 0);
3979
3980                 if (*next_buflen)
3981                         scatterwalk_map_and_copy(next_buf, req->src,
3982                                                  to_hash - *buflen,
3983                                                  *next_buflen, 0);
3984
3985                 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
3986                                                   qm_sg_bytes, DMA_TO_DEVICE);
3987                 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3988                         dev_err(ctx->dev, "unable to map S/G table\n");
3989                         ret = -ENOMEM;
3990                         goto unmap_ctx;
3991                 }
3992                 edesc->qm_sg_bytes = qm_sg_bytes;
3993
3994                 state->ctx_dma_len = ctx->ctx_len;
3995                 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx,
3996                                                 ctx->ctx_len, DMA_FROM_DEVICE);
3997                 if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
3998                         dev_err(ctx->dev, "unable to map ctx\n");
3999                         state->ctx_dma = 0;
4000                         ret = -ENOMEM;
4001                         goto unmap_ctx;
4002                 }
4003
4004                 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
4005                 dpaa2_fl_set_final(in_fle, true);
4006                 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
4007                 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
4008                 dpaa2_fl_set_len(in_fle, to_hash);
4009                 dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
4010                 dpaa2_fl_set_addr(out_fle, state->ctx_dma);
4011                 dpaa2_fl_set_len(out_fle, ctx->ctx_len);
4012
4013                 req_ctx->flc = &ctx->flc[UPDATE_FIRST];
4014                 req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST];
4015                 req_ctx->cbk = ahash_done_ctx_dst;
4016                 req_ctx->ctx = &req->base;
4017                 req_ctx->edesc = edesc;
4018
4019                 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
4020                 if (ret != -EINPROGRESS &&
4021                     !(ret == -EBUSY &&
4022                       req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
4023                         goto unmap_ctx;
4024
4025                 state->update = ahash_update_ctx;
4026                 state->finup = ahash_finup_ctx;
4027                 state->final = ahash_final_ctx;
4028         } else if (*next_buflen) {
4029                 scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
4030                                          req->nbytes, 0);
4031                 *buflen = *next_buflen;
4032                 *next_buflen = 0;
4033         }
4034
4035         print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
4036                              DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
4037         print_hex_dump_debug("next buf@" __stringify(__LINE__)": ",
4038                              DUMP_PREFIX_ADDRESS, 16, 4, next_buf, *next_buflen,
4039                              1);
4040
4041         return ret;
4042 unmap_ctx:
4043         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE);
4044         qi_cache_free(edesc);
4045         return ret;
4046 }
4047
4048 static int ahash_finup_no_ctx(struct ahash_request *req)
4049 {
4050         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
4051         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
4052         struct caam_hash_state *state = ahash_request_ctx(req);
4053         struct caam_request *req_ctx = &state->caam_req;
4054         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
4055         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
4056         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
4057                       GFP_KERNEL : GFP_ATOMIC;
4058         int buflen = *current_buflen(state);
4059         int qm_sg_bytes, src_nents, mapped_nents;
4060         int digestsize = crypto_ahash_digestsize(ahash);
4061         struct ahash_edesc *edesc;
4062         struct dpaa2_sg_entry *sg_table;
4063         int ret;
4064
4065         src_nents = sg_nents_for_len(req->src, req->nbytes);
4066         if (src_nents < 0) {
4067                 dev_err(ctx->dev, "Invalid number of src SG.\n");
4068                 return src_nents;
4069         }
4070
4071         if (src_nents) {
4072                 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
4073                                           DMA_TO_DEVICE);
4074                 if (!mapped_nents) {
4075                         dev_err(ctx->dev, "unable to DMA map source\n");
4076                         return -ENOMEM;
4077                 }
4078         } else {
4079                 mapped_nents = 0;
4080         }
4081
4082         /* allocate space for base edesc and link tables */
4083         edesc = qi_cache_zalloc(GFP_DMA | flags);
4084         if (!edesc) {
4085                 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
4086                 return -ENOMEM;
4087         }
4088
4089         edesc->src_nents = src_nents;
4090         qm_sg_bytes = pad_sg_nents(2 + mapped_nents) * sizeof(*sg_table);
4091         sg_table = &edesc->sgt[0];
4092
4093         ret = buf_map_to_qm_sg(ctx->dev, sg_table, state);
4094         if (ret)
4095                 goto unmap;
4096
4097         sg_to_qm_sg_last(req->src, req->nbytes, sg_table + 1, 0);
4098
4099         edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
4100                                           DMA_TO_DEVICE);
4101         if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
4102                 dev_err(ctx->dev, "unable to map S/G table\n");
4103                 ret = -ENOMEM;
4104                 goto unmap;
4105         }
4106         edesc->qm_sg_bytes = qm_sg_bytes;
4107
4108         state->ctx_dma_len = digestsize;
4109         state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize,
4110                                         DMA_FROM_DEVICE);
4111         if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
4112                 dev_err(ctx->dev, "unable to map ctx\n");
4113                 state->ctx_dma = 0;
4114                 ret = -ENOMEM;
4115                 goto unmap;
4116         }
4117
4118         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
4119         dpaa2_fl_set_final(in_fle, true);
4120         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
4121         dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
4122         dpaa2_fl_set_len(in_fle, buflen + req->nbytes);
4123         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
4124         dpaa2_fl_set_addr(out_fle, state->ctx_dma);
4125         dpaa2_fl_set_len(out_fle, digestsize);
4126
4127         req_ctx->flc = &ctx->flc[DIGEST];
4128         req_ctx->flc_dma = ctx->flc_dma[DIGEST];
4129         req_ctx->cbk = ahash_done;
4130         req_ctx->ctx = &req->base;
4131         req_ctx->edesc = edesc;
4132         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
4133         if (ret != -EINPROGRESS &&
4134             !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
4135                 goto unmap;
4136
4137         return ret;
4138 unmap:
4139         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
4140         qi_cache_free(edesc);
4141         return -ENOMEM;
4142 }
4143
4144 static int ahash_update_first(struct ahash_request *req)
4145 {
4146         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
4147         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
4148         struct caam_hash_state *state = ahash_request_ctx(req);
4149         struct caam_request *req_ctx = &state->caam_req;
4150         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
4151         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
4152         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
4153                       GFP_KERNEL : GFP_ATOMIC;
4154         u8 *next_buf = alt_buf(state);
4155         int *next_buflen = alt_buflen(state);
4156         int to_hash;
4157         int src_nents, mapped_nents;
4158         struct ahash_edesc *edesc;
4159         int ret = 0;
4160
4161         *next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) -
4162                                       1);
4163         to_hash = req->nbytes - *next_buflen;
4164
4165         if (to_hash) {
4166                 struct dpaa2_sg_entry *sg_table;
4167                 int src_len = req->nbytes - *next_buflen;
4168
4169                 src_nents = sg_nents_for_len(req->src, src_len);
4170                 if (src_nents < 0) {
4171                         dev_err(ctx->dev, "Invalid number of src SG.\n");
4172                         return src_nents;
4173                 }
4174
4175                 if (src_nents) {
4176                         mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
4177                                                   DMA_TO_DEVICE);
4178                         if (!mapped_nents) {
4179                                 dev_err(ctx->dev, "unable to map source for DMA\n");
4180                                 return -ENOMEM;
4181                         }
4182                 } else {
4183                         mapped_nents = 0;
4184                 }
4185
4186                 /* allocate space for base edesc and link tables */
4187                 edesc = qi_cache_zalloc(GFP_DMA | flags);
4188                 if (!edesc) {
4189                         dma_unmap_sg(ctx->dev, req->src, src_nents,
4190                                      DMA_TO_DEVICE);
4191                         return -ENOMEM;
4192                 }
4193
4194                 edesc->src_nents = src_nents;
4195                 sg_table = &edesc->sgt[0];
4196
4197                 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
4198                 dpaa2_fl_set_final(in_fle, true);
4199                 dpaa2_fl_set_len(in_fle, to_hash);
4200
4201                 if (mapped_nents > 1) {
4202                         int qm_sg_bytes;
4203
4204                         sg_to_qm_sg_last(req->src, src_len, sg_table, 0);
4205                         qm_sg_bytes = pad_sg_nents(mapped_nents) *
4206                                       sizeof(*sg_table);
4207                         edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
4208                                                           qm_sg_bytes,
4209                                                           DMA_TO_DEVICE);
4210                         if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
4211                                 dev_err(ctx->dev, "unable to map S/G table\n");
4212                                 ret = -ENOMEM;
4213                                 goto unmap_ctx;
4214                         }
4215                         edesc->qm_sg_bytes = qm_sg_bytes;
4216                         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
4217                         dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
4218                 } else {
4219                         dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
4220                         dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src));
4221                 }
4222
4223                 if (*next_buflen)
4224                         scatterwalk_map_and_copy(next_buf, req->src, to_hash,
4225                                                  *next_buflen, 0);
4226
4227                 state->ctx_dma_len = ctx->ctx_len;
4228                 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx,
4229                                                 ctx->ctx_len, DMA_FROM_DEVICE);
4230                 if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
4231                         dev_err(ctx->dev, "unable to map ctx\n");
4232                         state->ctx_dma = 0;
4233                         ret = -ENOMEM;
4234                         goto unmap_ctx;
4235                 }
4236
4237                 dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
4238                 dpaa2_fl_set_addr(out_fle, state->ctx_dma);
4239                 dpaa2_fl_set_len(out_fle, ctx->ctx_len);
4240
4241                 req_ctx->flc = &ctx->flc[UPDATE_FIRST];
4242                 req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST];
4243                 req_ctx->cbk = ahash_done_ctx_dst;
4244                 req_ctx->ctx = &req->base;
4245                 req_ctx->edesc = edesc;
4246
4247                 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
4248                 if (ret != -EINPROGRESS &&
4249                     !(ret == -EBUSY && req->base.flags &
4250                       CRYPTO_TFM_REQ_MAY_BACKLOG))
4251                         goto unmap_ctx;
4252
4253                 state->update = ahash_update_ctx;
4254                 state->finup = ahash_finup_ctx;
4255                 state->final = ahash_final_ctx;
4256         } else if (*next_buflen) {
4257                 state->update = ahash_update_no_ctx;
4258                 state->finup = ahash_finup_no_ctx;
4259                 state->final = ahash_final_no_ctx;
4260                 scatterwalk_map_and_copy(next_buf, req->src, 0,
4261                                          req->nbytes, 0);
4262                 switch_buf(state);
4263         }
4264
4265         print_hex_dump_debug("next buf@" __stringify(__LINE__)": ",
4266                              DUMP_PREFIX_ADDRESS, 16, 4, next_buf, *next_buflen,
4267                              1);
4268
4269         return ret;
4270 unmap_ctx:
4271         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE);
4272         qi_cache_free(edesc);
4273         return ret;
4274 }
4275
4276 static int ahash_finup_first(struct ahash_request *req)
4277 {
4278         return ahash_digest(req);
4279 }
4280
4281 static int ahash_init(struct ahash_request *req)
4282 {
4283         struct caam_hash_state *state = ahash_request_ctx(req);
4284
4285         state->update = ahash_update_first;
4286         state->finup = ahash_finup_first;
4287         state->final = ahash_final_no_ctx;
4288
4289         state->ctx_dma = 0;
4290         state->ctx_dma_len = 0;
4291         state->current_buf = 0;
4292         state->buf_dma = 0;
4293         state->buflen_0 = 0;
4294         state->buflen_1 = 0;
4295
4296         return 0;
4297 }
4298
4299 static int ahash_update(struct ahash_request *req)
4300 {
4301         struct caam_hash_state *state = ahash_request_ctx(req);
4302
4303         return state->update(req);
4304 }
4305
4306 static int ahash_finup(struct ahash_request *req)
4307 {
4308         struct caam_hash_state *state = ahash_request_ctx(req);
4309
4310         return state->finup(req);
4311 }
4312
4313 static int ahash_final(struct ahash_request *req)
4314 {
4315         struct caam_hash_state *state = ahash_request_ctx(req);
4316
4317         return state->final(req);
4318 }
4319
4320 static int ahash_export(struct ahash_request *req, void *out)
4321 {
4322         struct caam_hash_state *state = ahash_request_ctx(req);
4323         struct caam_export_state *export = out;
4324         int len;
4325         u8 *buf;
4326
4327         if (state->current_buf) {
4328                 buf = state->buf_1;
4329                 len = state->buflen_1;
4330         } else {
4331                 buf = state->buf_0;
4332                 len = state->buflen_0;
4333         }
4334
4335         memcpy(export->buf, buf, len);
4336         memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx));
4337         export->buflen = len;
4338         export->update = state->update;
4339         export->final = state->final;
4340         export->finup = state->finup;
4341
4342         return 0;
4343 }
4344
4345 static int ahash_import(struct ahash_request *req, const void *in)
4346 {
4347         struct caam_hash_state *state = ahash_request_ctx(req);
4348         const struct caam_export_state *export = in;
4349
4350         memset(state, 0, sizeof(*state));
4351         memcpy(state->buf_0, export->buf, export->buflen);
4352         memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx));
4353         state->buflen_0 = export->buflen;
4354         state->update = export->update;
4355         state->final = export->final;
4356         state->finup = export->finup;
4357
4358         return 0;
4359 }
4360
4361 struct caam_hash_template {
4362         char name[CRYPTO_MAX_ALG_NAME];
4363         char driver_name[CRYPTO_MAX_ALG_NAME];
4364         char hmac_name[CRYPTO_MAX_ALG_NAME];
4365         char hmac_driver_name[CRYPTO_MAX_ALG_NAME];
4366         unsigned int blocksize;
4367         struct ahash_alg template_ahash;
4368         u32 alg_type;
4369 };
4370
4371 /* ahash descriptors */
4372 static struct caam_hash_template driver_hash[] = {
4373         {
4374                 .name = "sha1",
4375                 .driver_name = "sha1-caam-qi2",
4376                 .hmac_name = "hmac(sha1)",
4377                 .hmac_driver_name = "hmac-sha1-caam-qi2",
4378                 .blocksize = SHA1_BLOCK_SIZE,
4379                 .template_ahash = {
4380                         .init = ahash_init,
4381                         .update = ahash_update,
4382                         .final = ahash_final,
4383                         .finup = ahash_finup,
4384                         .digest = ahash_digest,
4385                         .export = ahash_export,
4386                         .import = ahash_import,
4387                         .setkey = ahash_setkey,
4388                         .halg = {
4389                                 .digestsize = SHA1_DIGEST_SIZE,
4390                                 .statesize = sizeof(struct caam_export_state),
4391                         },
4392                 },
4393                 .alg_type = OP_ALG_ALGSEL_SHA1,
4394         }, {
4395                 .name = "sha224",
4396                 .driver_name = "sha224-caam-qi2",
4397                 .hmac_name = "hmac(sha224)",
4398                 .hmac_driver_name = "hmac-sha224-caam-qi2",
4399                 .blocksize = SHA224_BLOCK_SIZE,
4400                 .template_ahash = {
4401                         .init = ahash_init,
4402                         .update = ahash_update,
4403                         .final = ahash_final,
4404                         .finup = ahash_finup,
4405                         .digest = ahash_digest,
4406                         .export = ahash_export,
4407                         .import = ahash_import,
4408                         .setkey = ahash_setkey,
4409                         .halg = {
4410                                 .digestsize = SHA224_DIGEST_SIZE,
4411                                 .statesize = sizeof(struct caam_export_state),
4412                         },
4413                 },
4414                 .alg_type = OP_ALG_ALGSEL_SHA224,
4415         }, {
4416                 .name = "sha256",
4417                 .driver_name = "sha256-caam-qi2",
4418                 .hmac_name = "hmac(sha256)",
4419                 .hmac_driver_name = "hmac-sha256-caam-qi2",
4420                 .blocksize = SHA256_BLOCK_SIZE,
4421                 .template_ahash = {
4422                         .init = ahash_init,
4423                         .update = ahash_update,
4424                         .final = ahash_final,
4425                         .finup = ahash_finup,
4426                         .digest = ahash_digest,
4427                         .export = ahash_export,
4428                         .import = ahash_import,
4429                         .setkey = ahash_setkey,
4430                         .halg = {
4431                                 .digestsize = SHA256_DIGEST_SIZE,
4432                                 .statesize = sizeof(struct caam_export_state),
4433                         },
4434                 },
4435                 .alg_type = OP_ALG_ALGSEL_SHA256,
4436         }, {
4437                 .name = "sha384",
4438                 .driver_name = "sha384-caam-qi2",
4439                 .hmac_name = "hmac(sha384)",
4440                 .hmac_driver_name = "hmac-sha384-caam-qi2",
4441                 .blocksize = SHA384_BLOCK_SIZE,
4442                 .template_ahash = {
4443                         .init = ahash_init,
4444                         .update = ahash_update,
4445                         .final = ahash_final,
4446                         .finup = ahash_finup,
4447                         .digest = ahash_digest,
4448                         .export = ahash_export,
4449                         .import = ahash_import,
4450                         .setkey = ahash_setkey,
4451                         .halg = {
4452                                 .digestsize = SHA384_DIGEST_SIZE,
4453                                 .statesize = sizeof(struct caam_export_state),
4454                         },
4455                 },
4456                 .alg_type = OP_ALG_ALGSEL_SHA384,
4457         }, {
4458                 .name = "sha512",
4459                 .driver_name = "sha512-caam-qi2",
4460                 .hmac_name = "hmac(sha512)",
4461                 .hmac_driver_name = "hmac-sha512-caam-qi2",
4462                 .blocksize = SHA512_BLOCK_SIZE,
4463                 .template_ahash = {
4464                         .init = ahash_init,
4465                         .update = ahash_update,
4466                         .final = ahash_final,
4467                         .finup = ahash_finup,
4468                         .digest = ahash_digest,
4469                         .export = ahash_export,
4470                         .import = ahash_import,
4471                         .setkey = ahash_setkey,
4472                         .halg = {
4473                                 .digestsize = SHA512_DIGEST_SIZE,
4474                                 .statesize = sizeof(struct caam_export_state),
4475                         },
4476                 },
4477                 .alg_type = OP_ALG_ALGSEL_SHA512,
4478         }, {
4479                 .name = "md5",
4480                 .driver_name = "md5-caam-qi2",
4481                 .hmac_name = "hmac(md5)",
4482                 .hmac_driver_name = "hmac-md5-caam-qi2",
4483                 .blocksize = MD5_BLOCK_WORDS * 4,
4484                 .template_ahash = {
4485                         .init = ahash_init,
4486                         .update = ahash_update,
4487                         .final = ahash_final,
4488                         .finup = ahash_finup,
4489                         .digest = ahash_digest,
4490                         .export = ahash_export,
4491                         .import = ahash_import,
4492                         .setkey = ahash_setkey,
4493                         .halg = {
4494                                 .digestsize = MD5_DIGEST_SIZE,
4495                                 .statesize = sizeof(struct caam_export_state),
4496                         },
4497                 },
4498                 .alg_type = OP_ALG_ALGSEL_MD5,
4499         }
4500 };
4501
4502 struct caam_hash_alg {
4503         struct list_head entry;
4504         struct device *dev;
4505         int alg_type;
4506         struct ahash_alg ahash_alg;
4507 };
4508
4509 static int caam_hash_cra_init(struct crypto_tfm *tfm)
4510 {
4511         struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
4512         struct crypto_alg *base = tfm->__crt_alg;
4513         struct hash_alg_common *halg =
4514                  container_of(base, struct hash_alg_common, base);
4515         struct ahash_alg *alg =
4516                  container_of(halg, struct ahash_alg, halg);
4517         struct caam_hash_alg *caam_hash =
4518                  container_of(alg, struct caam_hash_alg, ahash_alg);
4519         struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
4520         /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */
4521         static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE,
4522                                          HASH_MSG_LEN + SHA1_DIGEST_SIZE,
4523                                          HASH_MSG_LEN + 32,
4524                                          HASH_MSG_LEN + SHA256_DIGEST_SIZE,
4525                                          HASH_MSG_LEN + 64,
4526                                          HASH_MSG_LEN + SHA512_DIGEST_SIZE };
4527         dma_addr_t dma_addr;
4528         int i;
4529
4530         ctx->dev = caam_hash->dev;
4531
4532         if (alg->setkey) {
4533                 ctx->adata.key_dma = dma_map_single_attrs(ctx->dev, ctx->key,
4534                                                           ARRAY_SIZE(ctx->key),
4535                                                           DMA_TO_DEVICE,
4536                                                           DMA_ATTR_SKIP_CPU_SYNC);
4537                 if (dma_mapping_error(ctx->dev, ctx->adata.key_dma)) {
4538                         dev_err(ctx->dev, "unable to map key\n");
4539                         return -ENOMEM;
4540                 }
4541         }
4542
4543         dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, sizeof(ctx->flc),
4544                                         DMA_BIDIRECTIONAL,
4545                                         DMA_ATTR_SKIP_CPU_SYNC);
4546         if (dma_mapping_error(ctx->dev, dma_addr)) {
4547                 dev_err(ctx->dev, "unable to map shared descriptors\n");
4548                 if (ctx->adata.key_dma)
4549                         dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma,
4550                                                ARRAY_SIZE(ctx->key),
4551                                                DMA_TO_DEVICE,
4552                                                DMA_ATTR_SKIP_CPU_SYNC);
4553                 return -ENOMEM;
4554         }
4555
4556         for (i = 0; i < HASH_NUM_OP; i++)
4557                 ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]);
4558
4559         /* copy descriptor header template value */
4560         ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type;
4561
4562         ctx->ctx_len = runninglen[(ctx->adata.algtype &
4563                                    OP_ALG_ALGSEL_SUBMASK) >>
4564                                   OP_ALG_ALGSEL_SHIFT];
4565
4566         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
4567                                  sizeof(struct caam_hash_state));
4568
4569         return ahash_set_sh_desc(ahash);
4570 }
4571
4572 static void caam_hash_cra_exit(struct crypto_tfm *tfm)
4573 {
4574         struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
4575
4576         dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], sizeof(ctx->flc),
4577                                DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC);
4578         if (ctx->adata.key_dma)
4579                 dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma,
4580                                        ARRAY_SIZE(ctx->key), DMA_TO_DEVICE,
4581                                        DMA_ATTR_SKIP_CPU_SYNC);
4582 }
4583
4584 static struct caam_hash_alg *caam_hash_alloc(struct device *dev,
4585         struct caam_hash_template *template, bool keyed)
4586 {
4587         struct caam_hash_alg *t_alg;
4588         struct ahash_alg *halg;
4589         struct crypto_alg *alg;
4590
4591         t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
4592         if (!t_alg)
4593                 return ERR_PTR(-ENOMEM);
4594
4595         t_alg->ahash_alg = template->template_ahash;
4596         halg = &t_alg->ahash_alg;
4597         alg = &halg->halg.base;
4598
4599         if (keyed) {
4600                 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
4601                          template->hmac_name);
4602                 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
4603                          template->hmac_driver_name);
4604         } else {
4605                 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
4606                          template->name);
4607                 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
4608                          template->driver_name);
4609                 t_alg->ahash_alg.setkey = NULL;
4610         }
4611         alg->cra_module = THIS_MODULE;
4612         alg->cra_init = caam_hash_cra_init;
4613         alg->cra_exit = caam_hash_cra_exit;
4614         alg->cra_ctxsize = sizeof(struct caam_hash_ctx);
4615         alg->cra_priority = CAAM_CRA_PRIORITY;
4616         alg->cra_blocksize = template->blocksize;
4617         alg->cra_alignmask = 0;
4618         alg->cra_flags = CRYPTO_ALG_ASYNC;
4619
4620         t_alg->alg_type = template->alg_type;
4621         t_alg->dev = dev;
4622
4623         return t_alg;
4624 }
4625
4626 static void dpaa2_caam_fqdan_cb(struct dpaa2_io_notification_ctx *nctx)
4627 {
4628         struct dpaa2_caam_priv_per_cpu *ppriv;
4629
4630         ppriv = container_of(nctx, struct dpaa2_caam_priv_per_cpu, nctx);
4631         napi_schedule_irqoff(&ppriv->napi);
4632 }
4633
4634 static int __cold dpaa2_dpseci_dpio_setup(struct dpaa2_caam_priv *priv)
4635 {
4636         struct device *dev = priv->dev;
4637         struct dpaa2_io_notification_ctx *nctx;
4638         struct dpaa2_caam_priv_per_cpu *ppriv;
4639         int err, i = 0, cpu;
4640
4641         for_each_online_cpu(cpu) {
4642                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
4643                 ppriv->priv = priv;
4644                 nctx = &ppriv->nctx;
4645                 nctx->is_cdan = 0;
4646                 nctx->id = ppriv->rsp_fqid;
4647                 nctx->desired_cpu = cpu;
4648                 nctx->cb = dpaa2_caam_fqdan_cb;
4649
4650                 /* Register notification callbacks */
4651                 ppriv->dpio = dpaa2_io_service_select(cpu);
4652                 err = dpaa2_io_service_register(ppriv->dpio, nctx, dev);
4653                 if (unlikely(err)) {
4654                         dev_dbg(dev, "No affine DPIO for cpu %d\n", cpu);
4655                         nctx->cb = NULL;
4656                         /*
4657                          * If no affine DPIO for this core, there's probably
4658                          * none available for next cores either. Signal we want
4659                          * to retry later, in case the DPIO devices weren't
4660                          * probed yet.
4661                          */
4662                         err = -EPROBE_DEFER;
4663                         goto err;
4664                 }
4665
4666                 ppriv->store = dpaa2_io_store_create(DPAA2_CAAM_STORE_SIZE,
4667                                                      dev);
4668                 if (unlikely(!ppriv->store)) {
4669                         dev_err(dev, "dpaa2_io_store_create() failed\n");
4670                         err = -ENOMEM;
4671                         goto err;
4672                 }
4673
4674                 if (++i == priv->num_pairs)
4675                         break;
4676         }
4677
4678         return 0;
4679
4680 err:
4681         for_each_online_cpu(cpu) {
4682                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
4683                 if (!ppriv->nctx.cb)
4684                         break;
4685                 dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx, dev);
4686         }
4687
4688         for_each_online_cpu(cpu) {
4689                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
4690                 if (!ppriv->store)
4691                         break;
4692                 dpaa2_io_store_destroy(ppriv->store);
4693         }
4694
4695         return err;
4696 }
4697
4698 static void __cold dpaa2_dpseci_dpio_free(struct dpaa2_caam_priv *priv)
4699 {
4700         struct dpaa2_caam_priv_per_cpu *ppriv;
4701         int i = 0, cpu;
4702
4703         for_each_online_cpu(cpu) {
4704                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
4705                 dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx,
4706                                             priv->dev);
4707                 dpaa2_io_store_destroy(ppriv->store);
4708
4709                 if (++i == priv->num_pairs)
4710                         return;
4711         }
4712 }
4713
4714 static int dpaa2_dpseci_bind(struct dpaa2_caam_priv *priv)
4715 {
4716         struct dpseci_rx_queue_cfg rx_queue_cfg;
4717         struct device *dev = priv->dev;
4718         struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
4719         struct dpaa2_caam_priv_per_cpu *ppriv;
4720         int err = 0, i = 0, cpu;
4721
4722         /* Configure Rx queues */
4723         for_each_online_cpu(cpu) {
4724                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
4725
4726                 rx_queue_cfg.options = DPSECI_QUEUE_OPT_DEST |
4727                                        DPSECI_QUEUE_OPT_USER_CTX;
4728                 rx_queue_cfg.order_preservation_en = 0;
4729                 rx_queue_cfg.dest_cfg.dest_type = DPSECI_DEST_DPIO;
4730                 rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id;
4731                 /*
4732                  * Rx priority (WQ) doesn't really matter, since we use
4733                  * pull mode, i.e. volatile dequeues from specific FQs
4734                  */
4735                 rx_queue_cfg.dest_cfg.priority = 0;
4736                 rx_queue_cfg.user_ctx = ppriv->nctx.qman64;
4737
4738                 err = dpseci_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
4739                                           &rx_queue_cfg);
4740                 if (err) {
4741                         dev_err(dev, "dpseci_set_rx_queue() failed with err %d\n",
4742                                 err);
4743                         return err;
4744                 }
4745
4746                 if (++i == priv->num_pairs)
4747                         break;
4748         }
4749
4750         return err;
4751 }
4752
4753 static void dpaa2_dpseci_congestion_free(struct dpaa2_caam_priv *priv)
4754 {
4755         struct device *dev = priv->dev;
4756
4757         if (!priv->cscn_mem)
4758                 return;
4759
4760         dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
4761         kfree(priv->cscn_mem);
4762 }
4763
4764 static void dpaa2_dpseci_free(struct dpaa2_caam_priv *priv)
4765 {
4766         struct device *dev = priv->dev;
4767         struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
4768
4769         dpaa2_dpseci_congestion_free(priv);
4770         dpseci_close(priv->mc_io, 0, ls_dev->mc_handle);
4771 }
4772
4773 static void dpaa2_caam_process_fd(struct dpaa2_caam_priv *priv,
4774                                   const struct dpaa2_fd *fd)
4775 {
4776         struct caam_request *req;
4777         u32 fd_err;
4778
4779         if (dpaa2_fd_get_format(fd) != dpaa2_fd_list) {
4780                 dev_err(priv->dev, "Only Frame List FD format is supported!\n");
4781                 return;
4782         }
4783
4784         fd_err = dpaa2_fd_get_ctrl(fd) & FD_CTRL_ERR_MASK;
4785         if (unlikely(fd_err))
4786                 dev_err_ratelimited(priv->dev, "FD error: %08x\n", fd_err);
4787
4788         /*
4789          * FD[ADDR] is guaranteed to be valid, irrespective of errors reported
4790          * in FD[ERR] or FD[FRC].
4791          */
4792         req = dpaa2_caam_iova_to_virt(priv, dpaa2_fd_get_addr(fd));
4793         dma_unmap_single(priv->dev, req->fd_flt_dma, sizeof(req->fd_flt),
4794                          DMA_BIDIRECTIONAL);
4795         req->cbk(req->ctx, dpaa2_fd_get_frc(fd));
4796 }
4797
4798 static int dpaa2_caam_pull_fq(struct dpaa2_caam_priv_per_cpu *ppriv)
4799 {
4800         int err;
4801
4802         /* Retry while portal is busy */
4803         do {
4804                 err = dpaa2_io_service_pull_fq(ppriv->dpio, ppriv->rsp_fqid,
4805                                                ppriv->store);
4806         } while (err == -EBUSY);
4807
4808         if (unlikely(err))
4809                 dev_err(ppriv->priv->dev, "dpaa2_io_service_pull err %d", err);
4810
4811         return err;
4812 }
4813
4814 static int dpaa2_caam_store_consume(struct dpaa2_caam_priv_per_cpu *ppriv)
4815 {
4816         struct dpaa2_dq *dq;
4817         int cleaned = 0, is_last;
4818
4819         do {
4820                 dq = dpaa2_io_store_next(ppriv->store, &is_last);
4821                 if (unlikely(!dq)) {
4822                         if (unlikely(!is_last)) {
4823                                 dev_dbg(ppriv->priv->dev,
4824                                         "FQ %d returned no valid frames\n",
4825                                         ppriv->rsp_fqid);
4826                                 /*
4827                                  * MUST retry until we get some sort of
4828                                  * valid response token (be it "empty dequeue"
4829                                  * or a valid frame).
4830                                  */
4831                                 continue;
4832                         }
4833                         break;
4834                 }
4835
4836                 /* Process FD */
4837                 dpaa2_caam_process_fd(ppriv->priv, dpaa2_dq_fd(dq));
4838                 cleaned++;
4839         } while (!is_last);
4840
4841         return cleaned;
4842 }
4843
4844 static int dpaa2_dpseci_poll(struct napi_struct *napi, int budget)
4845 {
4846         struct dpaa2_caam_priv_per_cpu *ppriv;
4847         struct dpaa2_caam_priv *priv;
4848         int err, cleaned = 0, store_cleaned;
4849
4850         ppriv = container_of(napi, struct dpaa2_caam_priv_per_cpu, napi);
4851         priv = ppriv->priv;
4852
4853         if (unlikely(dpaa2_caam_pull_fq(ppriv)))
4854                 return 0;
4855
4856         do {
4857                 store_cleaned = dpaa2_caam_store_consume(ppriv);
4858                 cleaned += store_cleaned;
4859
4860                 if (store_cleaned == 0 ||
4861                     cleaned > budget - DPAA2_CAAM_STORE_SIZE)
4862                         break;
4863
4864                 /* Try to dequeue some more */
4865                 err = dpaa2_caam_pull_fq(ppriv);
4866                 if (unlikely(err))
4867                         break;
4868         } while (1);
4869
4870         if (cleaned < budget) {
4871                 napi_complete_done(napi, cleaned);
4872                 err = dpaa2_io_service_rearm(ppriv->dpio, &ppriv->nctx);
4873                 if (unlikely(err))
4874                         dev_err(priv->dev, "Notification rearm failed: %d\n",
4875                                 err);
4876         }
4877
4878         return cleaned;
4879 }
4880
4881 static int dpaa2_dpseci_congestion_setup(struct dpaa2_caam_priv *priv,
4882                                          u16 token)
4883 {
4884         struct dpseci_congestion_notification_cfg cong_notif_cfg = { 0 };
4885         struct device *dev = priv->dev;
4886         int err;
4887
4888         /*
4889          * Congestion group feature supported starting with DPSECI API v5.1
4890          * and only when object has been created with this capability.
4891          */
4892         if ((DPSECI_VER(priv->major_ver, priv->minor_ver) < DPSECI_VER(5, 1)) ||
4893             !(priv->dpseci_attr.options & DPSECI_OPT_HAS_CG))
4894                 return 0;
4895
4896         priv->cscn_mem = kzalloc(DPAA2_CSCN_SIZE + DPAA2_CSCN_ALIGN,
4897                                  GFP_KERNEL | GFP_DMA);
4898         if (!priv->cscn_mem)
4899                 return -ENOMEM;
4900
4901         priv->cscn_mem_aligned = PTR_ALIGN(priv->cscn_mem, DPAA2_CSCN_ALIGN);
4902         priv->cscn_dma = dma_map_single(dev, priv->cscn_mem_aligned,
4903                                         DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
4904         if (dma_mapping_error(dev, priv->cscn_dma)) {
4905                 dev_err(dev, "Error mapping CSCN memory area\n");
4906                 err = -ENOMEM;
4907                 goto err_dma_map;
4908         }
4909
4910         cong_notif_cfg.units = DPSECI_CONGESTION_UNIT_BYTES;
4911         cong_notif_cfg.threshold_entry = DPAA2_SEC_CONG_ENTRY_THRESH;
4912         cong_notif_cfg.threshold_exit = DPAA2_SEC_CONG_EXIT_THRESH;
4913         cong_notif_cfg.message_ctx = (uintptr_t)priv;
4914         cong_notif_cfg.message_iova = priv->cscn_dma;
4915         cong_notif_cfg.notification_mode = DPSECI_CGN_MODE_WRITE_MEM_ON_ENTER |
4916                                         DPSECI_CGN_MODE_WRITE_MEM_ON_EXIT |
4917                                         DPSECI_CGN_MODE_COHERENT_WRITE;
4918
4919         err = dpseci_set_congestion_notification(priv->mc_io, 0, token,
4920                                                  &cong_notif_cfg);
4921         if (err) {
4922                 dev_err(dev, "dpseci_set_congestion_notification failed\n");
4923                 goto err_set_cong;
4924         }
4925
4926         return 0;
4927
4928 err_set_cong:
4929         dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
4930 err_dma_map:
4931         kfree(priv->cscn_mem);
4932
4933         return err;
4934 }
4935
4936 static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev)
4937 {
4938         struct device *dev = &ls_dev->dev;
4939         struct dpaa2_caam_priv *priv;
4940         struct dpaa2_caam_priv_per_cpu *ppriv;
4941         int err, cpu;
4942         u8 i;
4943
4944         priv = dev_get_drvdata(dev);
4945
4946         priv->dev = dev;
4947         priv->dpsec_id = ls_dev->obj_desc.id;
4948
4949         /* Get a handle for the DPSECI this interface is associate with */
4950         err = dpseci_open(priv->mc_io, 0, priv->dpsec_id, &ls_dev->mc_handle);
4951         if (err) {
4952                 dev_err(dev, "dpseci_open() failed: %d\n", err);
4953                 goto err_open;
4954         }
4955
4956         err = dpseci_get_api_version(priv->mc_io, 0, &priv->major_ver,
4957                                      &priv->minor_ver);
4958         if (err) {
4959                 dev_err(dev, "dpseci_get_api_version() failed\n");
4960                 goto err_get_vers;
4961         }
4962
4963         dev_info(dev, "dpseci v%d.%d\n", priv->major_ver, priv->minor_ver);
4964
4965         err = dpseci_get_attributes(priv->mc_io, 0, ls_dev->mc_handle,
4966                                     &priv->dpseci_attr);
4967         if (err) {
4968                 dev_err(dev, "dpseci_get_attributes() failed\n");
4969                 goto err_get_vers;
4970         }
4971
4972         err = dpseci_get_sec_attr(priv->mc_io, 0, ls_dev->mc_handle,
4973                                   &priv->sec_attr);
4974         if (err) {
4975                 dev_err(dev, "dpseci_get_sec_attr() failed\n");
4976                 goto err_get_vers;
4977         }
4978
4979         err = dpaa2_dpseci_congestion_setup(priv, ls_dev->mc_handle);
4980         if (err) {
4981                 dev_err(dev, "setup_congestion() failed\n");
4982                 goto err_get_vers;
4983         }
4984
4985         priv->num_pairs = min(priv->dpseci_attr.num_rx_queues,
4986                               priv->dpseci_attr.num_tx_queues);
4987         if (priv->num_pairs > num_online_cpus()) {
4988                 dev_warn(dev, "%d queues won't be used\n",
4989                          priv->num_pairs - num_online_cpus());
4990                 priv->num_pairs = num_online_cpus();
4991         }
4992
4993         for (i = 0; i < priv->dpseci_attr.num_rx_queues; i++) {
4994                 err = dpseci_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
4995                                           &priv->rx_queue_attr[i]);
4996                 if (err) {
4997                         dev_err(dev, "dpseci_get_rx_queue() failed\n");
4998                         goto err_get_rx_queue;
4999                 }
5000         }
5001
5002         for (i = 0; i < priv->dpseci_attr.num_tx_queues; i++) {
5003                 err = dpseci_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
5004                                           &priv->tx_queue_attr[i]);
5005                 if (err) {
5006                         dev_err(dev, "dpseci_get_tx_queue() failed\n");
5007                         goto err_get_rx_queue;
5008                 }
5009         }
5010
5011         i = 0;
5012         for_each_online_cpu(cpu) {
5013                 u8 j;
5014
5015                 j = i % priv->num_pairs;
5016
5017                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
5018                 ppriv->req_fqid = priv->tx_queue_attr[j].fqid;
5019
5020                 /*
5021                  * Allow all cores to enqueue, while only some of them
5022                  * will take part in dequeuing.
5023                  */
5024                 if (++i > priv->num_pairs)
5025                         continue;
5026
5027                 ppriv->rsp_fqid = priv->rx_queue_attr[j].fqid;
5028                 ppriv->prio = j;
5029
5030                 dev_dbg(dev, "pair %d: rx queue %d, tx queue %d\n", j,
5031                         priv->rx_queue_attr[j].fqid,
5032                         priv->tx_queue_attr[j].fqid);
5033
5034                 ppriv->net_dev.dev = *dev;
5035                 INIT_LIST_HEAD(&ppriv->net_dev.napi_list);
5036                 netif_napi_add(&ppriv->net_dev, &ppriv->napi, dpaa2_dpseci_poll,
5037                                DPAA2_CAAM_NAPI_WEIGHT);
5038         }
5039
5040         return 0;
5041
5042 err_get_rx_queue:
5043         dpaa2_dpseci_congestion_free(priv);
5044 err_get_vers:
5045         dpseci_close(priv->mc_io, 0, ls_dev->mc_handle);
5046 err_open:
5047         return err;
5048 }
5049
5050 static int dpaa2_dpseci_enable(struct dpaa2_caam_priv *priv)
5051 {
5052         struct device *dev = priv->dev;
5053         struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
5054         struct dpaa2_caam_priv_per_cpu *ppriv;
5055         int i;
5056
5057         for (i = 0; i < priv->num_pairs; i++) {
5058                 ppriv = per_cpu_ptr(priv->ppriv, i);
5059                 napi_enable(&ppriv->napi);
5060         }
5061
5062         return dpseci_enable(priv->mc_io, 0, ls_dev->mc_handle);
5063 }
5064
5065 static int __cold dpaa2_dpseci_disable(struct dpaa2_caam_priv *priv)
5066 {
5067         struct device *dev = priv->dev;
5068         struct dpaa2_caam_priv_per_cpu *ppriv;
5069         struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
5070         int i, err = 0, enabled;
5071
5072         err = dpseci_disable(priv->mc_io, 0, ls_dev->mc_handle);
5073         if (err) {
5074                 dev_err(dev, "dpseci_disable() failed\n");
5075                 return err;
5076         }
5077
5078         err = dpseci_is_enabled(priv->mc_io, 0, ls_dev->mc_handle, &enabled);
5079         if (err) {
5080                 dev_err(dev, "dpseci_is_enabled() failed\n");
5081                 return err;
5082         }
5083
5084         dev_dbg(dev, "disable: %s\n", enabled ? "false" : "true");
5085
5086         for (i = 0; i < priv->num_pairs; i++) {
5087                 ppriv = per_cpu_ptr(priv->ppriv, i);
5088                 napi_disable(&ppriv->napi);
5089                 netif_napi_del(&ppriv->napi);
5090         }
5091
5092         return 0;
5093 }
5094
5095 static struct list_head hash_list;
5096
5097 static int dpaa2_caam_probe(struct fsl_mc_device *dpseci_dev)
5098 {
5099         struct device *dev;
5100         struct dpaa2_caam_priv *priv;
5101         int i, err = 0;
5102         bool registered = false;
5103
5104         /*
5105          * There is no way to get CAAM endianness - there is no direct register
5106          * space access and MC f/w does not provide this attribute.
5107          * All DPAA2-based SoCs have little endian CAAM, thus hard-code this
5108          * property.
5109          */
5110         caam_little_end = true;
5111
5112         caam_imx = false;
5113
5114         dev = &dpseci_dev->dev;
5115
5116         priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
5117         if (!priv)
5118                 return -ENOMEM;
5119
5120         dev_set_drvdata(dev, priv);
5121
5122         priv->domain = iommu_get_domain_for_dev(dev);
5123
5124         qi_cache = kmem_cache_create("dpaa2_caamqicache", CAAM_QI_MEMCACHE_SIZE,
5125                                      0, SLAB_CACHE_DMA, NULL);
5126         if (!qi_cache) {
5127                 dev_err(dev, "Can't allocate SEC cache\n");
5128                 return -ENOMEM;
5129         }
5130
5131         err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(49));
5132         if (err) {
5133                 dev_err(dev, "dma_set_mask_and_coherent() failed\n");
5134                 goto err_dma_mask;
5135         }
5136
5137         /* Obtain a MC portal */
5138         err = fsl_mc_portal_allocate(dpseci_dev, 0, &priv->mc_io);
5139         if (err) {
5140                 if (err == -ENXIO)
5141                         err = -EPROBE_DEFER;
5142                 else
5143                         dev_err(dev, "MC portal allocation failed\n");
5144
5145                 goto err_dma_mask;
5146         }
5147
5148         priv->ppriv = alloc_percpu(*priv->ppriv);
5149         if (!priv->ppriv) {
5150                 dev_err(dev, "alloc_percpu() failed\n");
5151                 err = -ENOMEM;
5152                 goto err_alloc_ppriv;
5153         }
5154
5155         /* DPSECI initialization */
5156         err = dpaa2_dpseci_setup(dpseci_dev);
5157         if (err) {
5158                 dev_err(dev, "dpaa2_dpseci_setup() failed\n");
5159                 goto err_dpseci_setup;
5160         }
5161
5162         /* DPIO */
5163         err = dpaa2_dpseci_dpio_setup(priv);
5164         if (err) {
5165                 if (err != -EPROBE_DEFER)
5166                         dev_err(dev, "dpaa2_dpseci_dpio_setup() failed\n");
5167                 goto err_dpio_setup;
5168         }
5169
5170         /* DPSECI binding to DPIO */
5171         err = dpaa2_dpseci_bind(priv);
5172         if (err) {
5173                 dev_err(dev, "dpaa2_dpseci_bind() failed\n");
5174                 goto err_bind;
5175         }
5176
5177         /* DPSECI enable */
5178         err = dpaa2_dpseci_enable(priv);
5179         if (err) {
5180                 dev_err(dev, "dpaa2_dpseci_enable() failed\n");
5181                 goto err_bind;
5182         }
5183
5184         dpaa2_dpseci_debugfs_init(priv);
5185
5186         /* register crypto algorithms the device supports */
5187         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
5188                 struct caam_skcipher_alg *t_alg = driver_algs + i;
5189                 u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK;
5190
5191                 /* Skip DES algorithms if not supported by device */
5192                 if (!priv->sec_attr.des_acc_num &&
5193                     (alg_sel == OP_ALG_ALGSEL_3DES ||
5194                      alg_sel == OP_ALG_ALGSEL_DES))
5195                         continue;
5196
5197                 /* Skip AES algorithms if not supported by device */
5198                 if (!priv->sec_attr.aes_acc_num &&
5199                     alg_sel == OP_ALG_ALGSEL_AES)
5200                         continue;
5201
5202                 /* Skip CHACHA20 algorithms if not supported by device */
5203                 if (alg_sel == OP_ALG_ALGSEL_CHACHA20 &&
5204                     !priv->sec_attr.ccha_acc_num)
5205                         continue;
5206
5207                 t_alg->caam.dev = dev;
5208                 caam_skcipher_alg_init(t_alg);
5209
5210                 err = crypto_register_skcipher(&t_alg->skcipher);
5211                 if (err) {
5212                         dev_warn(dev, "%s alg registration failed: %d\n",
5213                                  t_alg->skcipher.base.cra_driver_name, err);
5214                         continue;
5215                 }
5216
5217                 t_alg->registered = true;
5218                 registered = true;
5219         }
5220
5221         for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
5222                 struct caam_aead_alg *t_alg = driver_aeads + i;
5223                 u32 c1_alg_sel = t_alg->caam.class1_alg_type &
5224                                  OP_ALG_ALGSEL_MASK;
5225                 u32 c2_alg_sel = t_alg->caam.class2_alg_type &
5226                                  OP_ALG_ALGSEL_MASK;
5227
5228                 /* Skip DES algorithms if not supported by device */
5229                 if (!priv->sec_attr.des_acc_num &&
5230                     (c1_alg_sel == OP_ALG_ALGSEL_3DES ||
5231                      c1_alg_sel == OP_ALG_ALGSEL_DES))
5232                         continue;
5233
5234                 /* Skip AES algorithms if not supported by device */
5235                 if (!priv->sec_attr.aes_acc_num &&
5236                     c1_alg_sel == OP_ALG_ALGSEL_AES)
5237                         continue;
5238
5239                 /* Skip CHACHA20 algorithms if not supported by device */
5240                 if (c1_alg_sel == OP_ALG_ALGSEL_CHACHA20 &&
5241                     !priv->sec_attr.ccha_acc_num)
5242                         continue;
5243
5244                 /* Skip POLY1305 algorithms if not supported by device */
5245                 if (c2_alg_sel == OP_ALG_ALGSEL_POLY1305 &&
5246                     !priv->sec_attr.ptha_acc_num)
5247                         continue;
5248
5249                 /*
5250                  * Skip algorithms requiring message digests
5251                  * if MD not supported by device.
5252                  */
5253                 if ((c2_alg_sel & ~OP_ALG_ALGSEL_SUBMASK) == 0x40 &&
5254                     !priv->sec_attr.md_acc_num)
5255                         continue;
5256
5257                 t_alg->caam.dev = dev;
5258                 caam_aead_alg_init(t_alg);
5259
5260                 err = crypto_register_aead(&t_alg->aead);
5261                 if (err) {
5262                         dev_warn(dev, "%s alg registration failed: %d\n",
5263                                  t_alg->aead.base.cra_driver_name, err);
5264                         continue;
5265                 }
5266
5267                 t_alg->registered = true;
5268                 registered = true;
5269         }
5270         if (registered)
5271                 dev_info(dev, "algorithms registered in /proc/crypto\n");
5272
5273         /* register hash algorithms the device supports */
5274         INIT_LIST_HEAD(&hash_list);
5275
5276         /*
5277          * Skip registration of any hashing algorithms if MD block
5278          * is not present.
5279          */
5280         if (!priv->sec_attr.md_acc_num)
5281                 return 0;
5282
5283         for (i = 0; i < ARRAY_SIZE(driver_hash); i++) {
5284                 struct caam_hash_alg *t_alg;
5285                 struct caam_hash_template *alg = driver_hash + i;
5286
5287                 /* register hmac version */
5288                 t_alg = caam_hash_alloc(dev, alg, true);
5289                 if (IS_ERR(t_alg)) {
5290                         err = PTR_ERR(t_alg);
5291                         dev_warn(dev, "%s hash alg allocation failed: %d\n",
5292                                  alg->driver_name, err);
5293                         continue;
5294                 }
5295
5296                 err = crypto_register_ahash(&t_alg->ahash_alg);
5297                 if (err) {
5298                         dev_warn(dev, "%s alg registration failed: %d\n",
5299                                  t_alg->ahash_alg.halg.base.cra_driver_name,
5300                                  err);
5301                         kfree(t_alg);
5302                 } else {
5303                         list_add_tail(&t_alg->entry, &hash_list);
5304                 }
5305
5306                 /* register unkeyed version */
5307                 t_alg = caam_hash_alloc(dev, alg, false);
5308                 if (IS_ERR(t_alg)) {
5309                         err = PTR_ERR(t_alg);
5310                         dev_warn(dev, "%s alg allocation failed: %d\n",
5311                                  alg->driver_name, err);
5312                         continue;
5313                 }
5314
5315                 err = crypto_register_ahash(&t_alg->ahash_alg);
5316                 if (err) {
5317                         dev_warn(dev, "%s alg registration failed: %d\n",
5318                                  t_alg->ahash_alg.halg.base.cra_driver_name,
5319                                  err);
5320                         kfree(t_alg);
5321                 } else {
5322                         list_add_tail(&t_alg->entry, &hash_list);
5323                 }
5324         }
5325         if (!list_empty(&hash_list))
5326                 dev_info(dev, "hash algorithms registered in /proc/crypto\n");
5327
5328         return err;
5329
5330 err_bind:
5331         dpaa2_dpseci_dpio_free(priv);
5332 err_dpio_setup:
5333         dpaa2_dpseci_free(priv);
5334 err_dpseci_setup:
5335         free_percpu(priv->ppriv);
5336 err_alloc_ppriv:
5337         fsl_mc_portal_free(priv->mc_io);
5338 err_dma_mask:
5339         kmem_cache_destroy(qi_cache);
5340
5341         return err;
5342 }
5343
5344 static int __cold dpaa2_caam_remove(struct fsl_mc_device *ls_dev)
5345 {
5346         struct device *dev;
5347         struct dpaa2_caam_priv *priv;
5348         int i;
5349
5350         dev = &ls_dev->dev;
5351         priv = dev_get_drvdata(dev);
5352
5353         dpaa2_dpseci_debugfs_exit(priv);
5354
5355         for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
5356                 struct caam_aead_alg *t_alg = driver_aeads + i;
5357
5358                 if (t_alg->registered)
5359                         crypto_unregister_aead(&t_alg->aead);
5360         }
5361
5362         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
5363                 struct caam_skcipher_alg *t_alg = driver_algs + i;
5364
5365                 if (t_alg->registered)
5366                         crypto_unregister_skcipher(&t_alg->skcipher);
5367         }
5368
5369         if (hash_list.next) {
5370                 struct caam_hash_alg *t_hash_alg, *p;
5371
5372                 list_for_each_entry_safe(t_hash_alg, p, &hash_list, entry) {
5373                         crypto_unregister_ahash(&t_hash_alg->ahash_alg);
5374                         list_del(&t_hash_alg->entry);
5375                         kfree(t_hash_alg);
5376                 }
5377         }
5378
5379         dpaa2_dpseci_disable(priv);
5380         dpaa2_dpseci_dpio_free(priv);
5381         dpaa2_dpseci_free(priv);
5382         free_percpu(priv->ppriv);
5383         fsl_mc_portal_free(priv->mc_io);
5384         kmem_cache_destroy(qi_cache);
5385
5386         return 0;
5387 }
5388
5389 int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req)
5390 {
5391         struct dpaa2_fd fd;
5392         struct dpaa2_caam_priv *priv = dev_get_drvdata(dev);
5393         struct dpaa2_caam_priv_per_cpu *ppriv;
5394         int err = 0, i;
5395
5396         if (IS_ERR(req))
5397                 return PTR_ERR(req);
5398
5399         if (priv->cscn_mem) {
5400                 dma_sync_single_for_cpu(priv->dev, priv->cscn_dma,
5401                                         DPAA2_CSCN_SIZE,
5402                                         DMA_FROM_DEVICE);
5403                 if (unlikely(dpaa2_cscn_state_congested(priv->cscn_mem_aligned))) {
5404                         dev_dbg_ratelimited(dev, "Dropping request\n");
5405                         return -EBUSY;
5406                 }
5407         }
5408
5409         dpaa2_fl_set_flc(&req->fd_flt[1], req->flc_dma);
5410
5411         req->fd_flt_dma = dma_map_single(dev, req->fd_flt, sizeof(req->fd_flt),
5412                                          DMA_BIDIRECTIONAL);
5413         if (dma_mapping_error(dev, req->fd_flt_dma)) {
5414                 dev_err(dev, "DMA mapping error for QI enqueue request\n");
5415                 goto err_out;
5416         }
5417
5418         memset(&fd, 0, sizeof(fd));
5419         dpaa2_fd_set_format(&fd, dpaa2_fd_list);
5420         dpaa2_fd_set_addr(&fd, req->fd_flt_dma);
5421         dpaa2_fd_set_len(&fd, dpaa2_fl_get_len(&req->fd_flt[1]));
5422         dpaa2_fd_set_flc(&fd, req->flc_dma);
5423
5424         ppriv = this_cpu_ptr(priv->ppriv);
5425         for (i = 0; i < (priv->dpseci_attr.num_tx_queues << 1); i++) {
5426                 err = dpaa2_io_service_enqueue_fq(ppriv->dpio, ppriv->req_fqid,
5427                                                   &fd);
5428                 if (err != -EBUSY)
5429                         break;
5430
5431                 cpu_relax();
5432         }
5433
5434         if (unlikely(err)) {
5435                 dev_err_ratelimited(dev, "Error enqueuing frame: %d\n", err);
5436                 goto err_out;
5437         }
5438
5439         return -EINPROGRESS;
5440
5441 err_out:
5442         dma_unmap_single(dev, req->fd_flt_dma, sizeof(req->fd_flt),
5443                          DMA_BIDIRECTIONAL);
5444         return -EIO;
5445 }
5446 EXPORT_SYMBOL(dpaa2_caam_enqueue);
5447
5448 static const struct fsl_mc_device_id dpaa2_caam_match_id_table[] = {
5449         {
5450                 .vendor = FSL_MC_VENDOR_FREESCALE,
5451                 .obj_type = "dpseci",
5452         },
5453         { .vendor = 0x0 }
5454 };
5455
5456 static struct fsl_mc_driver dpaa2_caam_driver = {
5457         .driver = {
5458                 .name           = KBUILD_MODNAME,
5459                 .owner          = THIS_MODULE,
5460         },
5461         .probe          = dpaa2_caam_probe,
5462         .remove         = dpaa2_caam_remove,
5463         .match_id_table = dpaa2_caam_match_id_table
5464 };
5465
5466 MODULE_LICENSE("Dual BSD/GPL");
5467 MODULE_AUTHOR("Freescale Semiconductor, Inc");
5468 MODULE_DESCRIPTION("Freescale DPAA2 CAAM Driver");
5469
5470 module_fsl_mc_driver(dpaa2_caam_driver);