drivers/crypto/caam/caamalg_qi2.c

   1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
   2 /*
   3  * Copyright 2015-2016 Freescale Semiconductor Inc.
   4  * Copyright 2017-2019 NXP
   5  */
   6
   7 #include "compat.h"
   8 #include "regs.h"
   9 #include "caamalg_qi2.h"
  10 #include "dpseci_cmd.h"
  11 #include "desc_constr.h"
  12 #include "error.h"
  13 #include "sg_sw_sec4.h"
  14 #include "sg_sw_qm2.h"
  15 #include "key_gen.h"
  16 #include "caamalg_desc.h"
  17 #include "caamhash_desc.h"
  18 #include "dpseci-debugfs.h"
  19 #include <linux/fsl/mc.h>
  20 #include <soc/fsl/dpaa2-io.h>
  21 #include <soc/fsl/dpaa2-fd.h>
  22
  23 #define CAAM_CRA_PRIORITY       2000
  24
  25 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */
  26 #define CAAM_MAX_KEY_SIZE       (AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE + \
  27                                  SHA512_DIGEST_SIZE * 2)
  28
  29 /*
  30  * This is a a cache of buffers, from which the users of CAAM QI driver
  31  * can allocate short buffers. It's speedier than doing kmalloc on the hotpath.
  32  * NOTE: A more elegant solution would be to have some headroom in the frames
  33  *       being processed. This can be added by the dpaa2-eth driver. This would
  34  *       pose a problem for userspace application processing which cannot
  35  *       know of this limitation. So for now, this will work.
  36  * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here
  37  */
  38 static struct kmem_cache *qi_cache;
  39
  40 struct caam_alg_entry {
  41         struct device *dev;
  42         int class1_alg_type;
  43         int class2_alg_type;
  44         bool rfc3686;
  45         bool geniv;
  46         bool nodkp;
  47 };
  48
  49 struct caam_aead_alg {
  50         struct aead_alg aead;
  51         struct caam_alg_entry caam;
  52         bool registered;
  53 };
  54
  55 struct caam_skcipher_alg {
  56         struct skcipher_alg skcipher;
  57         struct caam_alg_entry caam;
  58         bool registered;
  59 };
  60
  61 /**
  62  * caam_ctx - per-session context
  63  * @flc: Flow Contexts array
  64  * @key:  [authentication key], encryption key
  65  * @flc_dma: I/O virtual addresses of the Flow Contexts
  66  * @key_dma: I/O virtual address of the key
  67  * @dir: DMA direction for mapping key and Flow Contexts
  68  * @dev: dpseci device
  69  * @adata: authentication algorithm details
  70  * @cdata: encryption algorithm details
  71  * @authsize: authentication tag (a.k.a. ICV / MAC) size
  72  */
  73 struct caam_ctx {
  74         struct caam_flc flc[NUM_OP];
  75         u8 key[CAAM_MAX_KEY_SIZE];
  76         dma_addr_t flc_dma[NUM_OP];
  77         dma_addr_t key_dma;
  78         enum dma_data_direction dir;
  79         struct device *dev;
  80         struct alginfo adata;
  81         struct alginfo cdata;
  82         unsigned int authsize;
  83 };
  84
  85 static void *dpaa2_caam_iova_to_virt(struct dpaa2_caam_priv *priv,
  86                                      dma_addr_t iova_addr)
  87 {
  88         phys_addr_t phys_addr;
  89
  90         phys_addr = priv->domain ? iommu_iova_to_phys(priv->domain, iova_addr) :
  91                                    iova_addr;
  92
  93         return phys_to_virt(phys_addr);
  94 }
  95
  96 /*
  97  * qi_cache_zalloc - Allocate buffers from CAAM-QI cache
  98  *
  99  * Allocate data on the hotpath. Instead of using kzalloc, one can use the
 100  * services of the CAAM QI memory cache (backed by kmem_cache). The buffers
 101  * will have a size of CAAM_QI_MEMCACHE_SIZE, which should be sufficient for
 102  * hosting 16 SG entries.
 103  *
 104  * @flags - flags that would be used for the equivalent kmalloc(..) call
 105  *
 106  * Returns a pointer to a retrieved buffer on success or NULL on failure.
 107  */
 108 static inline void *qi_cache_zalloc(gfp_t flags)
 109 {
 110         return kmem_cache_zalloc(qi_cache, flags);
 111 }
 112
 113 /*
 114  * qi_cache_free - Frees buffers allocated from CAAM-QI cache
 115  *
 116  * @obj - buffer previously allocated by qi_cache_zalloc
 117  *
 118  * No checking is being done, the call is a passthrough call to
 119  * kmem_cache_free(...)
 120  */
 121 static inline void qi_cache_free(void *obj)
 122 {
 123         kmem_cache_free(qi_cache, obj);
 124 }
 125
 126 static struct caam_request *to_caam_req(struct crypto_async_request *areq)
 127 {
 128         switch (crypto_tfm_alg_type(areq->tfm)) {
 129         case CRYPTO_ALG_TYPE_SKCIPHER:
 130                 return skcipher_request_ctx(skcipher_request_cast(areq));
 131         case CRYPTO_ALG_TYPE_AEAD:
 132                 return aead_request_ctx(container_of(areq, struct aead_request,
 133                                                      base));
 134         case CRYPTO_ALG_TYPE_AHASH:
 135                 return ahash_request_ctx(ahash_request_cast(areq));
 136         default:
 137                 return ERR_PTR(-EINVAL);
 138         }
 139 }
 140
 141 static void caam_unmap(struct device *dev, struct scatterlist *src,
 142                        struct scatterlist *dst, int src_nents,
 143                        int dst_nents, dma_addr_t iv_dma, int ivsize,
 144                        enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma,
 145                        int qm_sg_bytes)
 146 {
 147         if (dst != src) {
 148                 if (src_nents)
 149                         dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
 150                 if (dst_nents)
 151                         dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
 152         } else {
 153                 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
 154         }
 155
 156         if (iv_dma)
 157                 dma_unmap_single(dev, iv_dma, ivsize, iv_dir);
 158
 159         if (qm_sg_bytes)
 160                 dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE);
 161 }
 162
 163 static int aead_set_sh_desc(struct crypto_aead *aead)
 164 {
 165         struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
 166                                                  typeof(*alg), aead);
 167         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 168         unsigned int ivsize = crypto_aead_ivsize(aead);
 169         struct device *dev = ctx->dev;
 170         struct dpaa2_caam_priv *priv = dev_get_drvdata(dev);
 171         struct caam_flc *flc;
 172         u32 *desc;
 173         u32 ctx1_iv_off = 0;
 174         u32 *nonce = NULL;
 175         unsigned int data_len[2];
 176         u32 inl_mask;
 177         const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
 178                                OP_ALG_AAI_CTR_MOD128);
 179         const bool is_rfc3686 = alg->caam.rfc3686;
 180
 181         if (!ctx->cdata.keylen || !ctx->authsize)
 182                 return 0;
 183
 184         /*
 185          * AES-CTR needs to load IV in CONTEXT1 reg
 186          * at an offset of 128bits (16bytes)
 187          * CONTEXT1[255:128] = IV
 188          */
 189         if (ctr_mode)
 190                 ctx1_iv_off = 16;
 191
 192         /*
 193          * RFC3686 specific:
 194          *      CONTEXT1[255:128] = {NONCE, IV, COUNTER}
 195          */
 196         if (is_rfc3686) {
 197                 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
 198                 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad +
 199                                 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE);
 200         }
 201
 202         /*
 203          * In case |user key| > |derived key|, using DKP<imm,imm> would result
 204          * in invalid opcodes (last bytes of user key) in the resulting
 205          * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key
 206          * addresses are needed.
 207          */
 208         ctx->adata.key_virt = ctx->key;
 209         ctx->adata.key_dma = ctx->key_dma;
 210
 211         ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
 212         ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
 213
 214         data_len[0] = ctx->adata.keylen_pad;
 215         data_len[1] = ctx->cdata.keylen;
 216
 217         /* aead_encrypt shared descriptor */
 218         if (desc_inline_query((alg->caam.geniv ? DESC_QI_AEAD_GIVENC_LEN :
 219                                                  DESC_QI_AEAD_ENC_LEN) +
 220                               (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
 221                               DESC_JOB_IO_LEN, data_len, &inl_mask,
 222                               ARRAY_SIZE(data_len)) < 0)
 223                 return -EINVAL;
 224
 225         ctx->adata.key_inline = !!(inl_mask & 1);
 226         ctx->cdata.key_inline = !!(inl_mask & 2);
 227
 228         flc = &ctx->flc[ENCRYPT];
 229         desc = flc->sh_desc;
 230
 231         if (alg->caam.geniv)
 232                 cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata,
 233                                           ivsize, ctx->authsize, is_rfc3686,
 234                                           nonce, ctx1_iv_off, true,
 235                                           priv->sec_attr.era);
 236         else
 237                 cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata,
 238                                        ivsize, ctx->authsize, is_rfc3686, nonce,
 239                                        ctx1_iv_off, true, priv->sec_attr.era);
 240
 241         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 242         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 243                                    sizeof(flc->flc) + desc_bytes(desc),
 244                                    ctx->dir);
 245
 246         /* aead_decrypt shared descriptor */
 247         if (desc_inline_query(DESC_QI_AEAD_DEC_LEN +
 248                               (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
 249                               DESC_JOB_IO_LEN, data_len, &inl_mask,
 250                               ARRAY_SIZE(data_len)) < 0)
 251                 return -EINVAL;
 252
 253         ctx->adata.key_inline = !!(inl_mask & 1);
 254         ctx->cdata.key_inline = !!(inl_mask & 2);
 255
 256         flc = &ctx->flc[DECRYPT];
 257         desc = flc->sh_desc;
 258         cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata,
 259                                ivsize, ctx->authsize, alg->caam.geniv,
 260                                is_rfc3686, nonce, ctx1_iv_off, true,
 261                                priv->sec_attr.era);
 262         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 263         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 264                                    sizeof(flc->flc) + desc_bytes(desc),
 265                                    ctx->dir);
 266
 267         return 0;
 268 }
 269
 270 static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
 271 {
 272         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 273
 274         ctx->authsize = authsize;
 275         aead_set_sh_desc(authenc);
 276
 277         return 0;
 278 }
 279
 280 static int aead_setkey(struct crypto_aead *aead, const u8 *key,
 281                        unsigned int keylen)
 282 {
 283         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 284         struct device *dev = ctx->dev;
 285         struct crypto_authenc_keys keys;
 286
 287         if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
 288                 goto badkey;
 289
 290         dev_dbg(dev, "keylen %d enckeylen %d authkeylen %d\n",
 291                 keys.authkeylen + keys.enckeylen, keys.enckeylen,
 292                 keys.authkeylen);
 293         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 294                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 295
 296         ctx->adata.keylen = keys.authkeylen;
 297         ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
 298                                               OP_ALG_ALGSEL_MASK);
 299
 300         if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE)
 301                 goto badkey;
 302
 303         memcpy(ctx->key, keys.authkey, keys.authkeylen);
 304         memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
 305         dma_sync_single_for_device(dev, ctx->key_dma, ctx->adata.keylen_pad +
 306                                    keys.enckeylen, ctx->dir);
 307         print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ",
 308                              DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
 309                              ctx->adata.keylen_pad + keys.enckeylen, 1);
 310
 311         ctx->cdata.keylen = keys.enckeylen;
 312
 313         memzero_explicit(&keys, sizeof(keys));
 314         return aead_set_sh_desc(aead);
 315 badkey:
 316         memzero_explicit(&keys, sizeof(keys));
 317         return -EINVAL;
 318 }
 319
 320 static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key,
 321                             unsigned int keylen)
 322 {
 323         struct crypto_authenc_keys keys;
 324         int err;
 325
 326         err = crypto_authenc_extractkeys(&keys, key, keylen);
 327         if (unlikely(err))
 328                 goto out;
 329
 330         err = -EINVAL;
 331         if (keys.enckeylen != DES3_EDE_KEY_SIZE)
 332                 goto out;
 333
 334         err = crypto_des3_ede_verify_key(crypto_aead_tfm(aead), keys.enckey) ?:
 335               aead_setkey(aead, key, keylen);
 336
 337 out:
 338         memzero_explicit(&keys, sizeof(keys));
 339         return err;
 340 }
 341
 342 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
 343                                            bool encrypt)
 344 {
 345         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 346         struct caam_request *req_ctx = aead_request_ctx(req);
 347         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
 348         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
 349         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 350         struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
 351                                                  typeof(*alg), aead);
 352         struct device *dev = ctx->dev;
 353         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
 354                       GFP_KERNEL : GFP_ATOMIC;
 355         int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
 356         int src_len, dst_len = 0;
 357         struct aead_edesc *edesc;
 358         dma_addr_t qm_sg_dma, iv_dma = 0;
 359         int ivsize = 0;
 360         unsigned int authsize = ctx->authsize;
 361         int qm_sg_index = 0, qm_sg_nents = 0, qm_sg_bytes;
 362         int in_len, out_len;
 363         struct dpaa2_sg_entry *sg_table;
 364
 365         /* allocate space for base edesc, link tables and IV */
 366         edesc = qi_cache_zalloc(GFP_DMA | flags);
 367         if (unlikely(!edesc)) {
 368                 dev_err(dev, "could not allocate extended descriptor\n");
 369                 return ERR_PTR(-ENOMEM);
 370         }
 371
 372         if (unlikely(req->dst != req->src)) {
 373                 src_len = req->assoclen + req->cryptlen;
 374                 dst_len = src_len + (encrypt ? authsize : (-authsize));
 375
 376                 src_nents = sg_nents_for_len(req->src, src_len);
 377                 if (unlikely(src_nents < 0)) {
 378                         dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
 379                                 src_len);
 380                         qi_cache_free(edesc);
 381                         return ERR_PTR(src_nents);
 382                 }
 383
 384                 dst_nents = sg_nents_for_len(req->dst, dst_len);
 385                 if (unlikely(dst_nents < 0)) {
 386                         dev_err(dev, "Insufficient bytes (%d) in dst S/G\n",
 387                                 dst_len);
 388                         qi_cache_free(edesc);
 389                         return ERR_PTR(dst_nents);
 390                 }
 391
 392                 if (src_nents) {
 393                         mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
 394                                                       DMA_TO_DEVICE);
 395                         if (unlikely(!mapped_src_nents)) {
 396                                 dev_err(dev, "unable to map source\n");
 397                                 qi_cache_free(edesc);
 398                                 return ERR_PTR(-ENOMEM);
 399                         }
 400                 } else {
 401                         mapped_src_nents = 0;
 402                 }
 403
 404                 if (dst_nents) {
 405                         mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents,
 406                                                       DMA_FROM_DEVICE);
 407                         if (unlikely(!mapped_dst_nents)) {
 408                                 dev_err(dev, "unable to map destination\n");
 409                                 dma_unmap_sg(dev, req->src, src_nents,
 410                                              DMA_TO_DEVICE);
 411                                 qi_cache_free(edesc);
 412                                 return ERR_PTR(-ENOMEM);
 413                         }
 414                 } else {
 415                         mapped_dst_nents = 0;
 416                 }
 417         } else {
 418                 src_len = req->assoclen + req->cryptlen +
 419                           (encrypt ? authsize : 0);
 420
 421                 src_nents = sg_nents_for_len(req->src, src_len);
 422                 if (unlikely(src_nents < 0)) {
 423                         dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
 424                                 src_len);
 425                         qi_cache_free(edesc);
 426                         return ERR_PTR(src_nents);
 427                 }
 428
 429                 mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
 430                                               DMA_BIDIRECTIONAL);
 431                 if (unlikely(!mapped_src_nents)) {
 432                         dev_err(dev, "unable to map source\n");
 433                         qi_cache_free(edesc);
 434                         return ERR_PTR(-ENOMEM);
 435                 }
 436         }
 437
 438         if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv)
 439                 ivsize = crypto_aead_ivsize(aead);
 440
 441         /*
 442          * Create S/G table: req->assoclen, [IV,] req->src [, req->dst].
 443          * Input is not contiguous.
 444          * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
 445          * the end of the table by allocating more S/G entries. Logic:
 446          * if (src != dst && output S/G)
 447          *      pad output S/G, if needed
 448          * else if (src == dst && S/G)
 449          *      overlapping S/Gs; pad one of them
 450          * else if (input S/G) ...
 451          *      pad input S/G, if needed
 452          */
 453         qm_sg_nents = 1 + !!ivsize + mapped_src_nents;
 454         if (mapped_dst_nents > 1)
 455                 qm_sg_nents += pad_sg_nents(mapped_dst_nents);
 456         else if ((req->src == req->dst) && (mapped_src_nents > 1))
 457                 qm_sg_nents = max(pad_sg_nents(qm_sg_nents),
 458                                   1 + !!ivsize +
 459                                   pad_sg_nents(mapped_src_nents));
 460         else
 461                 qm_sg_nents = pad_sg_nents(qm_sg_nents);
 462
 463         sg_table = &edesc->sgt[0];
 464         qm_sg_bytes = qm_sg_nents * sizeof(*sg_table);
 465         if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize >
 466                      CAAM_QI_MEMCACHE_SIZE)) {
 467                 dev_err(dev, "No space for %d S/G entries and/or %dB IV\n",
 468                         qm_sg_nents, ivsize);
 469                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
 470                            0, DMA_NONE, 0, 0);
 471                 qi_cache_free(edesc);
 472                 return ERR_PTR(-ENOMEM);
 473         }
 474
 475         if (ivsize) {
 476                 u8 *iv = (u8 *)(sg_table + qm_sg_nents);
 477
 478                 /* Make sure IV is located in a DMAable area */
 479                 memcpy(iv, req->iv, ivsize);
 480
 481                 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
 482                 if (dma_mapping_error(dev, iv_dma)) {
 483                         dev_err(dev, "unable to map IV\n");
 484                         caam_unmap(dev, req->src, req->dst, src_nents,
 485                                    dst_nents, 0, 0, DMA_NONE, 0, 0);
 486                         qi_cache_free(edesc);
 487                         return ERR_PTR(-ENOMEM);
 488                 }
 489         }
 490
 491         edesc->src_nents = src_nents;
 492         edesc->dst_nents = dst_nents;
 493         edesc->iv_dma = iv_dma;
 494
 495         if ((alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK) ==
 496             OP_ALG_ALGSEL_CHACHA20 && ivsize != CHACHAPOLY_IV_SIZE)
 497                 /*
 498                  * The associated data comes already with the IV but we need
 499                  * to skip it when we authenticate or encrypt...
 500                  */
 501                 edesc->assoclen = cpu_to_caam32(req->assoclen - ivsize);
 502         else
 503                 edesc->assoclen = cpu_to_caam32(req->assoclen);
 504         edesc->assoclen_dma = dma_map_single(dev, &edesc->assoclen, 4,
 505                                              DMA_TO_DEVICE);
 506         if (dma_mapping_error(dev, edesc->assoclen_dma)) {
 507                 dev_err(dev, "unable to map assoclen\n");
 508                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
 509                            iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
 510                 qi_cache_free(edesc);
 511                 return ERR_PTR(-ENOMEM);
 512         }
 513
 514         dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0);
 515         qm_sg_index++;
 516         if (ivsize) {
 517                 dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0);
 518                 qm_sg_index++;
 519         }
 520         sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0);
 521         qm_sg_index += mapped_src_nents;
 522
 523         if (mapped_dst_nents > 1)
 524                 sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0);
 525
 526         qm_sg_dma = dma_map_single(dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE);
 527         if (dma_mapping_error(dev, qm_sg_dma)) {
 528                 dev_err(dev, "unable to map S/G table\n");
 529                 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
 530                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
 531                            iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
 532                 qi_cache_free(edesc);
 533                 return ERR_PTR(-ENOMEM);
 534         }
 535
 536         edesc->qm_sg_dma = qm_sg_dma;
 537         edesc->qm_sg_bytes = qm_sg_bytes;
 538
 539         out_len = req->assoclen + req->cryptlen +
 540                   (encrypt ? ctx->authsize : (-ctx->authsize));
 541         in_len = 4 + ivsize + req->assoclen + req->cryptlen;
 542
 543         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
 544         dpaa2_fl_set_final(in_fle, true);
 545         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
 546         dpaa2_fl_set_addr(in_fle, qm_sg_dma);
 547         dpaa2_fl_set_len(in_fle, in_len);
 548
 549         if (req->dst == req->src) {
 550                 if (mapped_src_nents == 1) {
 551                         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
 552                         dpaa2_fl_set_addr(out_fle, sg_dma_address(req->src));
 553                 } else {
 554                         dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
 555                         dpaa2_fl_set_addr(out_fle, qm_sg_dma +
 556                                           (1 + !!ivsize) * sizeof(*sg_table));
 557                 }
 558         } else if (!mapped_dst_nents) {
 559                 /*
 560                  * crypto engine requires the output entry to be present when
 561                  * "frame list" FD is used.
 562                  * Since engine does not support FMT=2'b11 (unused entry type),
 563                  * leaving out_fle zeroized is the best option.
 564                  */
 565                 goto skip_out_fle;
 566         } else if (mapped_dst_nents == 1) {
 567                 dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
 568                 dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst));
 569         } else {
 570                 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
 571                 dpaa2_fl_set_addr(out_fle, qm_sg_dma + qm_sg_index *
 572                                   sizeof(*sg_table));
 573         }
 574
 575         dpaa2_fl_set_len(out_fle, out_len);
 576
 577 skip_out_fle:
 578         return edesc;
 579 }
 580
 581 static int chachapoly_set_sh_desc(struct crypto_aead *aead)
 582 {
 583         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 584         unsigned int ivsize = crypto_aead_ivsize(aead);
 585         struct device *dev = ctx->dev;
 586         struct caam_flc *flc;
 587         u32 *desc;
 588
 589         if (!ctx->cdata.keylen || !ctx->authsize)
 590                 return 0;
 591
 592         flc = &ctx->flc[ENCRYPT];
 593         desc = flc->sh_desc;
 594         cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize,
 595                                ctx->authsize, true, true);
 596         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 597         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 598                                    sizeof(flc->flc) + desc_bytes(desc),
 599                                    ctx->dir);
 600
 601         flc = &ctx->flc[DECRYPT];
 602         desc = flc->sh_desc;
 603         cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize,
 604                                ctx->authsize, false, true);
 605         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 606         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 607                                    sizeof(flc->flc) + desc_bytes(desc),
 608                                    ctx->dir);
 609
 610         return 0;
 611 }
 612
 613 static int chachapoly_setauthsize(struct crypto_aead *aead,
 614                                   unsigned int authsize)
 615 {
 616         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 617
 618         if (authsize != POLY1305_DIGEST_SIZE)
 619                 return -EINVAL;
 620
 621         ctx->authsize = authsize;
 622         return chachapoly_set_sh_desc(aead);
 623 }
 624
 625 static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key,
 626                              unsigned int keylen)
 627 {
 628         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 629         unsigned int ivsize = crypto_aead_ivsize(aead);
 630         unsigned int saltlen = CHACHAPOLY_IV_SIZE - ivsize;
 631
 632         if (keylen != CHACHA_KEY_SIZE + saltlen)
 633                 return -EINVAL;
 634
 635         ctx->cdata.key_virt = key;
 636         ctx->cdata.keylen = keylen - saltlen;
 637
 638         return chachapoly_set_sh_desc(aead);
 639 }
 640
 641 static int gcm_set_sh_desc(struct crypto_aead *aead)
 642 {
 643         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 644         struct device *dev = ctx->dev;
 645         unsigned int ivsize = crypto_aead_ivsize(aead);
 646         struct caam_flc *flc;
 647         u32 *desc;
 648         int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
 649                         ctx->cdata.keylen;
 650
 651         if (!ctx->cdata.keylen || !ctx->authsize)
 652                 return 0;
 653
 654         /*
 655          * AES GCM encrypt shared descriptor
 656          * Job Descriptor and Shared Descriptor
 657          * must fit into the 64-word Descriptor h/w Buffer
 658          */
 659         if (rem_bytes >= DESC_QI_GCM_ENC_LEN) {
 660                 ctx->cdata.key_inline = true;
 661                 ctx->cdata.key_virt = ctx->key;
 662         } else {
 663                 ctx->cdata.key_inline = false;
 664                 ctx->cdata.key_dma = ctx->key_dma;
 665         }
 666
 667         flc = &ctx->flc[ENCRYPT];
 668         desc = flc->sh_desc;
 669         cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, true);
 670         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 671         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 672                                    sizeof(flc->flc) + desc_bytes(desc),
 673                                    ctx->dir);
 674
 675         /*
 676          * Job Descriptor and Shared Descriptors
 677          * must all fit into the 64-word Descriptor h/w Buffer
 678          */
 679         if (rem_bytes >= DESC_QI_GCM_DEC_LEN) {
 680                 ctx->cdata.key_inline = true;
 681                 ctx->cdata.key_virt = ctx->key;
 682         } else {
 683                 ctx->cdata.key_inline = false;
 684                 ctx->cdata.key_dma = ctx->key_dma;
 685         }
 686
 687         flc = &ctx->flc[DECRYPT];
 688         desc = flc->sh_desc;
 689         cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, true);
 690         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 691         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 692                                    sizeof(flc->flc) + desc_bytes(desc),
 693                                    ctx->dir);
 694
 695         return 0;
 696 }
 697
 698 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
 699 {
 700         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 701         int err;
 702
 703         err = crypto_gcm_check_authsize(authsize);
 704         if (err)
 705                 return err;
 706
 707         ctx->authsize = authsize;
 708         gcm_set_sh_desc(authenc);
 709
 710         return 0;
 711 }
 712
 713 static int gcm_setkey(struct crypto_aead *aead,
 714                       const u8 *key, unsigned int keylen)
 715 {
 716         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 717         struct device *dev = ctx->dev;
 718         int ret;
 719
 720         ret = aes_check_keylen(keylen);
 721         if (ret)
 722                 return ret;
 723         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 724                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 725
 726         memcpy(ctx->key, key, keylen);
 727         dma_sync_single_for_device(dev, ctx->key_dma, keylen, ctx->dir);
 728         ctx->cdata.keylen = keylen;
 729
 730         return gcm_set_sh_desc(aead);
 731 }
 732
 733 static int rfc4106_set_sh_desc(struct crypto_aead *aead)
 734 {
 735         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 736         struct device *dev = ctx->dev;
 737         unsigned int ivsize = crypto_aead_ivsize(aead);
 738         struct caam_flc *flc;
 739         u32 *desc;
 740         int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
 741                         ctx->cdata.keylen;
 742
 743         if (!ctx->cdata.keylen || !ctx->authsize)
 744                 return 0;
 745
 746         ctx->cdata.key_virt = ctx->key;
 747
 748         /*
 749          * RFC4106 encrypt shared descriptor
 750          * Job Descriptor and Shared Descriptor
 751          * must fit into the 64-word Descriptor h/w Buffer
 752          */
 753         if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) {
 754                 ctx->cdata.key_inline = true;
 755         } else {
 756                 ctx->cdata.key_inline = false;
 757                 ctx->cdata.key_dma = ctx->key_dma;
 758         }
 759
 760         flc = &ctx->flc[ENCRYPT];
 761         desc = flc->sh_desc;
 762         cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
 763                                   true);
 764         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 765         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 766                                    sizeof(flc->flc) + desc_bytes(desc),
 767                                    ctx->dir);
 768
 769         /*
 770          * Job Descriptor and Shared Descriptors
 771          * must all fit into the 64-word Descriptor h/w Buffer
 772          */
 773         if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) {
 774                 ctx->cdata.key_inline = true;
 775         } else {
 776                 ctx->cdata.key_inline = false;
 777                 ctx->cdata.key_dma = ctx->key_dma;
 778         }
 779
 780         flc = &ctx->flc[DECRYPT];
 781         desc = flc->sh_desc;
 782         cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
 783                                   true);
 784         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 785         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 786                                    sizeof(flc->flc) + desc_bytes(desc),
 787                                    ctx->dir);
 788
 789         return 0;
 790 }
 791
 792 static int rfc4106_setauthsize(struct crypto_aead *authenc,
 793                                unsigned int authsize)
 794 {
 795         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 796         int err;
 797
 798         err = crypto_rfc4106_check_authsize(authsize);
 799         if (err)
 800                 return err;
 801
 802         ctx->authsize = authsize;
 803         rfc4106_set_sh_desc(authenc);
 804
 805         return 0;
 806 }
 807
 808 static int rfc4106_setkey(struct crypto_aead *aead,
 809                           const u8 *key, unsigned int keylen)
 810 {
 811         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 812         struct device *dev = ctx->dev;
 813         int ret;
 814
 815         ret = aes_check_keylen(keylen - 4);
 816         if (ret)
 817                 return ret;
 818
 819         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 820                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 821
 822         memcpy(ctx->key, key, keylen);
 823         /*
 824          * The last four bytes of the key material are used as the salt value
 825          * in the nonce. Update the AES key length.
 826          */
 827         ctx->cdata.keylen = keylen - 4;
 828         dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen,
 829                                    ctx->dir);
 830
 831         return rfc4106_set_sh_desc(aead);
 832 }
 833
 834 static int rfc4543_set_sh_desc(struct crypto_aead *aead)
 835 {
 836         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 837         struct device *dev = ctx->dev;
 838         unsigned int ivsize = crypto_aead_ivsize(aead);
 839         struct caam_flc *flc;
 840         u32 *desc;
 841         int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
 842                         ctx->cdata.keylen;
 843
 844         if (!ctx->cdata.keylen || !ctx->authsize)
 845                 return 0;
 846
 847         ctx->cdata.key_virt = ctx->key;
 848
 849         /*
 850          * RFC4543 encrypt shared descriptor
 851          * Job Descriptor and Shared Descriptor
 852          * must fit into the 64-word Descriptor h/w Buffer
 853          */
 854         if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) {
 855                 ctx->cdata.key_inline = true;
 856         } else {
 857                 ctx->cdata.key_inline = false;
 858                 ctx->cdata.key_dma = ctx->key_dma;
 859         }
 860
 861         flc = &ctx->flc[ENCRYPT];
 862         desc = flc->sh_desc;
 863         cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
 864                                   true);
 865         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 866         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 867                                    sizeof(flc->flc) + desc_bytes(desc),
 868                                    ctx->dir);
 869
 870         /*
 871          * Job Descriptor and Shared Descriptors
 872          * must all fit into the 64-word Descriptor h/w Buffer
 873          */
 874         if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) {
 875                 ctx->cdata.key_inline = true;
 876         } else {
 877                 ctx->cdata.key_inline = false;
 878                 ctx->cdata.key_dma = ctx->key_dma;
 879         }
 880
 881         flc = &ctx->flc[DECRYPT];
 882         desc = flc->sh_desc;
 883         cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
 884                                   true);
 885         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 886         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 887                                    sizeof(flc->flc) + desc_bytes(desc),
 888                                    ctx->dir);
 889
 890         return 0;
 891 }
 892
 893 static int rfc4543_setauthsize(struct crypto_aead *authenc,
 894                                unsigned int authsize)
 895 {
 896         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 897
 898         if (authsize != 16)
 899                 return -EINVAL;
 900
 901         ctx->authsize = authsize;
 902         rfc4543_set_sh_desc(authenc);
 903
 904         return 0;
 905 }
 906
 907 static int rfc4543_setkey(struct crypto_aead *aead,
 908                           const u8 *key, unsigned int keylen)
 909 {
 910         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 911         struct device *dev = ctx->dev;
 912         int ret;
 913
 914         ret = aes_check_keylen(keylen - 4);
 915         if (ret)
 916                 return ret;
 917
 918         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 919                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 920
 921         memcpy(ctx->key, key, keylen);
 922         /*
 923          * The last four bytes of the key material are used as the salt value
 924          * in the nonce. Update the AES key length.
 925          */
 926         ctx->cdata.keylen = keylen - 4;
 927         dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen,
 928                                    ctx->dir);
 929
 930         return rfc4543_set_sh_desc(aead);
 931 }
 932
 933 static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
 934                            unsigned int keylen, const u32 ctx1_iv_off)
 935 {
 936         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
 937         struct caam_skcipher_alg *alg =
 938                 container_of(crypto_skcipher_alg(skcipher),
 939                              struct caam_skcipher_alg, skcipher);
 940         struct device *dev = ctx->dev;
 941         struct caam_flc *flc;
 942         unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
 943         u32 *desc;
 944         const bool is_rfc3686 = alg->caam.rfc3686;
 945
 946         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 947                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 948
 949         ctx->cdata.keylen = keylen;
 950         ctx->cdata.key_virt = key;
 951         ctx->cdata.key_inline = true;
 952
 953         /* skcipher_encrypt shared descriptor */
 954         flc = &ctx->flc[ENCRYPT];
 955         desc = flc->sh_desc;
 956         cnstr_shdsc_skcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686,
 957                                    ctx1_iv_off);
 958         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 959         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 960                                    sizeof(flc->flc) + desc_bytes(desc),
 961                                    ctx->dir);
 962
 963         /* skcipher_decrypt shared descriptor */
 964         flc = &ctx->flc[DECRYPT];
 965         desc = flc->sh_desc;
 966         cnstr_shdsc_skcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686,
 967                                    ctx1_iv_off);
 968         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 969         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 970                                    sizeof(flc->flc) + desc_bytes(desc),
 971                                    ctx->dir);
 972
 973         return 0;
 974 }
 975
 976 static int aes_skcipher_setkey(struct crypto_skcipher *skcipher,
 977                                const u8 *key, unsigned int keylen)
 978 {
 979         int err;
 980
 981         err = aes_check_keylen(keylen);
 982         if (err)
 983                 return err;
 984
 985         return skcipher_setkey(skcipher, key, keylen, 0);
 986 }
 987
 988 static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher,
 989                                    const u8 *key, unsigned int keylen)
 990 {
 991         u32 ctx1_iv_off;
 992         int err;
 993
 994         /*
 995          * RFC3686 specific:
 996          *      | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
 997          *      | *key = {KEY, NONCE}
 998          */
 999         ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
1000         keylen -= CTR_RFC3686_NONCE_SIZE;
1001
1002         err = aes_check_keylen(keylen);
1003         if (err)
1004                 return err;
1005
1006         return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
1007 }
1008
1009 static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher,
1010                                const u8 *key, unsigned int keylen)
1011 {
1012         u32 ctx1_iv_off;
1013         int err;
1014
1015         /*
1016          * AES-CTR needs to load IV in CONTEXT1 reg
1017          * at an offset of 128bits (16bytes)
1018          * CONTEXT1[255:128] = IV
1019          */
1020         ctx1_iv_off = 16;
1021
1022         err = aes_check_keylen(keylen);
1023         if (err)
1024                 return err;
1025
1026         return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
1027 }
1028
1029 static int chacha20_skcipher_setkey(struct crypto_skcipher *skcipher,
1030                                     const u8 *key, unsigned int keylen)
1031 {
1032         if (keylen != CHACHA_KEY_SIZE)
1033                 return -EINVAL;
1034
1035         return skcipher_setkey(skcipher, key, keylen, 0);
1036 }
1037
1038 static int des_skcipher_setkey(struct crypto_skcipher *skcipher,
1039                                const u8 *key, unsigned int keylen)
1040 {
1041         return verify_skcipher_des_key(skcipher, key) ?:
1042                skcipher_setkey(skcipher, key, keylen, 0);
1043 }
1044
1045 static int des3_skcipher_setkey(struct crypto_skcipher *skcipher,
1046                                 const u8 *key, unsigned int keylen)
1047 {
1048         return verify_skcipher_des3_key(skcipher, key) ?:
1049                skcipher_setkey(skcipher, key, keylen, 0);
1050 }
1051
1052 static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
1053                                unsigned int keylen)
1054 {
1055         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1056         struct device *dev = ctx->dev;
1057         struct caam_flc *flc;
1058         u32 *desc;
1059
1060         if (keylen != 2 * AES_MIN_KEY_SIZE  && keylen != 2 * AES_MAX_KEY_SIZE) {
1061                 dev_err(dev, "key size mismatch\n");
1062                 return -EINVAL;
1063         }
1064
1065         ctx->cdata.keylen = keylen;
1066         ctx->cdata.key_virt = key;
1067         ctx->cdata.key_inline = true;
1068
1069         /* xts_skcipher_encrypt shared descriptor */
1070         flc = &ctx->flc[ENCRYPT];
1071         desc = flc->sh_desc;
1072         cnstr_shdsc_xts_skcipher_encap(desc, &ctx->cdata);
1073         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
1074         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
1075                                    sizeof(flc->flc) + desc_bytes(desc),
1076                                    ctx->dir);
1077
1078         /* xts_skcipher_decrypt shared descriptor */
1079         flc = &ctx->flc[DECRYPT];
1080         desc = flc->sh_desc;
1081         cnstr_shdsc_xts_skcipher_decap(desc, &ctx->cdata);
1082         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
1083         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
1084                                    sizeof(flc->flc) + desc_bytes(desc),
1085                                    ctx->dir);
1086
1087         return 0;
1088 }
1089
1090 static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req)
1091 {
1092         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1093         struct caam_request *req_ctx = skcipher_request_ctx(req);
1094         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
1095         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
1096         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1097         struct device *dev = ctx->dev;
1098         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1099                        GFP_KERNEL : GFP_ATOMIC;
1100         int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
1101         struct skcipher_edesc *edesc;
1102         dma_addr_t iv_dma;
1103         u8 *iv;
1104         int ivsize = crypto_skcipher_ivsize(skcipher);
1105         int dst_sg_idx, qm_sg_ents, qm_sg_bytes;
1106         struct dpaa2_sg_entry *sg_table;
1107
1108         src_nents = sg_nents_for_len(req->src, req->cryptlen);
1109         if (unlikely(src_nents < 0)) {
1110                 dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
1111                         req->cryptlen);
1112                 return ERR_PTR(src_nents);
1113         }
1114
1115         if (unlikely(req->dst != req->src)) {
1116                 dst_nents = sg_nents_for_len(req->dst, req->cryptlen);
1117                 if (unlikely(dst_nents < 0)) {
1118                         dev_err(dev, "Insufficient bytes (%d) in dst S/G\n",
1119                                 req->cryptlen);
1120                         return ERR_PTR(dst_nents);
1121                 }
1122
1123                 mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
1124                                               DMA_TO_DEVICE);
1125                 if (unlikely(!mapped_src_nents)) {
1126                         dev_err(dev, "unable to map source\n");
1127                         return ERR_PTR(-ENOMEM);
1128                 }
1129
1130                 mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents,
1131                                               DMA_FROM_DEVICE);
1132                 if (unlikely(!mapped_dst_nents)) {
1133                         dev_err(dev, "unable to map destination\n");
1134                         dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE);
1135                         return ERR_PTR(-ENOMEM);
1136                 }
1137         } else {
1138                 mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
1139                                               DMA_BIDIRECTIONAL);
1140                 if (unlikely(!mapped_src_nents)) {
1141                         dev_err(dev, "unable to map source\n");
1142                         return ERR_PTR(-ENOMEM);
1143                 }
1144         }
1145
1146         qm_sg_ents = 1 + mapped_src_nents;
1147         dst_sg_idx = qm_sg_ents;
1148
1149         /*
1150          * Input, output HW S/G tables: [IV, src][dst, IV]
1151          * IV entries point to the same buffer
1152          * If src == dst, S/G entries are reused (S/G tables overlap)
1153          *
1154          * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
1155          * the end of the table by allocating more S/G entries.
1156          */
1157         if (req->src != req->dst)
1158                 qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1);
1159         else
1160                 qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents);
1161
1162         qm_sg_bytes = qm_sg_ents * sizeof(struct dpaa2_sg_entry);
1163         if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes +
1164                      ivsize > CAAM_QI_MEMCACHE_SIZE)) {
1165                 dev_err(dev, "No space for %d S/G entries and/or %dB IV\n",
1166                         qm_sg_ents, ivsize);
1167                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
1168                            0, DMA_NONE, 0, 0);
1169                 return ERR_PTR(-ENOMEM);
1170         }
1171
1172         /* allocate space for base edesc, link tables and IV */
1173         edesc = qi_cache_zalloc(GFP_DMA | flags);
1174         if (unlikely(!edesc)) {
1175                 dev_err(dev, "could not allocate extended descriptor\n");
1176                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
1177                            0, DMA_NONE, 0, 0);
1178                 return ERR_PTR(-ENOMEM);
1179         }
1180
1181         /* Make sure IV is located in a DMAable area */
1182         sg_table = &edesc->sgt[0];
1183         iv = (u8 *)(sg_table + qm_sg_ents);
1184         memcpy(iv, req->iv, ivsize);
1185
1186         iv_dma = dma_map_single(dev, iv, ivsize, DMA_BIDIRECTIONAL);
1187         if (dma_mapping_error(dev, iv_dma)) {
1188                 dev_err(dev, "unable to map IV\n");
1189                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
1190                            0, DMA_NONE, 0, 0);
1191                 qi_cache_free(edesc);
1192                 return ERR_PTR(-ENOMEM);
1193         }
1194
1195         edesc->src_nents = src_nents;
1196         edesc->dst_nents = dst_nents;
1197         edesc->iv_dma = iv_dma;
1198         edesc->qm_sg_bytes = qm_sg_bytes;
1199
1200         dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0);
1201         sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0);
1202
1203         if (req->src != req->dst)
1204                 sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0);
1205
1206         dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma,
1207                          ivsize, 0);
1208
1209         edesc->qm_sg_dma = dma_map_single(dev, sg_table, edesc->qm_sg_bytes,
1210                                           DMA_TO_DEVICE);
1211         if (dma_mapping_error(dev, edesc->qm_sg_dma)) {
1212                 dev_err(dev, "unable to map S/G table\n");
1213                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
1214                            iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0);
1215                 qi_cache_free(edesc);
1216                 return ERR_PTR(-ENOMEM);
1217         }
1218
1219         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
1220         dpaa2_fl_set_final(in_fle, true);
1221         dpaa2_fl_set_len(in_fle, req->cryptlen + ivsize);
1222         dpaa2_fl_set_len(out_fle, req->cryptlen + ivsize);
1223
1224         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
1225         dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
1226
1227         dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
1228
1229         if (req->src == req->dst)
1230                 dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma +
1231                                   sizeof(*sg_table));
1232         else
1233                 dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + dst_sg_idx *
1234                                   sizeof(*sg_table));
1235
1236         return edesc;
1237 }
1238
1239 static void aead_unmap(struct device *dev, struct aead_edesc *edesc,
1240                        struct aead_request *req)
1241 {
1242         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1243         int ivsize = crypto_aead_ivsize(aead);
1244
1245         caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
1246                    edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma,
1247                    edesc->qm_sg_bytes);
1248         dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
1249 }
1250
1251 static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc,
1252                            struct skcipher_request *req)
1253 {
1254         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1255         int ivsize = crypto_skcipher_ivsize(skcipher);
1256
1257         caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
1258                    edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma,
1259                    edesc->qm_sg_bytes);
1260 }
1261
1262 static void aead_encrypt_done(void *cbk_ctx, u32 status)
1263 {
1264         struct crypto_async_request *areq = cbk_ctx;
1265         struct aead_request *req = container_of(areq, struct aead_request,
1266                                                 base);
1267         struct caam_request *req_ctx = to_caam_req(areq);
1268         struct aead_edesc *edesc = req_ctx->edesc;
1269         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1270         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1271         int ecode = 0;
1272
1273         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1274
1275         if (unlikely(status))
1276                 ecode = caam_qi2_strstatus(ctx->dev, status);
1277
1278         aead_unmap(ctx->dev, edesc, req);
1279         qi_cache_free(edesc);
1280         aead_request_complete(req, ecode);
1281 }
1282
1283 static void aead_decrypt_done(void *cbk_ctx, u32 status)
1284 {
1285         struct crypto_async_request *areq = cbk_ctx;
1286         struct aead_request *req = container_of(areq, struct aead_request,
1287                                                 base);
1288         struct caam_request *req_ctx = to_caam_req(areq);
1289         struct aead_edesc *edesc = req_ctx->edesc;
1290         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1291         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1292         int ecode = 0;
1293
1294         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1295
1296         if (unlikely(status))
1297                 ecode = caam_qi2_strstatus(ctx->dev, status);
1298
1299         aead_unmap(ctx->dev, edesc, req);
1300         qi_cache_free(edesc);
1301         aead_request_complete(req, ecode);
1302 }
1303
1304 static int aead_encrypt(struct aead_request *req)
1305 {
1306         struct aead_edesc *edesc;
1307         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1308         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1309         struct caam_request *caam_req = aead_request_ctx(req);
1310         int ret;
1311
1312         /* allocate extended descriptor */
1313         edesc = aead_edesc_alloc(req, true);
1314         if (IS_ERR(edesc))
1315                 return PTR_ERR(edesc);
1316
1317         caam_req->flc = &ctx->flc[ENCRYPT];
1318         caam_req->flc_dma = ctx->flc_dma[ENCRYPT];
1319         caam_req->cbk = aead_encrypt_done;
1320         caam_req->ctx = &req->base;
1321         caam_req->edesc = edesc;
1322         ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1323         if (ret != -EINPROGRESS &&
1324             !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1325                 aead_unmap(ctx->dev, edesc, req);
1326                 qi_cache_free(edesc);
1327         }
1328
1329         return ret;
1330 }
1331
1332 static int aead_decrypt(struct aead_request *req)
1333 {
1334         struct aead_edesc *edesc;
1335         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1336         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1337         struct caam_request *caam_req = aead_request_ctx(req);
1338         int ret;
1339
1340         /* allocate extended descriptor */
1341         edesc = aead_edesc_alloc(req, false);
1342         if (IS_ERR(edesc))
1343                 return PTR_ERR(edesc);
1344
1345         caam_req->flc = &ctx->flc[DECRYPT];
1346         caam_req->flc_dma = ctx->flc_dma[DECRYPT];
1347         caam_req->cbk = aead_decrypt_done;
1348         caam_req->ctx = &req->base;
1349         caam_req->edesc = edesc;
1350         ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1351         if (ret != -EINPROGRESS &&
1352             !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1353                 aead_unmap(ctx->dev, edesc, req);
1354                 qi_cache_free(edesc);
1355         }
1356
1357         return ret;
1358 }
1359
1360 static int ipsec_gcm_encrypt(struct aead_request *req)
1361 {
1362         return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_encrypt(req);
1363 }
1364
1365 static int ipsec_gcm_decrypt(struct aead_request *req)
1366 {
1367         return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_decrypt(req);
1368 }
1369
1370 static void skcipher_encrypt_done(void *cbk_ctx, u32 status)
1371 {
1372         struct crypto_async_request *areq = cbk_ctx;
1373         struct skcipher_request *req = skcipher_request_cast(areq);
1374         struct caam_request *req_ctx = to_caam_req(areq);
1375         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1376         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1377         struct skcipher_edesc *edesc = req_ctx->edesc;
1378         int ecode = 0;
1379         int ivsize = crypto_skcipher_ivsize(skcipher);
1380
1381         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1382
1383         if (unlikely(status))
1384                 ecode = caam_qi2_strstatus(ctx->dev, status);
1385
1386         print_hex_dump_debug("dstiv  @" __stringify(__LINE__)": ",
1387                              DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
1388                              edesc->src_nents > 1 ? 100 : ivsize, 1);
1389         caam_dump_sg("dst    @" __stringify(__LINE__)": ",
1390                      DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
1391                      edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
1392
1393         skcipher_unmap(ctx->dev, edesc, req);
1394
1395         /*
1396          * The crypto API expects us to set the IV (req->iv) to the last
1397          * ciphertext block (CBC mode) or last counter (CTR mode).
1398          * This is used e.g. by the CTS mode.
1399          */
1400         if (!ecode)
1401                 memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes,
1402                        ivsize);
1403
1404         qi_cache_free(edesc);
1405         skcipher_request_complete(req, ecode);
1406 }
1407
1408 static void skcipher_decrypt_done(void *cbk_ctx, u32 status)
1409 {
1410         struct crypto_async_request *areq = cbk_ctx;
1411         struct skcipher_request *req = skcipher_request_cast(areq);
1412         struct caam_request *req_ctx = to_caam_req(areq);
1413         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1414         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1415         struct skcipher_edesc *edesc = req_ctx->edesc;
1416         int ecode = 0;
1417         int ivsize = crypto_skcipher_ivsize(skcipher);
1418
1419         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1420
1421         if (unlikely(status))
1422                 ecode = caam_qi2_strstatus(ctx->dev, status);
1423
1424         print_hex_dump_debug("dstiv  @" __stringify(__LINE__)": ",
1425                              DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
1426                              edesc->src_nents > 1 ? 100 : ivsize, 1);
1427         caam_dump_sg("dst    @" __stringify(__LINE__)": ",
1428                      DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
1429                      edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
1430
1431         skcipher_unmap(ctx->dev, edesc, req);
1432
1433         /*
1434          * The crypto API expects us to set the IV (req->iv) to the last
1435          * ciphertext block (CBC mode) or last counter (CTR mode).
1436          * This is used e.g. by the CTS mode.
1437          */
1438         if (!ecode)
1439                 memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes,
1440                        ivsize);
1441
1442         qi_cache_free(edesc);
1443         skcipher_request_complete(req, ecode);
1444 }
1445
1446 static int skcipher_encrypt(struct skcipher_request *req)
1447 {
1448         struct skcipher_edesc *edesc;
1449         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1450         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1451         struct caam_request *caam_req = skcipher_request_ctx(req);
1452         int ret;
1453
1454         if (!req->cryptlen)
1455                 return 0;
1456
1457         /* allocate extended descriptor */
1458         edesc = skcipher_edesc_alloc(req);
1459         if (IS_ERR(edesc))
1460                 return PTR_ERR(edesc);
1461
1462         caam_req->flc = &ctx->flc[ENCRYPT];
1463         caam_req->flc_dma = ctx->flc_dma[ENCRYPT];
1464         caam_req->cbk = skcipher_encrypt_done;
1465         caam_req->ctx = &req->base;
1466         caam_req->edesc = edesc;
1467         ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1468         if (ret != -EINPROGRESS &&
1469             !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1470                 skcipher_unmap(ctx->dev, edesc, req);
1471                 qi_cache_free(edesc);
1472         }
1473
1474         return ret;
1475 }
1476
1477 static int skcipher_decrypt(struct skcipher_request *req)
1478 {
1479         struct skcipher_edesc *edesc;
1480         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1481         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1482         struct caam_request *caam_req = skcipher_request_ctx(req);
1483         int ret;
1484
1485         if (!req->cryptlen)
1486                 return 0;
1487         /* allocate extended descriptor */
1488         edesc = skcipher_edesc_alloc(req);
1489         if (IS_ERR(edesc))
1490                 return PTR_ERR(edesc);
1491
1492         caam_req->flc = &ctx->flc[DECRYPT];
1493         caam_req->flc_dma = ctx->flc_dma[DECRYPT];
1494         caam_req->cbk = skcipher_decrypt_done;
1495         caam_req->ctx = &req->base;
1496         caam_req->edesc = edesc;
1497         ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1498         if (ret != -EINPROGRESS &&
1499             !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1500                 skcipher_unmap(ctx->dev, edesc, req);
1501                 qi_cache_free(edesc);
1502         }
1503
1504         return ret;
1505 }
1506
1507 static int caam_cra_init(struct caam_ctx *ctx, struct caam_alg_entry *caam,
1508                          bool uses_dkp)
1509 {
1510         dma_addr_t dma_addr;
1511         int i;
1512
1513         /* copy descriptor header template value */
1514         ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
1515         ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
1516
1517         ctx->dev = caam->dev;
1518         ctx->dir = uses_dkp ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
1519
1520         dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc,
1521                                         offsetof(struct caam_ctx, flc_dma),
1522                                         ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
1523         if (dma_mapping_error(ctx->dev, dma_addr)) {
1524                 dev_err(ctx->dev, "unable to map key, shared descriptors\n");
1525                 return -ENOMEM;
1526         }
1527
1528         for (i = 0; i < NUM_OP; i++)
1529                 ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]);
1530         ctx->key_dma = dma_addr + NUM_OP * sizeof(ctx->flc[0]);
1531
1532         return 0;
1533 }
1534
1535 static int caam_cra_init_skcipher(struct crypto_skcipher *tfm)
1536 {
1537         struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
1538         struct caam_skcipher_alg *caam_alg =
1539                 container_of(alg, typeof(*caam_alg), skcipher);
1540
1541         crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_request));
1542         return caam_cra_init(crypto_skcipher_ctx(tfm), &caam_alg->caam, false);
1543 }
1544
1545 static int caam_cra_init_aead(struct crypto_aead *tfm)
1546 {
1547         struct aead_alg *alg = crypto_aead_alg(tfm);
1548         struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg),
1549                                                       aead);
1550
1551         crypto_aead_set_reqsize(tfm, sizeof(struct caam_request));
1552         return caam_cra_init(crypto_aead_ctx(tfm), &caam_alg->caam,
1553                              !caam_alg->caam.nodkp);
1554 }
1555
1556 static void caam_exit_common(struct caam_ctx *ctx)
1557 {
1558         dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0],
1559                                offsetof(struct caam_ctx, flc_dma), ctx->dir,
1560                                DMA_ATTR_SKIP_CPU_SYNC);
1561 }
1562
1563 static void caam_cra_exit(struct crypto_skcipher *tfm)
1564 {
1565         caam_exit_common(crypto_skcipher_ctx(tfm));
1566 }
1567
1568 static void caam_cra_exit_aead(struct crypto_aead *tfm)
1569 {
1570         caam_exit_common(crypto_aead_ctx(tfm));
1571 }
1572
1573 static struct caam_skcipher_alg driver_algs[] = {
1574         {
1575                 .skcipher = {
1576                         .base = {
1577                                 .cra_name = "cbc(aes)",
1578                                 .cra_driver_name = "cbc-aes-caam-qi2",
1579                                 .cra_blocksize = AES_BLOCK_SIZE,
1580                         },
1581                         .setkey = aes_skcipher_setkey,
1582                         .encrypt = skcipher_encrypt,
1583                         .decrypt = skcipher_decrypt,
1584                         .min_keysize = AES_MIN_KEY_SIZE,
1585                         .max_keysize = AES_MAX_KEY_SIZE,
1586                         .ivsize = AES_BLOCK_SIZE,
1587                 },
1588                 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1589         },
1590         {
1591                 .skcipher = {
1592                         .base = {
1593                                 .cra_name = "cbc(des3_ede)",
1594                                 .cra_driver_name = "cbc-3des-caam-qi2",
1595                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1596                         },
1597                         .setkey = des3_skcipher_setkey,
1598                         .encrypt = skcipher_encrypt,
1599                         .decrypt = skcipher_decrypt,
1600                         .min_keysize = DES3_EDE_KEY_SIZE,
1601                         .max_keysize = DES3_EDE_KEY_SIZE,
1602                         .ivsize = DES3_EDE_BLOCK_SIZE,
1603                 },
1604                 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1605         },
1606         {
1607                 .skcipher = {
1608                         .base = {
1609                                 .cra_name = "cbc(des)",
1610                                 .cra_driver_name = "cbc-des-caam-qi2",
1611                                 .cra_blocksize = DES_BLOCK_SIZE,
1612                         },
1613                         .setkey = des_skcipher_setkey,
1614                         .encrypt = skcipher_encrypt,
1615                         .decrypt = skcipher_decrypt,
1616                         .min_keysize = DES_KEY_SIZE,
1617                         .max_keysize = DES_KEY_SIZE,
1618                         .ivsize = DES_BLOCK_SIZE,
1619                 },
1620                 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
1621         },
1622         {
1623                 .skcipher = {
1624                         .base = {
1625                                 .cra_name = "ctr(aes)",
1626                                 .cra_driver_name = "ctr-aes-caam-qi2",
1627                                 .cra_blocksize = 1,
1628                         },
1629                         .setkey = ctr_skcipher_setkey,
1630                         .encrypt = skcipher_encrypt,
1631                         .decrypt = skcipher_decrypt,
1632                         .min_keysize = AES_MIN_KEY_SIZE,
1633                         .max_keysize = AES_MAX_KEY_SIZE,
1634                         .ivsize = AES_BLOCK_SIZE,
1635                         .chunksize = AES_BLOCK_SIZE,
1636                 },
1637                 .caam.class1_alg_type = OP_ALG_ALGSEL_AES |
1638                                         OP_ALG_AAI_CTR_MOD128,
1639         },
1640         {
1641                 .skcipher = {
1642                         .base = {
1643                                 .cra_name = "rfc3686(ctr(aes))",
1644                                 .cra_driver_name = "rfc3686-ctr-aes-caam-qi2",
1645                                 .cra_blocksize = 1,
1646                         },
1647                         .setkey = rfc3686_skcipher_setkey,
1648                         .encrypt = skcipher_encrypt,
1649                         .decrypt = skcipher_decrypt,
1650                         .min_keysize = AES_MIN_KEY_SIZE +
1651                                        CTR_RFC3686_NONCE_SIZE,
1652                         .max_keysize = AES_MAX_KEY_SIZE +
1653                                        CTR_RFC3686_NONCE_SIZE,
1654                         .ivsize = CTR_RFC3686_IV_SIZE,
1655                         .chunksize = AES_BLOCK_SIZE,
1656                 },
1657                 .caam = {
1658                         .class1_alg_type = OP_ALG_ALGSEL_AES |
1659                                            OP_ALG_AAI_CTR_MOD128,
1660                         .rfc3686 = true,
1661                 },
1662         },
1663         {
1664                 .skcipher = {
1665                         .base = {
1666                                 .cra_name = "xts(aes)",
1667                                 .cra_driver_name = "xts-aes-caam-qi2",
1668                                 .cra_blocksize = AES_BLOCK_SIZE,
1669                         },
1670                         .setkey = xts_skcipher_setkey,
1671                         .encrypt = skcipher_encrypt,
1672                         .decrypt = skcipher_decrypt,
1673                         .min_keysize = 2 * AES_MIN_KEY_SIZE,
1674                         .max_keysize = 2 * AES_MAX_KEY_SIZE,
1675                         .ivsize = AES_BLOCK_SIZE,
1676                 },
1677                 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS,
1678         },
1679         {
1680                 .skcipher = {
1681                         .base = {
1682                                 .cra_name = "chacha20",
1683                                 .cra_driver_name = "chacha20-caam-qi2",
1684                                 .cra_blocksize = 1,
1685                         },
1686                         .setkey = chacha20_skcipher_setkey,
1687                         .encrypt = skcipher_encrypt,
1688                         .decrypt = skcipher_decrypt,
1689                         .min_keysize = CHACHA_KEY_SIZE,
1690                         .max_keysize = CHACHA_KEY_SIZE,
1691                         .ivsize = CHACHA_IV_SIZE,
1692                 },
1693                 .caam.class1_alg_type = OP_ALG_ALGSEL_CHACHA20,
1694         },
1695 };
1696
1697 static struct caam_aead_alg driver_aeads[] = {
1698         {
1699                 .aead = {
1700                         .base = {
1701                                 .cra_name = "rfc4106(gcm(aes))",
1702                                 .cra_driver_name = "rfc4106-gcm-aes-caam-qi2",
1703                                 .cra_blocksize = 1,
1704                         },
1705                         .setkey = rfc4106_setkey,
1706                         .setauthsize = rfc4106_setauthsize,
1707                         .encrypt = ipsec_gcm_encrypt,
1708                         .decrypt = ipsec_gcm_decrypt,
1709                         .ivsize = 8,
1710                         .maxauthsize = AES_BLOCK_SIZE,
1711                 },
1712                 .caam = {
1713                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1714                         .nodkp = true,
1715                 },
1716         },
1717         {
1718                 .aead = {
1719                         .base = {
1720                                 .cra_name = "rfc4543(gcm(aes))",
1721                                 .cra_driver_name = "rfc4543-gcm-aes-caam-qi2",
1722                                 .cra_blocksize = 1,
1723                         },
1724                         .setkey = rfc4543_setkey,
1725                         .setauthsize = rfc4543_setauthsize,
1726                         .encrypt = ipsec_gcm_encrypt,
1727                         .decrypt = ipsec_gcm_decrypt,
1728                         .ivsize = 8,
1729                         .maxauthsize = AES_BLOCK_SIZE,
1730                 },
1731                 .caam = {
1732                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1733                         .nodkp = true,
1734                 },
1735         },
1736         /* Galois Counter Mode */
1737         {
1738                 .aead = {
1739                         .base = {
1740                                 .cra_name = "gcm(aes)",
1741                                 .cra_driver_name = "gcm-aes-caam-qi2",
1742                                 .cra_blocksize = 1,
1743                         },
1744                         .setkey = gcm_setkey,
1745                         .setauthsize = gcm_setauthsize,
1746                         .encrypt = aead_encrypt,
1747                         .decrypt = aead_decrypt,
1748                         .ivsize = 12,
1749                         .maxauthsize = AES_BLOCK_SIZE,
1750                 },
1751                 .caam = {
1752                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1753                         .nodkp = true,
1754                 }
1755         },
1756         /* single-pass ipsec_esp descriptor */
1757         {
1758                 .aead = {
1759                         .base = {
1760                                 .cra_name = "authenc(hmac(md5),cbc(aes))",
1761                                 .cra_driver_name = "authenc-hmac-md5-"
1762                                                    "cbc-aes-caam-qi2",
1763                                 .cra_blocksize = AES_BLOCK_SIZE,
1764                         },
1765                         .setkey = aead_setkey,
1766                         .setauthsize = aead_setauthsize,
1767                         .encrypt = aead_encrypt,
1768                         .decrypt = aead_decrypt,
1769                         .ivsize = AES_BLOCK_SIZE,
1770                         .maxauthsize = MD5_DIGEST_SIZE,
1771                 },
1772                 .caam = {
1773                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1774                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
1775                                            OP_ALG_AAI_HMAC_PRECOMP,
1776                 }
1777         },
1778         {
1779                 .aead = {
1780                         .base = {
1781                                 .cra_name = "echainiv(authenc(hmac(md5),"
1782                                             "cbc(aes)))",
1783                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
1784                                                    "cbc-aes-caam-qi2",
1785                                 .cra_blocksize = AES_BLOCK_SIZE,
1786                         },
1787                         .setkey = aead_setkey,
1788                         .setauthsize = aead_setauthsize,
1789                         .encrypt = aead_encrypt,
1790                         .decrypt = aead_decrypt,
1791                         .ivsize = AES_BLOCK_SIZE,
1792                         .maxauthsize = MD5_DIGEST_SIZE,
1793                 },
1794                 .caam = {
1795                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1796                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
1797                                            OP_ALG_AAI_HMAC_PRECOMP,
1798                         .geniv = true,
1799                 }
1800         },
1801         {
1802                 .aead = {
1803                         .base = {
1804                                 .cra_name = "authenc(hmac(sha1),cbc(aes))",
1805                                 .cra_driver_name = "authenc-hmac-sha1-"
1806                                                    "cbc-aes-caam-qi2",
1807                                 .cra_blocksize = AES_BLOCK_SIZE,
1808                         },
1809                         .setkey = aead_setkey,
1810                         .setauthsize = aead_setauthsize,
1811                         .encrypt = aead_encrypt,
1812                         .decrypt = aead_decrypt,
1813                         .ivsize = AES_BLOCK_SIZE,
1814                         .maxauthsize = SHA1_DIGEST_SIZE,
1815                 },
1816                 .caam = {
1817                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1818                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1819                                            OP_ALG_AAI_HMAC_PRECOMP,
1820                 }
1821         },
1822         {
1823                 .aead = {
1824                         .base = {
1825                                 .cra_name = "echainiv(authenc(hmac(sha1),"
1826                                             "cbc(aes)))",
1827                                 .cra_driver_name = "echainiv-authenc-"
1828                                                    "hmac-sha1-cbc-aes-caam-qi2",
1829                                 .cra_blocksize = AES_BLOCK_SIZE,
1830                         },
1831                         .setkey = aead_setkey,
1832                         .setauthsize = aead_setauthsize,
1833                         .encrypt = aead_encrypt,
1834                         .decrypt = aead_decrypt,
1835                         .ivsize = AES_BLOCK_SIZE,
1836                         .maxauthsize = SHA1_DIGEST_SIZE,
1837                 },
1838                 .caam = {
1839                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1840                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1841                                            OP_ALG_AAI_HMAC_PRECOMP,
1842                         .geniv = true,
1843                 },
1844         },
1845         {
1846                 .aead = {
1847                         .base = {
1848                                 .cra_name = "authenc(hmac(sha224),cbc(aes))",
1849                                 .cra_driver_name = "authenc-hmac-sha224-"
1850                                                    "cbc-aes-caam-qi2",
1851                                 .cra_blocksize = AES_BLOCK_SIZE,
1852                         },
1853                         .setkey = aead_setkey,
1854                         .setauthsize = aead_setauthsize,
1855                         .encrypt = aead_encrypt,
1856                         .decrypt = aead_decrypt,
1857                         .ivsize = AES_BLOCK_SIZE,
1858                         .maxauthsize = SHA224_DIGEST_SIZE,
1859                 },
1860                 .caam = {
1861                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1862                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1863                                            OP_ALG_AAI_HMAC_PRECOMP,
1864                 }
1865         },
1866         {
1867                 .aead = {
1868                         .base = {
1869                                 .cra_name = "echainiv(authenc(hmac(sha224),"
1870                                             "cbc(aes)))",
1871                                 .cra_driver_name = "echainiv-authenc-"
1872                                                    "hmac-sha224-cbc-aes-caam-qi2",
1873                                 .cra_blocksize = AES_BLOCK_SIZE,
1874                         },
1875                         .setkey = aead_setkey,
1876                         .setauthsize = aead_setauthsize,
1877                         .encrypt = aead_encrypt,
1878                         .decrypt = aead_decrypt,
1879                         .ivsize = AES_BLOCK_SIZE,
1880                         .maxauthsize = SHA224_DIGEST_SIZE,
1881                 },
1882                 .caam = {
1883                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1884                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1885                                            OP_ALG_AAI_HMAC_PRECOMP,
1886                         .geniv = true,
1887                 }
1888         },
1889         {
1890                 .aead = {
1891                         .base = {
1892                                 .cra_name = "authenc(hmac(sha256),cbc(aes))",
1893                                 .cra_driver_name = "authenc-hmac-sha256-"
1894                                                    "cbc-aes-caam-qi2",
1895                                 .cra_blocksize = AES_BLOCK_SIZE,
1896                         },
1897                         .setkey = aead_setkey,
1898                         .setauthsize = aead_setauthsize,
1899                         .encrypt = aead_encrypt,
1900                         .decrypt = aead_decrypt,
1901                         .ivsize = AES_BLOCK_SIZE,
1902                         .maxauthsize = SHA256_DIGEST_SIZE,
1903                 },
1904                 .caam = {
1905                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1906                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
1907                                            OP_ALG_AAI_HMAC_PRECOMP,
1908                 }
1909         },
1910         {
1911                 .aead = {
1912                         .base = {
1913                                 .cra_name = "echainiv(authenc(hmac(sha256),"
1914                                             "cbc(aes)))",
1915                                 .cra_driver_name = "echainiv-authenc-"
1916                                                    "hmac-sha256-cbc-aes-"
1917                                                    "caam-qi2",
1918                                 .cra_blocksize = AES_BLOCK_SIZE,
1919                         },
1920                         .setkey = aead_setkey,
1921                         .setauthsize = aead_setauthsize,
1922                         .encrypt = aead_encrypt,
1923                         .decrypt = aead_decrypt,
1924                         .ivsize = AES_BLOCK_SIZE,
1925                         .maxauthsize = SHA256_DIGEST_SIZE,
1926                 },
1927                 .caam = {
1928                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1929                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
1930                                            OP_ALG_AAI_HMAC_PRECOMP,
1931                         .geniv = true,
1932                 }
1933         },
1934         {
1935                 .aead = {
1936                         .base = {
1937                                 .cra_name = "authenc(hmac(sha384),cbc(aes))",
1938                                 .cra_driver_name = "authenc-hmac-sha384-"
1939                                                    "cbc-aes-caam-qi2",
1940                                 .cra_blocksize = AES_BLOCK_SIZE,
1941                         },
1942                         .setkey = aead_setkey,
1943                         .setauthsize = aead_setauthsize,
1944                         .encrypt = aead_encrypt,
1945                         .decrypt = aead_decrypt,
1946                         .ivsize = AES_BLOCK_SIZE,
1947                         .maxauthsize = SHA384_DIGEST_SIZE,
1948                 },
1949                 .caam = {
1950                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1951                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
1952                                            OP_ALG_AAI_HMAC_PRECOMP,
1953                 }
1954         },
1955         {
1956                 .aead = {
1957                         .base = {
1958                                 .cra_name = "echainiv(authenc(hmac(sha384),"
1959                                             "cbc(aes)))",
1960                                 .cra_driver_name = "echainiv-authenc-"
1961                                                    "hmac-sha384-cbc-aes-"
1962                                                    "caam-qi2",
1963                                 .cra_blocksize = AES_BLOCK_SIZE,
1964                         },
1965                         .setkey = aead_setkey,
1966                         .setauthsize = aead_setauthsize,
1967                         .encrypt = aead_encrypt,
1968                         .decrypt = aead_decrypt,
1969                         .ivsize = AES_BLOCK_SIZE,
1970                         .maxauthsize = SHA384_DIGEST_SIZE,
1971                 },
1972                 .caam = {
1973                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1974                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
1975                                            OP_ALG_AAI_HMAC_PRECOMP,
1976                         .geniv = true,
1977                 }
1978         },
1979         {
1980                 .aead = {
1981                         .base = {
1982                                 .cra_name = "authenc(hmac(sha512),cbc(aes))",
1983                                 .cra_driver_name = "authenc-hmac-sha512-"
1984                                                    "cbc-aes-caam-qi2",
1985                                 .cra_blocksize = AES_BLOCK_SIZE,
1986                         },
1987                         .setkey = aead_setkey,
1988                         .setauthsize = aead_setauthsize,
1989                         .encrypt = aead_encrypt,
1990                         .decrypt = aead_decrypt,
1991                         .ivsize = AES_BLOCK_SIZE,
1992                         .maxauthsize = SHA512_DIGEST_SIZE,
1993                 },
1994                 .caam = {
1995                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1996                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
1997                                            OP_ALG_AAI_HMAC_PRECOMP,
1998                 }
1999         },
2000         {
2001                 .aead = {
2002                         .base = {
2003                                 .cra_name = "echainiv(authenc(hmac(sha512),"
2004                                             "cbc(aes)))",
2005                                 .cra_driver_name = "echainiv-authenc-"
2006                                                    "hmac-sha512-cbc-aes-"
2007                                                    "caam-qi2",
2008                                 .cra_blocksize = AES_BLOCK_SIZE,
2009                         },
2010                         .setkey = aead_setkey,
2011                         .setauthsize = aead_setauthsize,
2012                         .encrypt = aead_encrypt,
2013                         .decrypt = aead_decrypt,
2014                         .ivsize = AES_BLOCK_SIZE,
2015                         .maxauthsize = SHA512_DIGEST_SIZE,
2016                 },
2017                 .caam = {
2018                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2019                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2020                                            OP_ALG_AAI_HMAC_PRECOMP,
2021                         .geniv = true,
2022                 }
2023         },
2024         {
2025                 .aead = {
2026                         .base = {
2027                                 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2028                                 .cra_driver_name = "authenc-hmac-md5-"
2029                                                    "cbc-des3_ede-caam-qi2",
2030                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2031                         },
2032                         .setkey = des3_aead_setkey,
2033                         .setauthsize = aead_setauthsize,
2034                         .encrypt = aead_encrypt,
2035                         .decrypt = aead_decrypt,
2036                         .ivsize = DES3_EDE_BLOCK_SIZE,
2037                         .maxauthsize = MD5_DIGEST_SIZE,
2038                 },
2039                 .caam = {
2040                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2041                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2042                                            OP_ALG_AAI_HMAC_PRECOMP,
2043                 }
2044         },
2045         {
2046                 .aead = {
2047                         .base = {
2048                                 .cra_name = "echainiv(authenc(hmac(md5),"
2049                                             "cbc(des3_ede)))",
2050                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
2051                                                    "cbc-des3_ede-caam-qi2",
2052                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2053                         },
2054                         .setkey = des3_aead_setkey,
2055                         .setauthsize = aead_setauthsize,
2056                         .encrypt = aead_encrypt,
2057                         .decrypt = aead_decrypt,
2058                         .ivsize = DES3_EDE_BLOCK_SIZE,
2059                         .maxauthsize = MD5_DIGEST_SIZE,
2060                 },
2061                 .caam = {
2062                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2063                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2064                                            OP_ALG_AAI_HMAC_PRECOMP,
2065                         .geniv = true,
2066                 }
2067         },
2068         {
2069                 .aead = {
2070                         .base = {
2071                                 .cra_name = "authenc(hmac(sha1),"
2072                                             "cbc(des3_ede))",
2073                                 .cra_driver_name = "authenc-hmac-sha1-"
2074                                                    "cbc-des3_ede-caam-qi2",
2075                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2076                         },
2077                         .setkey = des3_aead_setkey,
2078                         .setauthsize = aead_setauthsize,
2079                         .encrypt = aead_encrypt,
2080                         .decrypt = aead_decrypt,
2081                         .ivsize = DES3_EDE_BLOCK_SIZE,
2082                         .maxauthsize = SHA1_DIGEST_SIZE,
2083                 },
2084                 .caam = {
2085                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2086                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2087                                            OP_ALG_AAI_HMAC_PRECOMP,
2088                 },
2089         },
2090         {
2091                 .aead = {
2092                         .base = {
2093                                 .cra_name = "echainiv(authenc(hmac(sha1),"
2094                                             "cbc(des3_ede)))",
2095                                 .cra_driver_name = "echainiv-authenc-"
2096                                                    "hmac-sha1-"
2097                                                    "cbc-des3_ede-caam-qi2",
2098                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2099                         },
2100                         .setkey = des3_aead_setkey,
2101                         .setauthsize = aead_setauthsize,
2102                         .encrypt = aead_encrypt,
2103                         .decrypt = aead_decrypt,
2104                         .ivsize = DES3_EDE_BLOCK_SIZE,
2105                         .maxauthsize = SHA1_DIGEST_SIZE,
2106                 },
2107                 .caam = {
2108                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2109                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2110                                            OP_ALG_AAI_HMAC_PRECOMP,
2111                         .geniv = true,
2112                 }
2113         },
2114         {
2115                 .aead = {
2116                         .base = {
2117                                 .cra_name = "authenc(hmac(sha224),"
2118                                             "cbc(des3_ede))",
2119                                 .cra_driver_name = "authenc-hmac-sha224-"
2120                                                    "cbc-des3_ede-caam-qi2",
2121                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2122                         },
2123                         .setkey = des3_aead_setkey,
2124                         .setauthsize = aead_setauthsize,
2125                         .encrypt = aead_encrypt,
2126                         .decrypt = aead_decrypt,
2127                         .ivsize = DES3_EDE_BLOCK_SIZE,
2128                         .maxauthsize = SHA224_DIGEST_SIZE,
2129                 },
2130                 .caam = {
2131                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2132                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2133                                            OP_ALG_AAI_HMAC_PRECOMP,
2134                 },
2135         },
2136         {
2137                 .aead = {
2138                         .base = {
2139                                 .cra_name = "echainiv(authenc(hmac(sha224),"
2140                                             "cbc(des3_ede)))",
2141                                 .cra_driver_name = "echainiv-authenc-"
2142                                                    "hmac-sha224-"
2143                                                    "cbc-des3_ede-caam-qi2",
2144                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2145                         },
2146                         .setkey = des3_aead_setkey,
2147                         .setauthsize = aead_setauthsize,
2148                         .encrypt = aead_encrypt,
2149                         .decrypt = aead_decrypt,
2150                         .ivsize = DES3_EDE_BLOCK_SIZE,
2151                         .maxauthsize = SHA224_DIGEST_SIZE,
2152                 },
2153                 .caam = {
2154                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2155                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2156                                            OP_ALG_AAI_HMAC_PRECOMP,
2157                         .geniv = true,
2158                 }
2159         },
2160         {
2161                 .aead = {
2162                         .base = {
2163                                 .cra_name = "authenc(hmac(sha256),"
2164                                             "cbc(des3_ede))",
2165                                 .cra_driver_name = "authenc-hmac-sha256-"
2166                                                    "cbc-des3_ede-caam-qi2",
2167                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2168                         },
2169                         .setkey = des3_aead_setkey,
2170                         .setauthsize = aead_setauthsize,
2171                         .encrypt = aead_encrypt,
2172                         .decrypt = aead_decrypt,
2173                         .ivsize = DES3_EDE_BLOCK_SIZE,
2174                         .maxauthsize = SHA256_DIGEST_SIZE,
2175                 },
2176                 .caam = {
2177                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2178                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2179                                            OP_ALG_AAI_HMAC_PRECOMP,
2180                 },
2181         },
2182         {
2183                 .aead = {
2184                         .base = {
2185                                 .cra_name = "echainiv(authenc(hmac(sha256),"
2186                                             "cbc(des3_ede)))",
2187                                 .cra_driver_name = "echainiv-authenc-"
2188                                                    "hmac-sha256-"
2189                                                    "cbc-des3_ede-caam-qi2",
2190                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2191                         },
2192                         .setkey = des3_aead_setkey,
2193                         .setauthsize = aead_setauthsize,
2194                         .encrypt = aead_encrypt,
2195                         .decrypt = aead_decrypt,
2196                         .ivsize = DES3_EDE_BLOCK_SIZE,
2197                         .maxauthsize = SHA256_DIGEST_SIZE,
2198                 },
2199                 .caam = {
2200                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2201                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2202                                            OP_ALG_AAI_HMAC_PRECOMP,
2203                         .geniv = true,
2204                 }
2205         },
2206         {
2207                 .aead = {
2208                         .base = {
2209                                 .cra_name = "authenc(hmac(sha384),"
2210                                             "cbc(des3_ede))",
2211                                 .cra_driver_name = "authenc-hmac-sha384-"
2212                                                    "cbc-des3_ede-caam-qi2",
2213                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2214                         },
2215                         .setkey = des3_aead_setkey,
2216                         .setauthsize = aead_setauthsize,
2217                         .encrypt = aead_encrypt,
2218                         .decrypt = aead_decrypt,
2219                         .ivsize = DES3_EDE_BLOCK_SIZE,
2220                         .maxauthsize = SHA384_DIGEST_SIZE,
2221                 },
2222                 .caam = {
2223                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2224                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2225                                            OP_ALG_AAI_HMAC_PRECOMP,
2226                 },
2227         },
2228         {
2229                 .aead = {
2230                         .base = {
2231                                 .cra_name = "echainiv(authenc(hmac(sha384),"
2232                                             "cbc(des3_ede)))",
2233                                 .cra_driver_name = "echainiv-authenc-"
2234                                                    "hmac-sha384-"
2235                                                    "cbc-des3_ede-caam-qi2",
2236                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2237                         },
2238                         .setkey = des3_aead_setkey,
2239                         .setauthsize = aead_setauthsize,
2240                         .encrypt = aead_encrypt,
2241                         .decrypt = aead_decrypt,
2242                         .ivsize = DES3_EDE_BLOCK_SIZE,
2243                         .maxauthsize = SHA384_DIGEST_SIZE,
2244                 },
2245                 .caam = {
2246                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2247                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2248                                            OP_ALG_AAI_HMAC_PRECOMP,
2249                         .geniv = true,
2250                 }
2251         },
2252         {
2253                 .aead = {
2254                         .base = {
2255                                 .cra_name = "authenc(hmac(sha512),"
2256                                             "cbc(des3_ede))",
2257                                 .cra_driver_name = "authenc-hmac-sha512-"
2258                                                    "cbc-des3_ede-caam-qi2",
2259                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2260                         },
2261                         .setkey = des3_aead_setkey,
2262                         .setauthsize = aead_setauthsize,
2263                         .encrypt = aead_encrypt,
2264                         .decrypt = aead_decrypt,
2265                         .ivsize = DES3_EDE_BLOCK_SIZE,
2266                         .maxauthsize = SHA512_DIGEST_SIZE,
2267                 },
2268                 .caam = {
2269                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2270                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2271                                            OP_ALG_AAI_HMAC_PRECOMP,
2272                 },
2273         },
2274         {
2275                 .aead = {
2276                         .base = {
2277                                 .cra_name = "echainiv(authenc(hmac(sha512),"
2278                                             "cbc(des3_ede)))",
2279                                 .cra_driver_name = "echainiv-authenc-"
2280                                                    "hmac-sha512-"
2281                                                    "cbc-des3_ede-caam-qi2",
2282                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2283                         },
2284                         .setkey = des3_aead_setkey,
2285                         .setauthsize = aead_setauthsize,
2286                         .encrypt = aead_encrypt,
2287                         .decrypt = aead_decrypt,
2288                         .ivsize = DES3_EDE_BLOCK_SIZE,
2289                         .maxauthsize = SHA512_DIGEST_SIZE,
2290                 },
2291                 .caam = {
2292                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2293                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2294                                            OP_ALG_AAI_HMAC_PRECOMP,
2295                         .geniv = true,
2296                 }
2297         },
2298         {
2299                 .aead = {
2300                         .base = {
2301                                 .cra_name = "authenc(hmac(md5),cbc(des))",
2302                                 .cra_driver_name = "authenc-hmac-md5-"
2303                                                    "cbc-des-caam-qi2",
2304                                 .cra_blocksize = DES_BLOCK_SIZE,
2305                         },
2306                         .setkey = aead_setkey,
2307                         .setauthsize = aead_setauthsize,
2308                         .encrypt = aead_encrypt,
2309                         .decrypt = aead_decrypt,
2310                         .ivsize = DES_BLOCK_SIZE,
2311                         .maxauthsize = MD5_DIGEST_SIZE,
2312                 },
2313                 .caam = {
2314                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2315                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2316                                            OP_ALG_AAI_HMAC_PRECOMP,
2317                 },
2318         },
2319         {
2320                 .aead = {
2321                         .base = {
2322                                 .cra_name = "echainiv(authenc(hmac(md5),"
2323                                             "cbc(des)))",
2324                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
2325                                                    "cbc-des-caam-qi2",
2326                                 .cra_blocksize = DES_BLOCK_SIZE,
2327                         },
2328                         .setkey = aead_setkey,
2329                         .setauthsize = aead_setauthsize,
2330                         .encrypt = aead_encrypt,
2331                         .decrypt = aead_decrypt,
2332                         .ivsize = DES_BLOCK_SIZE,
2333                         .maxauthsize = MD5_DIGEST_SIZE,
2334                 },
2335                 .caam = {
2336                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2337                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2338                                            OP_ALG_AAI_HMAC_PRECOMP,
2339                         .geniv = true,
2340                 }
2341         },
2342         {
2343                 .aead = {
2344                         .base = {
2345                                 .cra_name = "authenc(hmac(sha1),cbc(des))",
2346                                 .cra_driver_name = "authenc-hmac-sha1-"
2347                                                    "cbc-des-caam-qi2",
2348                                 .cra_blocksize = DES_BLOCK_SIZE,
2349                         },
2350                         .setkey = aead_setkey,
2351                         .setauthsize = aead_setauthsize,
2352                         .encrypt = aead_encrypt,
2353                         .decrypt = aead_decrypt,
2354                         .ivsize = DES_BLOCK_SIZE,
2355                         .maxauthsize = SHA1_DIGEST_SIZE,
2356                 },
2357                 .caam = {
2358                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2359                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2360                                            OP_ALG_AAI_HMAC_PRECOMP,
2361                 },
2362         },
2363         {
2364                 .aead = {
2365                         .base = {
2366                                 .cra_name = "echainiv(authenc(hmac(sha1),"
2367                                             "cbc(des)))",
2368                                 .cra_driver_name = "echainiv-authenc-"
2369                                                    "hmac-sha1-cbc-des-caam-qi2",
2370                                 .cra_blocksize = DES_BLOCK_SIZE,
2371                         },
2372                         .setkey = aead_setkey,
2373                         .setauthsize = aead_setauthsize,
2374                         .encrypt = aead_encrypt,
2375                         .decrypt = aead_decrypt,
2376                         .ivsize = DES_BLOCK_SIZE,
2377                         .maxauthsize = SHA1_DIGEST_SIZE,
2378                 },
2379                 .caam = {
2380                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2381                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2382                                            OP_ALG_AAI_HMAC_PRECOMP,
2383                         .geniv = true,
2384                 }
2385         },
2386         {
2387                 .aead = {
2388                         .base = {
2389                                 .cra_name = "authenc(hmac(sha224),cbc(des))",
2390                                 .cra_driver_name = "authenc-hmac-sha224-"
2391                                                    "cbc-des-caam-qi2",
2392                                 .cra_blocksize = DES_BLOCK_SIZE,
2393                         },
2394                         .setkey = aead_setkey,
2395                         .setauthsize = aead_setauthsize,
2396                         .encrypt = aead_encrypt,
2397                         .decrypt = aead_decrypt,
2398                         .ivsize = DES_BLOCK_SIZE,
2399                         .maxauthsize = SHA224_DIGEST_SIZE,
2400                 },
2401                 .caam = {
2402                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2403                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2404                                            OP_ALG_AAI_HMAC_PRECOMP,
2405                 },
2406         },
2407         {
2408                 .aead = {
2409                         .base = {
2410                                 .cra_name = "echainiv(authenc(hmac(sha224),"
2411                                             "cbc(des)))",
2412                                 .cra_driver_name = "echainiv-authenc-"
2413                                                    "hmac-sha224-cbc-des-"
2414                                                    "caam-qi2",
2415                                 .cra_blocksize = DES_BLOCK_SIZE,
2416                         },
2417                         .setkey = aead_setkey,
2418                         .setauthsize = aead_setauthsize,
2419                         .encrypt = aead_encrypt,
2420                         .decrypt = aead_decrypt,
2421                         .ivsize = DES_BLOCK_SIZE,
2422                         .maxauthsize = SHA224_DIGEST_SIZE,
2423                 },
2424                 .caam = {
2425                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2426                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2427                                            OP_ALG_AAI_HMAC_PRECOMP,
2428                         .geniv = true,
2429                 }
2430         },
2431         {
2432                 .aead = {
2433                         .base = {
2434                                 .cra_name = "authenc(hmac(sha256),cbc(des))",
2435                                 .cra_driver_name = "authenc-hmac-sha256-"
2436                                                    "cbc-des-caam-qi2",
2437                                 .cra_blocksize = DES_BLOCK_SIZE,
2438                         },
2439                         .setkey = aead_setkey,
2440                         .setauthsize = aead_setauthsize,
2441                         .encrypt = aead_encrypt,
2442                         .decrypt = aead_decrypt,
2443                         .ivsize = DES_BLOCK_SIZE,
2444                         .maxauthsize = SHA256_DIGEST_SIZE,
2445                 },
2446                 .caam = {
2447                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2448                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2449                                            OP_ALG_AAI_HMAC_PRECOMP,
2450                 },
2451         },
2452         {
2453                 .aead = {
2454                         .base = {
2455                                 .cra_name = "echainiv(authenc(hmac(sha256),"
2456                                             "cbc(des)))",
2457                                 .cra_driver_name = "echainiv-authenc-"
2458                                                    "hmac-sha256-cbc-des-"
2459                                                    "caam-qi2",
2460                                 .cra_blocksize = DES_BLOCK_SIZE,
2461                         },
2462                         .setkey = aead_setkey,
2463                         .setauthsize = aead_setauthsize,
2464                         .encrypt = aead_encrypt,
2465                         .decrypt = aead_decrypt,
2466                         .ivsize = DES_BLOCK_SIZE,
2467                         .maxauthsize = SHA256_DIGEST_SIZE,
2468                 },
2469                 .caam = {
2470                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2471                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2472                                            OP_ALG_AAI_HMAC_PRECOMP,
2473                         .geniv = true,
2474                 },
2475         },
2476         {
2477                 .aead = {
2478                         .base = {
2479                                 .cra_name = "authenc(hmac(sha384),cbc(des))",
2480                                 .cra_driver_name = "authenc-hmac-sha384-"
2481                                                    "cbc-des-caam-qi2",
2482                                 .cra_blocksize = DES_BLOCK_SIZE,
2483                         },
2484                         .setkey = aead_setkey,
2485                         .setauthsize = aead_setauthsize,
2486                         .encrypt = aead_encrypt,
2487                         .decrypt = aead_decrypt,
2488                         .ivsize = DES_BLOCK_SIZE,
2489                         .maxauthsize = SHA384_DIGEST_SIZE,
2490                 },
2491                 .caam = {
2492                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2493                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2494                                            OP_ALG_AAI_HMAC_PRECOMP,
2495                 },
2496         },
2497         {
2498                 .aead = {
2499                         .base = {
2500                                 .cra_name = "echainiv(authenc(hmac(sha384),"
2501                                             "cbc(des)))",
2502                                 .cra_driver_name = "echainiv-authenc-"
2503                                                    "hmac-sha384-cbc-des-"
2504                                                    "caam-qi2",
2505                                 .cra_blocksize = DES_BLOCK_SIZE,
2506                         },
2507                         .setkey = aead_setkey,
2508                         .setauthsize = aead_setauthsize,
2509                         .encrypt = aead_encrypt,
2510                         .decrypt = aead_decrypt,
2511                         .ivsize = DES_BLOCK_SIZE,
2512                         .maxauthsize = SHA384_DIGEST_SIZE,
2513                 },
2514                 .caam = {
2515                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2516                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2517                                            OP_ALG_AAI_HMAC_PRECOMP,
2518                         .geniv = true,
2519                 }
2520         },
2521         {
2522                 .aead = {
2523                         .base = {
2524                                 .cra_name = "authenc(hmac(sha512),cbc(des))",
2525                                 .cra_driver_name = "authenc-hmac-sha512-"
2526                                                    "cbc-des-caam-qi2",
2527                                 .cra_blocksize = DES_BLOCK_SIZE,
2528                         },
2529                         .setkey = aead_setkey,
2530                         .setauthsize = aead_setauthsize,
2531                         .encrypt = aead_encrypt,
2532                         .decrypt = aead_decrypt,
2533                         .ivsize = DES_BLOCK_SIZE,
2534                         .maxauthsize = SHA512_DIGEST_SIZE,
2535                 },
2536                 .caam = {
2537                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2538                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2539                                            OP_ALG_AAI_HMAC_PRECOMP,
2540                 }
2541         },
2542         {
2543                 .aead = {
2544                         .base = {
2545                                 .cra_name = "echainiv(authenc(hmac(sha512),"
2546                                             "cbc(des)))",
2547                                 .cra_driver_name = "echainiv-authenc-"
2548                                                    "hmac-sha512-cbc-des-"
2549                                                    "caam-qi2",
2550                                 .cra_blocksize = DES_BLOCK_SIZE,
2551                         },
2552                         .setkey = aead_setkey,
2553                         .setauthsize = aead_setauthsize,
2554                         .encrypt = aead_encrypt,
2555                         .decrypt = aead_decrypt,
2556                         .ivsize = DES_BLOCK_SIZE,
2557                         .maxauthsize = SHA512_DIGEST_SIZE,
2558                 },
2559                 .caam = {
2560                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2561                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2562                                            OP_ALG_AAI_HMAC_PRECOMP,
2563                         .geniv = true,
2564                 }
2565         },
2566         {
2567                 .aead = {
2568                         .base = {
2569                                 .cra_name = "authenc(hmac(md5),"
2570                                             "rfc3686(ctr(aes)))",
2571                                 .cra_driver_name = "authenc-hmac-md5-"
2572                                                    "rfc3686-ctr-aes-caam-qi2",
2573                                 .cra_blocksize = 1,
2574                         },
2575                         .setkey = aead_setkey,
2576                         .setauthsize = aead_setauthsize,
2577                         .encrypt = aead_encrypt,
2578                         .decrypt = aead_decrypt,
2579                         .ivsize = CTR_RFC3686_IV_SIZE,
2580                         .maxauthsize = MD5_DIGEST_SIZE,
2581                 },
2582                 .caam = {
2583                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2584                                            OP_ALG_AAI_CTR_MOD128,
2585                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2586                                            OP_ALG_AAI_HMAC_PRECOMP,
2587                         .rfc3686 = true,
2588                 },
2589         },
2590         {
2591                 .aead = {
2592                         .base = {
2593                                 .cra_name = "seqiv(authenc("
2594                                             "hmac(md5),rfc3686(ctr(aes))))",
2595                                 .cra_driver_name = "seqiv-authenc-hmac-md5-"
2596                                                    "rfc3686-ctr-aes-caam-qi2",
2597                                 .cra_blocksize = 1,
2598                         },
2599                         .setkey = aead_setkey,
2600                         .setauthsize = aead_setauthsize,
2601                         .encrypt = aead_encrypt,
2602                         .decrypt = aead_decrypt,
2603                         .ivsize = CTR_RFC3686_IV_SIZE,
2604                         .maxauthsize = MD5_DIGEST_SIZE,
2605                 },
2606                 .caam = {
2607                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2608                                            OP_ALG_AAI_CTR_MOD128,
2609                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2610                                            OP_ALG_AAI_HMAC_PRECOMP,
2611                         .rfc3686 = true,
2612                         .geniv = true,
2613                 },
2614         },
2615         {
2616                 .aead = {
2617                         .base = {
2618                                 .cra_name = "authenc(hmac(sha1),"
2619                                             "rfc3686(ctr(aes)))",
2620                                 .cra_driver_name = "authenc-hmac-sha1-"
2621                                                    "rfc3686-ctr-aes-caam-qi2",
2622                                 .cra_blocksize = 1,
2623                         },
2624                         .setkey = aead_setkey,
2625                         .setauthsize = aead_setauthsize,
2626                         .encrypt = aead_encrypt,
2627                         .decrypt = aead_decrypt,
2628                         .ivsize = CTR_RFC3686_IV_SIZE,
2629                         .maxauthsize = SHA1_DIGEST_SIZE,
2630                 },
2631                 .caam = {
2632                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2633                                            OP_ALG_AAI_CTR_MOD128,
2634                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2635                                            OP_ALG_AAI_HMAC_PRECOMP,
2636                         .rfc3686 = true,
2637                 },
2638         },
2639         {
2640                 .aead = {
2641                         .base = {
2642                                 .cra_name = "seqiv(authenc("
2643                                             "hmac(sha1),rfc3686(ctr(aes))))",
2644                                 .cra_driver_name = "seqiv-authenc-hmac-sha1-"
2645                                                    "rfc3686-ctr-aes-caam-qi2",
2646                                 .cra_blocksize = 1,
2647                         },
2648                         .setkey = aead_setkey,
2649                         .setauthsize = aead_setauthsize,
2650                         .encrypt = aead_encrypt,
2651                         .decrypt = aead_decrypt,
2652                         .ivsize = CTR_RFC3686_IV_SIZE,
2653                         .maxauthsize = SHA1_DIGEST_SIZE,
2654                 },
2655                 .caam = {
2656                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2657                                            OP_ALG_AAI_CTR_MOD128,
2658                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2659                                            OP_ALG_AAI_HMAC_PRECOMP,
2660                         .rfc3686 = true,
2661                         .geniv = true,
2662                 },
2663         },
2664         {
2665                 .aead = {
2666                         .base = {
2667                                 .cra_name = "authenc(hmac(sha224),"
2668                                             "rfc3686(ctr(aes)))",
2669                                 .cra_driver_name = "authenc-hmac-sha224-"
2670                                                    "rfc3686-ctr-aes-caam-qi2",
2671                                 .cra_blocksize = 1,
2672                         },
2673                         .setkey = aead_setkey,
2674                         .setauthsize = aead_setauthsize,
2675                         .encrypt = aead_encrypt,
2676                         .decrypt = aead_decrypt,
2677                         .ivsize = CTR_RFC3686_IV_SIZE,
2678                         .maxauthsize = SHA224_DIGEST_SIZE,
2679                 },
2680                 .caam = {
2681                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2682                                            OP_ALG_AAI_CTR_MOD128,
2683                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2684                                            OP_ALG_AAI_HMAC_PRECOMP,
2685                         .rfc3686 = true,
2686                 },
2687         },
2688         {
2689                 .aead = {
2690                         .base = {
2691                                 .cra_name = "seqiv(authenc("
2692                                             "hmac(sha224),rfc3686(ctr(aes))))",
2693                                 .cra_driver_name = "seqiv-authenc-hmac-sha224-"
2694                                                    "rfc3686-ctr-aes-caam-qi2",
2695                                 .cra_blocksize = 1,
2696                         },
2697                         .setkey = aead_setkey,
2698                         .setauthsize = aead_setauthsize,
2699                         .encrypt = aead_encrypt,
2700                         .decrypt = aead_decrypt,
2701                         .ivsize = CTR_RFC3686_IV_SIZE,
2702                         .maxauthsize = SHA224_DIGEST_SIZE,
2703                 },
2704                 .caam = {
2705                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2706                                            OP_ALG_AAI_CTR_MOD128,
2707                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2708                                            OP_ALG_AAI_HMAC_PRECOMP,
2709                         .rfc3686 = true,
2710                         .geniv = true,
2711                 },
2712         },
2713         {
2714                 .aead = {
2715                         .base = {
2716                                 .cra_name = "authenc(hmac(sha256),"
2717                                             "rfc3686(ctr(aes)))",
2718                                 .cra_driver_name = "authenc-hmac-sha256-"
2719                                                    "rfc3686-ctr-aes-caam-qi2",
2720                                 .cra_blocksize = 1,
2721                         },
2722                         .setkey = aead_setkey,
2723                         .setauthsize = aead_setauthsize,
2724                         .encrypt = aead_encrypt,
2725                         .decrypt = aead_decrypt,
2726                         .ivsize = CTR_RFC3686_IV_SIZE,
2727                         .maxauthsize = SHA256_DIGEST_SIZE,
2728                 },
2729                 .caam = {
2730                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2731                                            OP_ALG_AAI_CTR_MOD128,
2732                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2733                                            OP_ALG_AAI_HMAC_PRECOMP,
2734                         .rfc3686 = true,
2735                 },
2736         },
2737         {
2738                 .aead = {
2739                         .base = {
2740                                 .cra_name = "seqiv(authenc(hmac(sha256),"
2741                                             "rfc3686(ctr(aes))))",
2742                                 .cra_driver_name = "seqiv-authenc-hmac-sha256-"
2743                                                    "rfc3686-ctr-aes-caam-qi2",
2744                                 .cra_blocksize = 1,
2745                         },
2746                         .setkey = aead_setkey,
2747                         .setauthsize = aead_setauthsize,
2748                         .encrypt = aead_encrypt,
2749                         .decrypt = aead_decrypt,
2750                         .ivsize = CTR_RFC3686_IV_SIZE,
2751                         .maxauthsize = SHA256_DIGEST_SIZE,
2752                 },
2753                 .caam = {
2754                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2755                                            OP_ALG_AAI_CTR_MOD128,
2756                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2757                                            OP_ALG_AAI_HMAC_PRECOMP,
2758                         .rfc3686 = true,
2759                         .geniv = true,
2760                 },
2761         },
2762         {
2763                 .aead = {
2764                         .base = {
2765                                 .cra_name = "authenc(hmac(sha384),"
2766                                             "rfc3686(ctr(aes)))",
2767                                 .cra_driver_name = "authenc-hmac-sha384-"
2768                                                    "rfc3686-ctr-aes-caam-qi2",
2769                                 .cra_blocksize = 1,
2770                         },
2771                         .setkey = aead_setkey,
2772                         .setauthsize = aead_setauthsize,
2773                         .encrypt = aead_encrypt,
2774                         .decrypt = aead_decrypt,
2775                         .ivsize = CTR_RFC3686_IV_SIZE,
2776                         .maxauthsize = SHA384_DIGEST_SIZE,
2777                 },
2778                 .caam = {
2779                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2780                                            OP_ALG_AAI_CTR_MOD128,
2781                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2782                                            OP_ALG_AAI_HMAC_PRECOMP,
2783                         .rfc3686 = true,
2784                 },
2785         },
2786         {
2787                 .aead = {
2788                         .base = {
2789                                 .cra_name = "seqiv(authenc(hmac(sha384),"
2790                                             "rfc3686(ctr(aes))))",
2791                                 .cra_driver_name = "seqiv-authenc-hmac-sha384-"
2792                                                    "rfc3686-ctr-aes-caam-qi2",
2793                                 .cra_blocksize = 1,
2794                         },
2795                         .setkey = aead_setkey,
2796                         .setauthsize = aead_setauthsize,
2797                         .encrypt = aead_encrypt,
2798                         .decrypt = aead_decrypt,
2799                         .ivsize = CTR_RFC3686_IV_SIZE,
2800                         .maxauthsize = SHA384_DIGEST_SIZE,
2801                 },
2802                 .caam = {
2803                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2804                                            OP_ALG_AAI_CTR_MOD128,
2805                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2806                                            OP_ALG_AAI_HMAC_PRECOMP,
2807                         .rfc3686 = true,
2808                         .geniv = true,
2809                 },
2810         },
2811         {
2812                 .aead = {
2813                         .base = {
2814                                 .cra_name = "rfc7539(chacha20,poly1305)",
2815                                 .cra_driver_name = "rfc7539-chacha20-poly1305-"
2816                                                    "caam-qi2",
2817                                 .cra_blocksize = 1,
2818                         },
2819                         .setkey = chachapoly_setkey,
2820                         .setauthsize = chachapoly_setauthsize,
2821                         .encrypt = aead_encrypt,
2822                         .decrypt = aead_decrypt,
2823                         .ivsize = CHACHAPOLY_IV_SIZE,
2824                         .maxauthsize = POLY1305_DIGEST_SIZE,
2825                 },
2826                 .caam = {
2827                         .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
2828                                            OP_ALG_AAI_AEAD,
2829                         .class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
2830                                            OP_ALG_AAI_AEAD,
2831                         .nodkp = true,
2832                 },
2833         },
2834         {
2835                 .aead = {
2836                         .base = {
2837                                 .cra_name = "rfc7539esp(chacha20,poly1305)",
2838                                 .cra_driver_name = "rfc7539esp-chacha20-"
2839                                                    "poly1305-caam-qi2",
2840                                 .cra_blocksize = 1,
2841                         },
2842                         .setkey = chachapoly_setkey,
2843                         .setauthsize = chachapoly_setauthsize,
2844                         .encrypt = aead_encrypt,
2845                         .decrypt = aead_decrypt,
2846                         .ivsize = 8,
2847                         .maxauthsize = POLY1305_DIGEST_SIZE,
2848                 },
2849                 .caam = {
2850                         .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
2851                                            OP_ALG_AAI_AEAD,
2852                         .class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
2853                                            OP_ALG_AAI_AEAD,
2854                         .nodkp = true,
2855                 },
2856         },
2857         {
2858                 .aead = {
2859                         .base = {
2860                                 .cra_name = "authenc(hmac(sha512),"
2861                                             "rfc3686(ctr(aes)))",
2862                                 .cra_driver_name = "authenc-hmac-sha512-"
2863                                                    "rfc3686-ctr-aes-caam-qi2",
2864                                 .cra_blocksize = 1,
2865                         },
2866                         .setkey = aead_setkey,
2867                         .setauthsize = aead_setauthsize,
2868                         .encrypt = aead_encrypt,
2869                         .decrypt = aead_decrypt,
2870                         .ivsize = CTR_RFC3686_IV_SIZE,
2871                         .maxauthsize = SHA512_DIGEST_SIZE,
2872                 },
2873                 .caam = {
2874                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2875                                            OP_ALG_AAI_CTR_MOD128,
2876                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2877                                            OP_ALG_AAI_HMAC_PRECOMP,
2878                         .rfc3686 = true,
2879                 },
2880         },
2881         {
2882                 .aead = {
2883                         .base = {
2884                                 .cra_name = "seqiv(authenc(hmac(sha512),"
2885                                             "rfc3686(ctr(aes))))",
2886                                 .cra_driver_name = "seqiv-authenc-hmac-sha512-"
2887                                                    "rfc3686-ctr-aes-caam-qi2",
2888                                 .cra_blocksize = 1,
2889                         },
2890                         .setkey = aead_setkey,
2891                         .setauthsize = aead_setauthsize,
2892                         .encrypt = aead_encrypt,
2893                         .decrypt = aead_decrypt,
2894                         .ivsize = CTR_RFC3686_IV_SIZE,
2895                         .maxauthsize = SHA512_DIGEST_SIZE,
2896                 },
2897                 .caam = {
2898                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2899                                            OP_ALG_AAI_CTR_MOD128,
2900                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2901                                            OP_ALG_AAI_HMAC_PRECOMP,
2902                         .rfc3686 = true,
2903                         .geniv = true,
2904                 },
2905         },
2906 };
2907
2908 static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg)
2909 {
2910         struct skcipher_alg *alg = &t_alg->skcipher;
2911
2912         alg->base.cra_module = THIS_MODULE;
2913         alg->base.cra_priority = CAAM_CRA_PRIORITY;
2914         alg->base.cra_ctxsize = sizeof(struct caam_ctx);
2915         alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
2916
2917         alg->init = caam_cra_init_skcipher;
2918         alg->exit = caam_cra_exit;
2919 }
2920
2921 static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
2922 {
2923         struct aead_alg *alg = &t_alg->aead;
2924
2925         alg->base.cra_module = THIS_MODULE;
2926         alg->base.cra_priority = CAAM_CRA_PRIORITY;
2927         alg->base.cra_ctxsize = sizeof(struct caam_ctx);
2928         alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
2929
2930         alg->init = caam_cra_init_aead;
2931         alg->exit = caam_cra_exit_aead;
2932 }
2933
2934 /* max hash key is max split key size */
2935 #define CAAM_MAX_HASH_KEY_SIZE          (SHA512_DIGEST_SIZE * 2)
2936
2937 #define CAAM_MAX_HASH_BLOCK_SIZE        SHA512_BLOCK_SIZE
2938
2939 /* caam context sizes for hashes: running digest + 8 */
2940 #define HASH_MSG_LEN                    8
2941 #define MAX_CTX_LEN                     (HASH_MSG_LEN + SHA512_DIGEST_SIZE)
2942
2943 enum hash_optype {
2944         UPDATE = 0,
2945         UPDATE_FIRST,
2946         FINALIZE,
2947         DIGEST,
2948         HASH_NUM_OP
2949 };
2950
2951 /**
2952  * caam_hash_ctx - ahash per-session context
2953  * @flc: Flow Contexts array
2954  * @key: authentication key
2955  * @flc_dma: I/O virtual addresses of the Flow Contexts
2956  * @dev: dpseci device
2957  * @ctx_len: size of Context Register
2958  * @adata: hashing algorithm details
2959  */
2960 struct caam_hash_ctx {
2961         struct caam_flc flc[HASH_NUM_OP];
2962         u8 key[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
2963         dma_addr_t flc_dma[HASH_NUM_OP];
2964         struct device *dev;
2965         int ctx_len;
2966         struct alginfo adata;
2967 };
2968
2969 /* ahash state */
2970 struct caam_hash_state {
2971         struct caam_request caam_req;
2972         dma_addr_t buf_dma;
2973         dma_addr_t ctx_dma;
2974         int ctx_dma_len;
2975         u8 buf[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
2976         int buflen;
2977         int next_buflen;
2978         u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned;
2979         int (*update)(struct ahash_request *req);
2980         int (*final)(struct ahash_request *req);
2981         int (*finup)(struct ahash_request *req);
2982 };
2983
2984 struct caam_export_state {
2985         u8 buf[CAAM_MAX_HASH_BLOCK_SIZE];
2986         u8 caam_ctx[MAX_CTX_LEN];
2987         int buflen;
2988         int (*update)(struct ahash_request *req);
2989         int (*final)(struct ahash_request *req);
2990         int (*finup)(struct ahash_request *req);
2991 };
2992
2993 /* Map current buffer in state (if length > 0) and put it in link table */
2994 static inline int buf_map_to_qm_sg(struct device *dev,
2995                                    struct dpaa2_sg_entry *qm_sg,
2996                                    struct caam_hash_state *state)
2997 {
2998         int buflen = state->buflen;
2999
3000         if (!buflen)
3001                 return 0;
3002
3003         state->buf_dma = dma_map_single(dev, state->buf, buflen,
3004                                         DMA_TO_DEVICE);
3005         if (dma_mapping_error(dev, state->buf_dma)) {
3006                 dev_err(dev, "unable to map buf\n");
3007                 state->buf_dma = 0;
3008                 return -ENOMEM;
3009         }
3010
3011         dma_to_qm_sg_one(qm_sg, state->buf_dma, buflen, 0);
3012
3013         return 0;
3014 }
3015
3016 /* Map state->caam_ctx, and add it to link table */
3017 static inline int ctx_map_to_qm_sg(struct device *dev,
3018                                    struct caam_hash_state *state, int ctx_len,
3019                                    struct dpaa2_sg_entry *qm_sg, u32 flag)
3020 {
3021         state->ctx_dma_len = ctx_len;
3022         state->ctx_dma = dma_map_single(dev, state->caam_ctx, ctx_len, flag);
3023         if (dma_mapping_error(dev, state->ctx_dma)) {
3024                 dev_err(dev, "unable to map ctx\n");
3025                 state->ctx_dma = 0;
3026                 return -ENOMEM;
3027         }
3028
3029         dma_to_qm_sg_one(qm_sg, state->ctx_dma, ctx_len, 0);
3030
3031         return 0;
3032 }
3033
3034 static int ahash_set_sh_desc(struct crypto_ahash *ahash)
3035 {
3036         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3037         int digestsize = crypto_ahash_digestsize(ahash);
3038         struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev);
3039         struct caam_flc *flc;
3040         u32 *desc;
3041
3042         /* ahash_update shared descriptor */
3043         flc = &ctx->flc[UPDATE];
3044         desc = flc->sh_desc;
3045         cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_UPDATE, ctx->ctx_len,
3046                           ctx->ctx_len, true, priv->sec_attr.era);
3047         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3048         dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE],
3049                                    desc_bytes(desc), DMA_BIDIRECTIONAL);
3050         print_hex_dump_debug("ahash update shdesc@" __stringify(__LINE__)": ",
3051                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3052                              1);
3053
3054         /* ahash_update_first shared descriptor */
3055         flc = &ctx->flc[UPDATE_FIRST];
3056         desc = flc->sh_desc;
3057         cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len,
3058                           ctx->ctx_len, false, priv->sec_attr.era);
3059         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3060         dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE_FIRST],
3061                                    desc_bytes(desc), DMA_BIDIRECTIONAL);
3062         print_hex_dump_debug("ahash update first shdesc@" __stringify(__LINE__)": ",
3063                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3064                              1);
3065
3066         /* ahash_final shared descriptor */
3067         flc = &ctx->flc[FINALIZE];
3068         desc = flc->sh_desc;
3069         cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, digestsize,
3070                           ctx->ctx_len, true, priv->sec_attr.era);
3071         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3072         dma_sync_single_for_device(ctx->dev, ctx->flc_dma[FINALIZE],
3073                                    desc_bytes(desc), DMA_BIDIRECTIONAL);
3074         print_hex_dump_debug("ahash final shdesc@" __stringify(__LINE__)": ",
3075                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3076                              1);
3077
3078         /* ahash_digest shared descriptor */
3079         flc = &ctx->flc[DIGEST];
3080         desc = flc->sh_desc;
3081         cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, digestsize,
3082                           ctx->ctx_len, false, priv->sec_attr.era);
3083         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3084         dma_sync_single_for_device(ctx->dev, ctx->flc_dma[DIGEST],
3085                                    desc_bytes(desc), DMA_BIDIRECTIONAL);
3086         print_hex_dump_debug("ahash digest shdesc@" __stringify(__LINE__)": ",
3087                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3088                              1);
3089
3090         return 0;
3091 }
3092
3093 struct split_key_sh_result {
3094         struct completion completion;
3095         int err;
3096         struct device *dev;
3097 };
3098
3099 static void split_key_sh_done(void *cbk_ctx, u32 err)
3100 {
3101         struct split_key_sh_result *res = cbk_ctx;
3102
3103         dev_dbg(res->dev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
3104
3105         res->err = err ? caam_qi2_strstatus(res->dev, err) : 0;
3106         complete(&res->completion);
3107 }
3108
3109 /* Digest hash size if it is too large */
3110 static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key,
3111                            u32 digestsize)
3112 {
3113         struct caam_request *req_ctx;
3114         u32 *desc;
3115         struct split_key_sh_result result;
3116         dma_addr_t key_dma;
3117         struct caam_flc *flc;
3118         dma_addr_t flc_dma;
3119         int ret = -ENOMEM;
3120         struct dpaa2_fl_entry *in_fle, *out_fle;
3121
3122         req_ctx = kzalloc(sizeof(*req_ctx), GFP_KERNEL | GFP_DMA);
3123         if (!req_ctx)
3124                 return -ENOMEM;
3125
3126         in_fle = &req_ctx->fd_flt[1];
3127         out_fle = &req_ctx->fd_flt[0];
3128
3129         flc = kzalloc(sizeof(*flc), GFP_KERNEL | GFP_DMA);
3130         if (!flc)
3131                 goto err_flc;
3132
3133         key_dma = dma_map_single(ctx->dev, key, *keylen, DMA_BIDIRECTIONAL);
3134         if (dma_mapping_error(ctx->dev, key_dma)) {
3135                 dev_err(ctx->dev, "unable to map key memory\n");
3136                 goto err_key_dma;
3137         }
3138
3139         desc = flc->sh_desc;
3140
3141         init_sh_desc(desc, 0);
3142
3143         /* descriptor to perform unkeyed hash on key_in */
3144         append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT |
3145                          OP_ALG_AS_INITFINAL);
3146         append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 |
3147                              FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG);
3148         append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
3149                          LDST_SRCDST_BYTE_CONTEXT);
3150
3151         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3152         flc_dma = dma_map_single(ctx->dev, flc, sizeof(flc->flc) +
3153                                  desc_bytes(desc), DMA_TO_DEVICE);
3154         if (dma_mapping_error(ctx->dev, flc_dma)) {
3155                 dev_err(ctx->dev, "unable to map shared descriptor\n");
3156                 goto err_flc_dma;
3157         }
3158
3159         dpaa2_fl_set_final(in_fle, true);
3160         dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
3161         dpaa2_fl_set_addr(in_fle, key_dma);
3162         dpaa2_fl_set_len(in_fle, *keylen);
3163         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3164         dpaa2_fl_set_addr(out_fle, key_dma);
3165         dpaa2_fl_set_len(out_fle, digestsize);
3166
3167         print_hex_dump_debug("key_in@" __stringify(__LINE__)": ",
3168                              DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1);
3169         print_hex_dump_debug("shdesc@" __stringify(__LINE__)": ",
3170                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3171                              1);
3172
3173         result.err = 0;
3174         init_completion(&result.completion);
3175         result.dev = ctx->dev;
3176
3177         req_ctx->flc = flc;
3178         req_ctx->flc_dma = flc_dma;
3179         req_ctx->cbk = split_key_sh_done;
3180         req_ctx->ctx = &result;
3181
3182         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3183         if (ret == -EINPROGRESS) {
3184                 /* in progress */
3185                 wait_for_completion(&result.completion);
3186                 ret = result.err;
3187                 print_hex_dump_debug("digested key@" __stringify(__LINE__)": ",
3188                                      DUMP_PREFIX_ADDRESS, 16, 4, key,
3189                                      digestsize, 1);
3190         }
3191
3192         dma_unmap_single(ctx->dev, flc_dma, sizeof(flc->flc) + desc_bytes(desc),
3193                          DMA_TO_DEVICE);
3194 err_flc_dma:
3195         dma_unmap_single(ctx->dev, key_dma, *keylen, DMA_BIDIRECTIONAL);
3196 err_key_dma:
3197         kfree(flc);
3198 err_flc:
3199         kfree(req_ctx);
3200
3201         *keylen = digestsize;
3202
3203         return ret;
3204 }
3205
3206 static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key,
3207                         unsigned int keylen)
3208 {
3209         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3210         unsigned int blocksize = crypto_tfm_alg_blocksize(&ahash->base);
3211         unsigned int digestsize = crypto_ahash_digestsize(ahash);
3212         int ret;
3213         u8 *hashed_key = NULL;
3214
3215         dev_dbg(ctx->dev, "keylen %d blocksize %d\n", keylen, blocksize);
3216
3217         if (keylen > blocksize) {
3218                 hashed_key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);
3219                 if (!hashed_key)
3220                         return -ENOMEM;
3221                 ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize);
3222                 if (ret)
3223                         goto bad_free_key;
3224                 key = hashed_key;
3225         }
3226
3227         ctx->adata.keylen = keylen;
3228         ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
3229                                               OP_ALG_ALGSEL_MASK);
3230         if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE)
3231                 goto bad_free_key;
3232
3233         ctx->adata.key_virt = key;
3234         ctx->adata.key_inline = true;
3235
3236         /*
3237          * In case |user key| > |derived key|, using DKP<imm,imm> would result
3238          * in invalid opcodes (last bytes of user key) in the resulting
3239          * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key
3240          * addresses are needed.
3241          */
3242         if (keylen > ctx->adata.keylen_pad) {
3243                 memcpy(ctx->key, key, keylen);
3244                 dma_sync_single_for_device(ctx->dev, ctx->adata.key_dma,
3245                                            ctx->adata.keylen_pad,
3246                                            DMA_TO_DEVICE);
3247         }
3248
3249         ret = ahash_set_sh_desc(ahash);
3250         kfree(hashed_key);
3251         return ret;
3252 bad_free_key:
3253         kfree(hashed_key);
3254         return -EINVAL;
3255 }
3256
3257 static inline void ahash_unmap(struct device *dev, struct ahash_edesc *edesc,
3258                                struct ahash_request *req)
3259 {
3260         struct caam_hash_state *state = ahash_request_ctx(req);
3261
3262         if (edesc->src_nents)
3263                 dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE);
3264
3265         if (edesc->qm_sg_bytes)
3266                 dma_unmap_single(dev, edesc->qm_sg_dma, edesc->qm_sg_bytes,
3267                                  DMA_TO_DEVICE);
3268
3269         if (state->buf_dma) {
3270                 dma_unmap_single(dev, state->buf_dma, state->buflen,
3271                                  DMA_TO_DEVICE);
3272                 state->buf_dma = 0;
3273         }
3274 }
3275
3276 static inline void ahash_unmap_ctx(struct device *dev,
3277                                    struct ahash_edesc *edesc,
3278                                    struct ahash_request *req, u32 flag)
3279 {
3280         struct caam_hash_state *state = ahash_request_ctx(req);
3281
3282         if (state->ctx_dma) {
3283                 dma_unmap_single(dev, state->ctx_dma, state->ctx_dma_len, flag);
3284                 state->ctx_dma = 0;
3285         }
3286         ahash_unmap(dev, edesc, req);
3287 }
3288
3289 static void ahash_done(void *cbk_ctx, u32 status)
3290 {
3291         struct crypto_async_request *areq = cbk_ctx;
3292         struct ahash_request *req = ahash_request_cast(areq);
3293         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3294         struct caam_hash_state *state = ahash_request_ctx(req);
3295         struct ahash_edesc *edesc = state->caam_req.edesc;
3296         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3297         int digestsize = crypto_ahash_digestsize(ahash);
3298         int ecode = 0;
3299
3300         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3301
3302         if (unlikely(status))
3303                 ecode = caam_qi2_strstatus(ctx->dev, status);
3304
3305         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3306         memcpy(req->result, state->caam_ctx, digestsize);
3307         qi_cache_free(edesc);
3308
3309         print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3310                              DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3311                              ctx->ctx_len, 1);
3312
3313         req->base.complete(&req->base, ecode);
3314 }
3315
3316 static void ahash_done_bi(void *cbk_ctx, u32 status)
3317 {
3318         struct crypto_async_request *areq = cbk_ctx;
3319         struct ahash_request *req = ahash_request_cast(areq);
3320         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3321         struct caam_hash_state *state = ahash_request_ctx(req);
3322         struct ahash_edesc *edesc = state->caam_req.edesc;
3323         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3324         int ecode = 0;
3325
3326         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3327
3328         if (unlikely(status))
3329                 ecode = caam_qi2_strstatus(ctx->dev, status);
3330
3331         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3332         qi_cache_free(edesc);
3333
3334         scatterwalk_map_and_copy(state->buf, req->src,
3335                                  req->nbytes - state->next_buflen,
3336                                  state->next_buflen, 0);
3337         state->buflen = state->next_buflen;
3338
3339         print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
3340                              DUMP_PREFIX_ADDRESS, 16, 4, state->buf,
3341                              state->buflen, 1);
3342
3343         print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3344                              DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3345                              ctx->ctx_len, 1);
3346         if (req->result)
3347                 print_hex_dump_debug("result@" __stringify(__LINE__)": ",
3348                                      DUMP_PREFIX_ADDRESS, 16, 4, req->result,
3349                                      crypto_ahash_digestsize(ahash), 1);
3350
3351         req->base.complete(&req->base, ecode);
3352 }
3353
3354 static void ahash_done_ctx_src(void *cbk_ctx, u32 status)
3355 {
3356         struct crypto_async_request *areq = cbk_ctx;
3357         struct ahash_request *req = ahash_request_cast(areq);
3358         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3359         struct caam_hash_state *state = ahash_request_ctx(req);
3360         struct ahash_edesc *edesc = state->caam_req.edesc;
3361         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3362         int digestsize = crypto_ahash_digestsize(ahash);
3363         int ecode = 0;
3364
3365         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3366
3367         if (unlikely(status))
3368                 ecode = caam_qi2_strstatus(ctx->dev, status);
3369
3370         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3371         memcpy(req->result, state->caam_ctx, digestsize);
3372         qi_cache_free(edesc);
3373
3374         print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3375                              DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3376                              ctx->ctx_len, 1);
3377
3378         req->base.complete(&req->base, ecode);
3379 }
3380
3381 static void ahash_done_ctx_dst(void *cbk_ctx, u32 status)
3382 {
3383         struct crypto_async_request *areq = cbk_ctx;
3384         struct ahash_request *req = ahash_request_cast(areq);
3385         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3386         struct caam_hash_state *state = ahash_request_ctx(req);
3387         struct ahash_edesc *edesc = state->caam_req.edesc;
3388         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3389         int ecode = 0;
3390
3391         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3392
3393         if (unlikely(status))
3394                 ecode = caam_qi2_strstatus(ctx->dev, status);
3395
3396         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3397         qi_cache_free(edesc);
3398
3399         scatterwalk_map_and_copy(state->buf, req->src,
3400                                  req->nbytes - state->next_buflen,
3401                                  state->next_buflen, 0);
3402         state->buflen = state->next_buflen;
3403
3404         print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
3405                              DUMP_PREFIX_ADDRESS, 16, 4, state->buf,
3406                              state->buflen, 1);
3407
3408         print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3409                              DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3410                              ctx->ctx_len, 1);
3411         if (req->result)
3412                 print_hex_dump_debug("result@" __stringify(__LINE__)": ",
3413                                      DUMP_PREFIX_ADDRESS, 16, 4, req->result,
3414                                      crypto_ahash_digestsize(ahash), 1);
3415
3416         req->base.complete(&req->base, ecode);
3417 }
3418
3419 static int ahash_update_ctx(struct ahash_request *req)
3420 {
3421         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3422         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3423         struct caam_hash_state *state = ahash_request_ctx(req);
3424         struct caam_request *req_ctx = &state->caam_req;
3425         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3426         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3427         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3428                       GFP_KERNEL : GFP_ATOMIC;
3429         u8 *buf = state->buf;
3430         int *buflen = &state->buflen;
3431         int *next_buflen = &state->next_buflen;
3432         int in_len = *buflen + req->nbytes, to_hash;
3433         int src_nents, mapped_nents, qm_sg_bytes, qm_sg_src_index;
3434         struct ahash_edesc *edesc;
3435         int ret = 0;
3436
3437         *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
3438         to_hash = in_len - *next_buflen;
3439
3440         if (to_hash) {
3441                 struct dpaa2_sg_entry *sg_table;
3442                 int src_len = req->nbytes - *next_buflen;
3443
3444                 src_nents = sg_nents_for_len(req->src, src_len);
3445                 if (src_nents < 0) {
3446                         dev_err(ctx->dev, "Invalid number of src SG.\n");
3447                         return src_nents;
3448                 }
3449
3450                 if (src_nents) {
3451                         mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
3452                                                   DMA_TO_DEVICE);
3453                         if (!mapped_nents) {
3454                                 dev_err(ctx->dev, "unable to DMA map source\n");
3455                                 return -ENOMEM;
3456                         }
3457                 } else {
3458                         mapped_nents = 0;
3459                 }
3460
3461                 /* allocate space for base edesc and link tables */
3462                 edesc = qi_cache_zalloc(GFP_DMA | flags);
3463                 if (!edesc) {
3464                         dma_unmap_sg(ctx->dev, req->src, src_nents,
3465                                      DMA_TO_DEVICE);
3466                         return -ENOMEM;
3467                 }
3468
3469                 edesc->src_nents = src_nents;
3470                 qm_sg_src_index = 1 + (*buflen ? 1 : 0);
3471                 qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) *
3472                               sizeof(*sg_table);
3473                 sg_table = &edesc->sgt[0];
3474
3475                 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
3476                                        DMA_BIDIRECTIONAL);
3477                 if (ret)
3478                         goto unmap_ctx;
3479
3480                 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
3481                 if (ret)
3482                         goto unmap_ctx;
3483
3484                 if (mapped_nents) {
3485                         sg_to_qm_sg_last(req->src, src_len,
3486                                          sg_table + qm_sg_src_index, 0);
3487                 } else {
3488                         dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1,
3489                                            true);
3490                 }
3491
3492                 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
3493                                                   qm_sg_bytes, DMA_TO_DEVICE);
3494                 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3495                         dev_err(ctx->dev, "unable to map S/G table\n");
3496                         ret = -ENOMEM;
3497                         goto unmap_ctx;
3498                 }
3499                 edesc->qm_sg_bytes = qm_sg_bytes;
3500
3501                 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3502                 dpaa2_fl_set_final(in_fle, true);
3503                 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3504                 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3505                 dpaa2_fl_set_len(in_fle, ctx->ctx_len + to_hash);
3506                 dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3507                 dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3508                 dpaa2_fl_set_len(out_fle, ctx->ctx_len);
3509
3510                 req_ctx->flc = &ctx->flc[UPDATE];
3511                 req_ctx->flc_dma = ctx->flc_dma[UPDATE];
3512                 req_ctx->cbk = ahash_done_bi;
3513                 req_ctx->ctx = &req->base;
3514                 req_ctx->edesc = edesc;
3515
3516                 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3517                 if (ret != -EINPROGRESS &&
3518                     !(ret == -EBUSY &&
3519                       req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3520                         goto unmap_ctx;
3521         } else if (*next_buflen) {
3522                 scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
3523                                          req->nbytes, 0);
3524                 *buflen = *next_buflen;
3525
3526                 print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
3527                                      DUMP_PREFIX_ADDRESS, 16, 4, buf,
3528                                      *buflen, 1);
3529         }
3530
3531         return ret;
3532 unmap_ctx:
3533         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3534         qi_cache_free(edesc);
3535         return ret;
3536 }
3537
3538 static int ahash_final_ctx(struct ahash_request *req)
3539 {
3540         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3541         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3542         struct caam_hash_state *state = ahash_request_ctx(req);
3543         struct caam_request *req_ctx = &state->caam_req;
3544         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3545         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3546         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3547                       GFP_KERNEL : GFP_ATOMIC;
3548         int buflen = state->buflen;
3549         int qm_sg_bytes;
3550         int digestsize = crypto_ahash_digestsize(ahash);
3551         struct ahash_edesc *edesc;
3552         struct dpaa2_sg_entry *sg_table;
3553         int ret;
3554
3555         /* allocate space for base edesc and link tables */
3556         edesc = qi_cache_zalloc(GFP_DMA | flags);
3557         if (!edesc)
3558                 return -ENOMEM;
3559
3560         qm_sg_bytes = pad_sg_nents(1 + (buflen ? 1 : 0)) * sizeof(*sg_table);
3561         sg_table = &edesc->sgt[0];
3562
3563         ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
3564                                DMA_BIDIRECTIONAL);
3565         if (ret)
3566                 goto unmap_ctx;
3567
3568         ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
3569         if (ret)
3570                 goto unmap_ctx;
3571
3572         dpaa2_sg_set_final(sg_table + (buflen ? 1 : 0), true);
3573
3574         edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
3575                                           DMA_TO_DEVICE);
3576         if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3577                 dev_err(ctx->dev, "unable to map S/G table\n");
3578                 ret = -ENOMEM;
3579                 goto unmap_ctx;
3580         }
3581         edesc->qm_sg_bytes = qm_sg_bytes;
3582
3583         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3584         dpaa2_fl_set_final(in_fle, true);
3585         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3586         dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3587         dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen);
3588         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3589         dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3590         dpaa2_fl_set_len(out_fle, digestsize);
3591
3592         req_ctx->flc = &ctx->flc[FINALIZE];
3593         req_ctx->flc_dma = ctx->flc_dma[FINALIZE];
3594         req_ctx->cbk = ahash_done_ctx_src;
3595         req_ctx->ctx = &req->base;
3596         req_ctx->edesc = edesc;
3597
3598         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3599         if (ret == -EINPROGRESS ||
3600             (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3601                 return ret;
3602
3603 unmap_ctx:
3604         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3605         qi_cache_free(edesc);
3606         return ret;
3607 }
3608
3609 static int ahash_finup_ctx(struct ahash_request *req)
3610 {
3611         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3612         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3613         struct caam_hash_state *state = ahash_request_ctx(req);
3614         struct caam_request *req_ctx = &state->caam_req;
3615         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3616         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3617         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3618                       GFP_KERNEL : GFP_ATOMIC;
3619         int buflen = state->buflen;
3620         int qm_sg_bytes, qm_sg_src_index;
3621         int src_nents, mapped_nents;
3622         int digestsize = crypto_ahash_digestsize(ahash);
3623         struct ahash_edesc *edesc;
3624         struct dpaa2_sg_entry *sg_table;
3625         int ret;
3626
3627         src_nents = sg_nents_for_len(req->src, req->nbytes);
3628         if (src_nents < 0) {
3629                 dev_err(ctx->dev, "Invalid number of src SG.\n");
3630                 return src_nents;
3631         }
3632
3633         if (src_nents) {
3634                 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
3635                                           DMA_TO_DEVICE);
3636                 if (!mapped_nents) {
3637                         dev_err(ctx->dev, "unable to DMA map source\n");
3638                         return -ENOMEM;
3639                 }
3640         } else {
3641                 mapped_nents = 0;
3642         }
3643
3644         /* allocate space for base edesc and link tables */
3645         edesc = qi_cache_zalloc(GFP_DMA | flags);
3646         if (!edesc) {
3647                 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
3648                 return -ENOMEM;
3649         }
3650
3651         edesc->src_nents = src_nents;
3652         qm_sg_src_index = 1 + (buflen ? 1 : 0);
3653         qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) *
3654                       sizeof(*sg_table);
3655         sg_table = &edesc->sgt[0];
3656
3657         ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
3658                                DMA_BIDIRECTIONAL);
3659         if (ret)
3660                 goto unmap_ctx;
3661
3662         ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
3663         if (ret)
3664                 goto unmap_ctx;
3665
3666         sg_to_qm_sg_last(req->src, req->nbytes, sg_table + qm_sg_src_index, 0);
3667
3668         edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
3669                                           DMA_TO_DEVICE);
3670         if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3671                 dev_err(ctx->dev, "unable to map S/G table\n");
3672                 ret = -ENOMEM;
3673                 goto unmap_ctx;
3674         }
3675         edesc->qm_sg_bytes = qm_sg_bytes;
3676
3677         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3678         dpaa2_fl_set_final(in_fle, true);
3679         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3680         dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3681         dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen + req->nbytes);
3682         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3683         dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3684         dpaa2_fl_set_len(out_fle, digestsize);
3685
3686         req_ctx->flc = &ctx->flc[FINALIZE];
3687         req_ctx->flc_dma = ctx->flc_dma[FINALIZE];
3688         req_ctx->cbk = ahash_done_ctx_src;
3689         req_ctx->ctx = &req->base;
3690         req_ctx->edesc = edesc;
3691
3692         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3693         if (ret == -EINPROGRESS ||
3694             (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3695                 return ret;
3696
3697 unmap_ctx:
3698         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3699         qi_cache_free(edesc);
3700         return ret;
3701 }
3702
3703 static int ahash_digest(struct ahash_request *req)
3704 {
3705         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3706         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3707         struct caam_hash_state *state = ahash_request_ctx(req);
3708         struct caam_request *req_ctx = &state->caam_req;
3709         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3710         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3711         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3712                       GFP_KERNEL : GFP_ATOMIC;
3713         int digestsize = crypto_ahash_digestsize(ahash);
3714         int src_nents, mapped_nents;
3715         struct ahash_edesc *edesc;
3716         int ret = -ENOMEM;
3717
3718         state->buf_dma = 0;
3719
3720         src_nents = sg_nents_for_len(req->src, req->nbytes);
3721         if (src_nents < 0) {
3722                 dev_err(ctx->dev, "Invalid number of src SG.\n");
3723                 return src_nents;
3724         }
3725
3726         if (src_nents) {
3727                 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
3728                                           DMA_TO_DEVICE);
3729                 if (!mapped_nents) {
3730                         dev_err(ctx->dev, "unable to map source for DMA\n");
3731                         return ret;
3732                 }
3733         } else {
3734                 mapped_nents = 0;
3735         }
3736
3737         /* allocate space for base edesc and link tables */
3738         edesc = qi_cache_zalloc(GFP_DMA | flags);
3739         if (!edesc) {
3740                 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
3741                 return ret;
3742         }
3743
3744         edesc->src_nents = src_nents;
3745         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3746
3747         if (mapped_nents > 1) {
3748                 int qm_sg_bytes;
3749                 struct dpaa2_sg_entry *sg_table = &edesc->sgt[0];
3750
3751                 qm_sg_bytes = pad_sg_nents(mapped_nents) * sizeof(*sg_table);
3752                 sg_to_qm_sg_last(req->src, req->nbytes, sg_table, 0);
3753                 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
3754                                                   qm_sg_bytes, DMA_TO_DEVICE);
3755                 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3756                         dev_err(ctx->dev, "unable to map S/G table\n");
3757                         goto unmap;
3758                 }
3759                 edesc->qm_sg_bytes = qm_sg_bytes;
3760                 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3761                 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3762         } else {
3763                 dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
3764                 dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src));
3765         }
3766
3767         state->ctx_dma_len = digestsize;
3768         state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize,
3769                                         DMA_FROM_DEVICE);
3770         if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
3771                 dev_err(ctx->dev, "unable to map ctx\n");
3772                 state->ctx_dma = 0;
3773                 goto unmap;
3774         }
3775
3776         dpaa2_fl_set_final(in_fle, true);
3777         dpaa2_fl_set_len(in_fle, req->nbytes);
3778         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3779         dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3780         dpaa2_fl_set_len(out_fle, digestsize);
3781
3782         req_ctx->flc = &ctx->flc[DIGEST];
3783         req_ctx->flc_dma = ctx->flc_dma[DIGEST];
3784         req_ctx->cbk = ahash_done;
3785         req_ctx->ctx = &req->base;
3786         req_ctx->edesc = edesc;
3787         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3788         if (ret == -EINPROGRESS ||
3789             (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3790                 return ret;
3791
3792 unmap:
3793         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3794         qi_cache_free(edesc);
3795         return ret;
3796 }
3797
3798 static int ahash_final_no_ctx(struct ahash_request *req)
3799 {
3800         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3801         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3802         struct caam_hash_state *state = ahash_request_ctx(req);
3803         struct caam_request *req_ctx = &state->caam_req;
3804         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3805         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3806         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3807                       GFP_KERNEL : GFP_ATOMIC;
3808         u8 *buf = state->buf;
3809         int buflen = state->buflen;
3810         int digestsize = crypto_ahash_digestsize(ahash);
3811         struct ahash_edesc *edesc;
3812         int ret = -ENOMEM;
3813
3814         /* allocate space for base edesc and link tables */
3815         edesc = qi_cache_zalloc(GFP_DMA | flags);
3816         if (!edesc)
3817                 return ret;
3818
3819         if (buflen) {
3820                 state->buf_dma = dma_map_single(ctx->dev, buf, buflen,
3821                                                 DMA_TO_DEVICE);
3822                 if (dma_mapping_error(ctx->dev, state->buf_dma)) {
3823                         dev_err(ctx->dev, "unable to map src\n");
3824                         goto unmap;
3825                 }
3826         }
3827
3828         state->ctx_dma_len = digestsize;
3829         state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize,
3830                                         DMA_FROM_DEVICE);
3831         if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
3832                 dev_err(ctx->dev, "unable to map ctx\n");
3833                 state->ctx_dma = 0;
3834                 goto unmap;
3835         }
3836
3837         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3838         dpaa2_fl_set_final(in_fle, true);
3839         /*
3840          * crypto engine requires the input entry to be present when
3841          * "frame list" FD is used.
3842          * Since engine does not support FMT=2'b11 (unused entry type), leaving
3843          * in_fle zeroized (except for "Final" flag) is the best option.
3844          */
3845         if (buflen) {
3846                 dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
3847                 dpaa2_fl_set_addr(in_fle, state->buf_dma);
3848                 dpaa2_fl_set_len(in_fle, buflen);
3849         }
3850         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3851         dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3852         dpaa2_fl_set_len(out_fle, digestsize);
3853
3854         req_ctx->flc = &ctx->flc[DIGEST];
3855         req_ctx->flc_dma = ctx->flc_dma[DIGEST];
3856         req_ctx->cbk = ahash_done;
3857         req_ctx->ctx = &req->base;
3858         req_ctx->edesc = edesc;
3859
3860         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3861         if (ret == -EINPROGRESS ||
3862             (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3863                 return ret;
3864
3865 unmap:
3866         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3867         qi_cache_free(edesc);
3868         return ret;
3869 }
3870
3871 static int ahash_update_no_ctx(struct ahash_request *req)
3872 {
3873         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3874         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3875         struct caam_hash_state *state = ahash_request_ctx(req);
3876         struct caam_request *req_ctx = &state->caam_req;
3877         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3878         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3879         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3880                       GFP_KERNEL : GFP_ATOMIC;
3881         u8 *buf = state->buf;
3882         int *buflen = &state->buflen;
3883         int *next_buflen = &state->next_buflen;
3884         int in_len = *buflen + req->nbytes, to_hash;
3885         int qm_sg_bytes, src_nents, mapped_nents;
3886         struct ahash_edesc *edesc;
3887         int ret = 0;
3888
3889         *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
3890         to_hash = in_len - *next_buflen;
3891
3892         if (to_hash) {
3893                 struct dpaa2_sg_entry *sg_table;
3894                 int src_len = req->nbytes - *next_buflen;
3895
3896                 src_nents = sg_nents_for_len(req->src, src_len);
3897                 if (src_nents < 0) {
3898                         dev_err(ctx->dev, "Invalid number of src SG.\n");
3899                         return src_nents;
3900                 }
3901
3902                 if (src_nents) {
3903                         mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
3904                                                   DMA_TO_DEVICE);
3905                         if (!mapped_nents) {
3906                                 dev_err(ctx->dev, "unable to DMA map source\n");
3907                                 return -ENOMEM;
3908                         }
3909                 } else {
3910                         mapped_nents = 0;
3911                 }
3912
3913                 /* allocate space for base edesc and link tables */
3914                 edesc = qi_cache_zalloc(GFP_DMA | flags);
3915                 if (!edesc) {
3916                         dma_unmap_sg(ctx->dev, req->src, src_nents,
3917                                      DMA_TO_DEVICE);
3918                         return -ENOMEM;
3919                 }
3920
3921                 edesc->src_nents = src_nents;
3922                 qm_sg_bytes = pad_sg_nents(1 + mapped_nents) *
3923                               sizeof(*sg_table);
3924                 sg_table = &edesc->sgt[0];
3925
3926                 ret = buf_map_to_qm_sg(ctx->dev, sg_table, state);
3927                 if (ret)
3928                         goto unmap_ctx;
3929
3930                 sg_to_qm_sg_last(req->src, src_len, sg_table + 1, 0);
3931
3932                 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
3933                                                   qm_sg_bytes, DMA_TO_DEVICE);
3934                 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3935                         dev_err(ctx->dev, "unable to map S/G table\n");
3936                         ret = -ENOMEM;
3937                         goto unmap_ctx;
3938                 }
3939                 edesc->qm_sg_bytes = qm_sg_bytes;
3940
3941                 state->ctx_dma_len = ctx->ctx_len;
3942                 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx,
3943                                                 ctx->ctx_len, DMA_FROM_DEVICE);
3944                 if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
3945                         dev_err(ctx->dev, "unable to map ctx\n");
3946                         state->ctx_dma = 0;
3947                         ret = -ENOMEM;
3948                         goto unmap_ctx;
3949                 }
3950
3951                 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3952                 dpaa2_fl_set_final(in_fle, true);
3953                 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3954                 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3955                 dpaa2_fl_set_len(in_fle, to_hash);
3956                 dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3957                 dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3958                 dpaa2_fl_set_len(out_fle, ctx->ctx_len);
3959
3960                 req_ctx->flc = &ctx->flc[UPDATE_FIRST];
3961                 req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST];
3962                 req_ctx->cbk = ahash_done_ctx_dst;
3963                 req_ctx->ctx = &req->base;
3964                 req_ctx->edesc = edesc;
3965
3966                 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3967                 if (ret != -EINPROGRESS &&
3968                     !(ret == -EBUSY &&
3969                       req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3970                         goto unmap_ctx;
3971
3972                 state->update = ahash_update_ctx;
3973                 state->finup = ahash_finup_ctx;
3974                 state->final = ahash_final_ctx;
3975         } else if (*next_buflen) {
3976                 scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
3977                                          req->nbytes, 0);
3978                 *buflen = *next_buflen;
3979
3980                 print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
3981                                      DUMP_PREFIX_ADDRESS, 16, 4, buf,
3982                                      *buflen, 1);
3983         }
3984
3985         return ret;
3986 unmap_ctx:
3987         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE);
3988         qi_cache_free(edesc);
3989         return ret;
3990 }
3991
3992 static int ahash_finup_no_ctx(struct ahash_request *req)
3993 {
3994         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3995         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3996         struct caam_hash_state *state = ahash_request_ctx(req);
3997         struct caam_request *req_ctx = &state->caam_req;
3998         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3999         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
4000         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
4001                       GFP_KERNEL : GFP_ATOMIC;
4002         int buflen = state->buflen;
4003         int qm_sg_bytes, src_nents, mapped_nents;
4004         int digestsize = crypto_ahash_digestsize(ahash);
4005         struct ahash_edesc *edesc;
4006         struct dpaa2_sg_entry *sg_table;
4007         int ret;
4008
4009         src_nents = sg_nents_for_len(req->src, req->nbytes);
4010         if (src_nents < 0) {
4011                 dev_err(ctx->dev, "Invalid number of src SG.\n");
4012                 return src_nents;
4013         }
4014
4015         if (src_nents) {
4016                 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
4017                                           DMA_TO_DEVICE);
4018                 if (!mapped_nents) {
4019                         dev_err(ctx->dev, "unable to DMA map source\n");
4020                         return -ENOMEM;
4021                 }
4022         } else {
4023                 mapped_nents = 0;
4024         }
4025
4026         /* allocate space for base edesc and link tables */
4027         edesc = qi_cache_zalloc(GFP_DMA | flags);
4028         if (!edesc) {
4029                 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
4030                 return -ENOMEM;
4031         }
4032
4033         edesc->src_nents = src_nents;
4034         qm_sg_bytes = pad_sg_nents(2 + mapped_nents) * sizeof(*sg_table);
4035         sg_table = &edesc->sgt[0];
4036
4037         ret = buf_map_to_qm_sg(ctx->dev, sg_table, state);
4038         if (ret)
4039                 goto unmap;
4040
4041         sg_to_qm_sg_last(req->src, req->nbytes, sg_table + 1, 0);
4042
4043         edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
4044                                           DMA_TO_DEVICE);
4045         if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
4046                 dev_err(ctx->dev, "unable to map S/G table\n");
4047                 ret = -ENOMEM;
4048                 goto unmap;
4049         }
4050         edesc->qm_sg_bytes = qm_sg_bytes;
4051
4052         state->ctx_dma_len = digestsize;
4053         state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize,
4054                                         DMA_FROM_DEVICE);
4055         if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
4056                 dev_err(ctx->dev, "unable to map ctx\n");
4057                 state->ctx_dma = 0;
4058                 ret = -ENOMEM;
4059                 goto unmap;
4060         }
4061
4062         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
4063         dpaa2_fl_set_final(in_fle, true);
4064         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
4065         dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
4066         dpaa2_fl_set_len(in_fle, buflen + req->nbytes);
4067         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
4068         dpaa2_fl_set_addr(out_fle, state->ctx_dma);
4069         dpaa2_fl_set_len(out_fle, digestsize);
4070
4071         req_ctx->flc = &ctx->flc[DIGEST];
4072         req_ctx->flc_dma = ctx->flc_dma[DIGEST];
4073         req_ctx->cbk = ahash_done;
4074         req_ctx->ctx = &req->base;
4075         req_ctx->edesc = edesc;
4076         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
4077         if (ret != -EINPROGRESS &&
4078             !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
4079                 goto unmap;
4080
4081         return ret;
4082 unmap:
4083         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
4084         qi_cache_free(edesc);
4085         return -ENOMEM;
4086 }
4087
4088 static int ahash_update_first(struct ahash_request *req)
4089 {
4090         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
4091         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
4092         struct caam_hash_state *state = ahash_request_ctx(req);
4093         struct caam_request *req_ctx = &state->caam_req;
4094         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
4095         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
4096         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
4097                       GFP_KERNEL : GFP_ATOMIC;
4098         u8 *buf = state->buf;
4099         int *buflen = &state->buflen;
4100         int *next_buflen = &state->next_buflen;
4101         int to_hash;
4102         int src_nents, mapped_nents;
4103         struct ahash_edesc *edesc;
4104         int ret = 0;
4105
4106         *next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) -
4107                                       1);
4108         to_hash = req->nbytes - *next_buflen;
4109
4110         if (to_hash) {
4111                 struct dpaa2_sg_entry *sg_table;
4112                 int src_len = req->nbytes - *next_buflen;
4113
4114                 src_nents = sg_nents_for_len(req->src, src_len);
4115                 if (src_nents < 0) {
4116                         dev_err(ctx->dev, "Invalid number of src SG.\n");
4117                         return src_nents;
4118                 }
4119
4120                 if (src_nents) {
4121                         mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
4122                                                   DMA_TO_DEVICE);
4123                         if (!mapped_nents) {
4124                                 dev_err(ctx->dev, "unable to map source for DMA\n");
4125                                 return -ENOMEM;
4126                         }
4127                 } else {
4128                         mapped_nents = 0;
4129                 }
4130
4131                 /* allocate space for base edesc and link tables */
4132                 edesc = qi_cache_zalloc(GFP_DMA | flags);
4133                 if (!edesc) {
4134                         dma_unmap_sg(ctx->dev, req->src, src_nents,
4135                                      DMA_TO_DEVICE);
4136                         return -ENOMEM;
4137                 }
4138
4139                 edesc->src_nents = src_nents;
4140                 sg_table = &edesc->sgt[0];
4141
4142                 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
4143                 dpaa2_fl_set_final(in_fle, true);
4144                 dpaa2_fl_set_len(in_fle, to_hash);
4145
4146                 if (mapped_nents > 1) {
4147                         int qm_sg_bytes;
4148
4149                         sg_to_qm_sg_last(req->src, src_len, sg_table, 0);
4150                         qm_sg_bytes = pad_sg_nents(mapped_nents) *
4151                                       sizeof(*sg_table);
4152                         edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
4153                                                           qm_sg_bytes,
4154                                                           DMA_TO_DEVICE);
4155                         if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
4156                                 dev_err(ctx->dev, "unable to map S/G table\n");
4157                                 ret = -ENOMEM;
4158                                 goto unmap_ctx;
4159                         }
4160                         edesc->qm_sg_bytes = qm_sg_bytes;
4161                         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
4162                         dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
4163                 } else {
4164                         dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
4165                         dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src));
4166                 }
4167
4168                 state->ctx_dma_len = ctx->ctx_len;
4169                 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx,
4170                                                 ctx->ctx_len, DMA_FROM_DEVICE);
4171                 if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
4172                         dev_err(ctx->dev, "unable to map ctx\n");
4173                         state->ctx_dma = 0;
4174                         ret = -ENOMEM;
4175                         goto unmap_ctx;
4176                 }
4177
4178                 dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
4179                 dpaa2_fl_set_addr(out_fle, state->ctx_dma);
4180                 dpaa2_fl_set_len(out_fle, ctx->ctx_len);
4181
4182                 req_ctx->flc = &ctx->flc[UPDATE_FIRST];
4183                 req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST];
4184                 req_ctx->cbk = ahash_done_ctx_dst;
4185                 req_ctx->ctx = &req->base;
4186                 req_ctx->edesc = edesc;
4187
4188                 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
4189                 if (ret != -EINPROGRESS &&
4190                     !(ret == -EBUSY && req->base.flags &
4191                       CRYPTO_TFM_REQ_MAY_BACKLOG))
4192                         goto unmap_ctx;
4193
4194                 state->update = ahash_update_ctx;
4195                 state->finup = ahash_finup_ctx;
4196                 state->final = ahash_final_ctx;
4197         } else if (*next_buflen) {
4198                 state->update = ahash_update_no_ctx;
4199                 state->finup = ahash_finup_no_ctx;
4200                 state->final = ahash_final_no_ctx;
4201                 scatterwalk_map_and_copy(buf, req->src, 0,
4202                                          req->nbytes, 0);
4203                 *buflen = *next_buflen;
4204
4205                 print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
4206                                      DUMP_PREFIX_ADDRESS, 16, 4, buf,
4207                                      *buflen, 1);
4208         }
4209
4210         return ret;
4211 unmap_ctx:
4212         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE);
4213         qi_cache_free(edesc);
4214         return ret;
4215 }
4216
4217 static int ahash_finup_first(struct ahash_request *req)
4218 {
4219         return ahash_digest(req);
4220 }
4221
4222 static int ahash_init(struct ahash_request *req)
4223 {
4224         struct caam_hash_state *state = ahash_request_ctx(req);
4225
4226         state->update = ahash_update_first;
4227         state->finup = ahash_finup_first;
4228         state->final = ahash_final_no_ctx;
4229
4230         state->ctx_dma = 0;
4231         state->ctx_dma_len = 0;
4232         state->buf_dma = 0;
4233         state->buflen = 0;
4234         state->next_buflen = 0;
4235
4236         return 0;
4237 }
4238
4239 static int ahash_update(struct ahash_request *req)
4240 {
4241         struct caam_hash_state *state = ahash_request_ctx(req);
4242
4243         return state->update(req);
4244 }
4245
4246 static int ahash_finup(struct ahash_request *req)
4247 {
4248         struct caam_hash_state *state = ahash_request_ctx(req);
4249
4250         return state->finup(req);
4251 }
4252
4253 static int ahash_final(struct ahash_request *req)
4254 {
4255         struct caam_hash_state *state = ahash_request_ctx(req);
4256
4257         return state->final(req);
4258 }
4259
4260 static int ahash_export(struct ahash_request *req, void *out)
4261 {
4262         struct caam_hash_state *state = ahash_request_ctx(req);
4263         struct caam_export_state *export = out;
4264         u8 *buf = state->buf;
4265         int len = state->buflen;
4266
4267         memcpy(export->buf, buf, len);
4268         memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx));
4269         export->buflen = len;
4270         export->update = state->update;
4271         export->final = state->final;
4272         export->finup = state->finup;
4273
4274         return 0;
4275 }
4276
4277 static int ahash_import(struct ahash_request *req, const void *in)
4278 {
4279         struct caam_hash_state *state = ahash_request_ctx(req);
4280         const struct caam_export_state *export = in;
4281
4282         memset(state, 0, sizeof(*state));
4283         memcpy(state->buf, export->buf, export->buflen);
4284         memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx));
4285         state->buflen = export->buflen;
4286         state->update = export->update;
4287         state->final = export->final;
4288         state->finup = export->finup;
4289
4290         return 0;
4291 }
4292
4293 struct caam_hash_template {
4294         char name[CRYPTO_MAX_ALG_NAME];
4295         char driver_name[CRYPTO_MAX_ALG_NAME];
4296         char hmac_name[CRYPTO_MAX_ALG_NAME];
4297         char hmac_driver_name[CRYPTO_MAX_ALG_NAME];
4298         unsigned int blocksize;
4299         struct ahash_alg template_ahash;
4300         u32 alg_type;
4301 };
4302
4303 /* ahash descriptors */
4304 static struct caam_hash_template driver_hash[] = {
4305         {
4306                 .name = "sha1",
4307                 .driver_name = "sha1-caam-qi2",
4308                 .hmac_name = "hmac(sha1)",
4309                 .hmac_driver_name = "hmac-sha1-caam-qi2",
4310                 .blocksize = SHA1_BLOCK_SIZE,
4311                 .template_ahash = {
4312                         .init = ahash_init,
4313                         .update = ahash_update,
4314                         .final = ahash_final,
4315                         .finup = ahash_finup,
4316                         .digest = ahash_digest,
4317                         .export = ahash_export,
4318                         .import = ahash_import,
4319                         .setkey = ahash_setkey,
4320                         .halg = {
4321                                 .digestsize = SHA1_DIGEST_SIZE,
4322                                 .statesize = sizeof(struct caam_export_state),
4323                         },
4324                 },
4325                 .alg_type = OP_ALG_ALGSEL_SHA1,
4326         }, {
4327                 .name = "sha224",
4328                 .driver_name = "sha224-caam-qi2",
4329                 .hmac_name = "hmac(sha224)",
4330                 .hmac_driver_name = "hmac-sha224-caam-qi2",
4331                 .blocksize = SHA224_BLOCK_SIZE,
4332                 .template_ahash = {
4333                         .init = ahash_init,
4334                         .update = ahash_update,
4335                         .final = ahash_final,
4336                         .finup = ahash_finup,
4337                         .digest = ahash_digest,
4338                         .export = ahash_export,
4339                         .import = ahash_import,
4340                         .setkey = ahash_setkey,
4341                         .halg = {
4342                                 .digestsize = SHA224_DIGEST_SIZE,
4343                                 .statesize = sizeof(struct caam_export_state),
4344                         },
4345                 },
4346                 .alg_type = OP_ALG_ALGSEL_SHA224,
4347         }, {
4348                 .name = "sha256",
4349                 .driver_name = "sha256-caam-qi2",
4350                 .hmac_name = "hmac(sha256)",
4351                 .hmac_driver_name = "hmac-sha256-caam-qi2",
4352                 .blocksize = SHA256_BLOCK_SIZE,
4353                 .template_ahash = {
4354                         .init = ahash_init,
4355                         .update = ahash_update,
4356                         .final = ahash_final,
4357                         .finup = ahash_finup,
4358                         .digest = ahash_digest,
4359                         .export = ahash_export,
4360                         .import = ahash_import,
4361                         .setkey = ahash_setkey,
4362                         .halg = {
4363                                 .digestsize = SHA256_DIGEST_SIZE,
4364                                 .statesize = sizeof(struct caam_export_state),
4365                         },
4366                 },
4367                 .alg_type = OP_ALG_ALGSEL_SHA256,
4368         }, {
4369                 .name = "sha384",
4370                 .driver_name = "sha384-caam-qi2",
4371                 .hmac_name = "hmac(sha384)",
4372                 .hmac_driver_name = "hmac-sha384-caam-qi2",
4373                 .blocksize = SHA384_BLOCK_SIZE,
4374                 .template_ahash = {
4375                         .init = ahash_init,
4376                         .update = ahash_update,
4377                         .final = ahash_final,
4378                         .finup = ahash_finup,
4379                         .digest = ahash_digest,
4380                         .export = ahash_export,
4381                         .import = ahash_import,
4382                         .setkey = ahash_setkey,
4383                         .halg = {
4384                                 .digestsize = SHA384_DIGEST_SIZE,
4385                                 .statesize = sizeof(struct caam_export_state),
4386                         },
4387                 },
4388                 .alg_type = OP_ALG_ALGSEL_SHA384,
4389         }, {
4390                 .name = "sha512",
4391                 .driver_name = "sha512-caam-qi2",
4392                 .hmac_name = "hmac(sha512)",
4393                 .hmac_driver_name = "hmac-sha512-caam-qi2",
4394                 .blocksize = SHA512_BLOCK_SIZE,
4395                 .template_ahash = {
4396                         .init = ahash_init,
4397                         .update = ahash_update,
4398                         .final = ahash_final,
4399                         .finup = ahash_finup,
4400                         .digest = ahash_digest,
4401                         .export = ahash_export,
4402                         .import = ahash_import,
4403                         .setkey = ahash_setkey,
4404                         .halg = {
4405                                 .digestsize = SHA512_DIGEST_SIZE,
4406                                 .statesize = sizeof(struct caam_export_state),
4407                         },
4408                 },
4409                 .alg_type = OP_ALG_ALGSEL_SHA512,
4410         }, {
4411                 .name = "md5",
4412                 .driver_name = "md5-caam-qi2",
4413                 .hmac_name = "hmac(md5)",
4414                 .hmac_driver_name = "hmac-md5-caam-qi2",
4415                 .blocksize = MD5_BLOCK_WORDS * 4,
4416                 .template_ahash = {
4417                         .init = ahash_init,
4418                         .update = ahash_update,
4419                         .final = ahash_final,
4420                         .finup = ahash_finup,
4421                         .digest = ahash_digest,
4422                         .export = ahash_export,
4423                         .import = ahash_import,
4424                         .setkey = ahash_setkey,
4425                         .halg = {
4426                                 .digestsize = MD5_DIGEST_SIZE,
4427                                 .statesize = sizeof(struct caam_export_state),
4428                         },
4429                 },
4430                 .alg_type = OP_ALG_ALGSEL_MD5,
4431         }
4432 };
4433
4434 struct caam_hash_alg {
4435         struct list_head entry;
4436         struct device *dev;
4437         int alg_type;
4438         struct ahash_alg ahash_alg;
4439 };
4440
4441 static int caam_hash_cra_init(struct crypto_tfm *tfm)
4442 {
4443         struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
4444         struct crypto_alg *base = tfm->__crt_alg;
4445         struct hash_alg_common *halg =
4446                  container_of(base, struct hash_alg_common, base);
4447         struct ahash_alg *alg =
4448                  container_of(halg, struct ahash_alg, halg);
4449         struct caam_hash_alg *caam_hash =
4450                  container_of(alg, struct caam_hash_alg, ahash_alg);
4451         struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
4452         /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */
4453         static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE,
4454                                          HASH_MSG_LEN + SHA1_DIGEST_SIZE,
4455                                          HASH_MSG_LEN + 32,
4456                                          HASH_MSG_LEN + SHA256_DIGEST_SIZE,
4457                                          HASH_MSG_LEN + 64,
4458                                          HASH_MSG_LEN + SHA512_DIGEST_SIZE };
4459         dma_addr_t dma_addr;
4460         int i;
4461
4462         ctx->dev = caam_hash->dev;
4463
4464         if (alg->setkey) {
4465                 ctx->adata.key_dma = dma_map_single_attrs(ctx->dev, ctx->key,
4466                                                           ARRAY_SIZE(ctx->key),
4467                                                           DMA_TO_DEVICE,
4468                                                           DMA_ATTR_SKIP_CPU_SYNC);
4469                 if (dma_mapping_error(ctx->dev, ctx->adata.key_dma)) {
4470                         dev_err(ctx->dev, "unable to map key\n");
4471                         return -ENOMEM;
4472                 }
4473         }
4474
4475         dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, sizeof(ctx->flc),
4476                                         DMA_BIDIRECTIONAL,
4477                                         DMA_ATTR_SKIP_CPU_SYNC);
4478         if (dma_mapping_error(ctx->dev, dma_addr)) {
4479                 dev_err(ctx->dev, "unable to map shared descriptors\n");
4480                 if (ctx->adata.key_dma)
4481                         dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma,
4482                                                ARRAY_SIZE(ctx->key),
4483                                                DMA_TO_DEVICE,
4484                                                DMA_ATTR_SKIP_CPU_SYNC);
4485                 return -ENOMEM;
4486         }
4487
4488         for (i = 0; i < HASH_NUM_OP; i++)
4489                 ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]);
4490
4491         /* copy descriptor header template value */
4492         ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type;
4493
4494         ctx->ctx_len = runninglen[(ctx->adata.algtype &
4495                                    OP_ALG_ALGSEL_SUBMASK) >>
4496                                   OP_ALG_ALGSEL_SHIFT];
4497
4498         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
4499                                  sizeof(struct caam_hash_state));
4500
4501         return ahash_set_sh_desc(ahash);
4502 }
4503
4504 static void caam_hash_cra_exit(struct crypto_tfm *tfm)
4505 {
4506         struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
4507
4508         dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], sizeof(ctx->flc),
4509                                DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC);
4510         if (ctx->adata.key_dma)
4511                 dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma,
4512                                        ARRAY_SIZE(ctx->key), DMA_TO_DEVICE,
4513                                        DMA_ATTR_SKIP_CPU_SYNC);
4514 }
4515
4516 static struct caam_hash_alg *caam_hash_alloc(struct device *dev,
4517         struct caam_hash_template *template, bool keyed)
4518 {
4519         struct caam_hash_alg *t_alg;
4520         struct ahash_alg *halg;
4521         struct crypto_alg *alg;
4522
4523         t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
4524         if (!t_alg)
4525                 return ERR_PTR(-ENOMEM);
4526
4527         t_alg->ahash_alg = template->template_ahash;
4528         halg = &t_alg->ahash_alg;
4529         alg = &halg->halg.base;
4530
4531         if (keyed) {
4532                 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
4533                          template->hmac_name);
4534                 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
4535                          template->hmac_driver_name);
4536         } else {
4537                 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
4538                          template->name);
4539                 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
4540                          template->driver_name);
4541                 t_alg->ahash_alg.setkey = NULL;
4542         }
4543         alg->cra_module = THIS_MODULE;
4544         alg->cra_init = caam_hash_cra_init;
4545         alg->cra_exit = caam_hash_cra_exit;
4546         alg->cra_ctxsize = sizeof(struct caam_hash_ctx);
4547         alg->cra_priority = CAAM_CRA_PRIORITY;
4548         alg->cra_blocksize = template->blocksize;
4549         alg->cra_alignmask = 0;
4550         alg->cra_flags = CRYPTO_ALG_ASYNC;
4551
4552         t_alg->alg_type = template->alg_type;
4553         t_alg->dev = dev;
4554
4555         return t_alg;
4556 }
4557
4558 static void dpaa2_caam_fqdan_cb(struct dpaa2_io_notification_ctx *nctx)
4559 {
4560         struct dpaa2_caam_priv_per_cpu *ppriv;
4561
4562         ppriv = container_of(nctx, struct dpaa2_caam_priv_per_cpu, nctx);
4563         napi_schedule_irqoff(&ppriv->napi);
4564 }
4565
4566 static int __cold dpaa2_dpseci_dpio_setup(struct dpaa2_caam_priv *priv)
4567 {
4568         struct device *dev = priv->dev;
4569         struct dpaa2_io_notification_ctx *nctx;
4570         struct dpaa2_caam_priv_per_cpu *ppriv;
4571         int err, i = 0, cpu;
4572
4573         for_each_online_cpu(cpu) {
4574                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
4575                 ppriv->priv = priv;
4576                 nctx = &ppriv->nctx;
4577                 nctx->is_cdan = 0;
4578                 nctx->id = ppriv->rsp_fqid;
4579                 nctx->desired_cpu = cpu;
4580                 nctx->cb = dpaa2_caam_fqdan_cb;
4581
4582                 /* Register notification callbacks */
4583                 ppriv->dpio = dpaa2_io_service_select(cpu);
4584                 err = dpaa2_io_service_register(ppriv->dpio, nctx, dev);
4585                 if (unlikely(err)) {
4586                         dev_dbg(dev, "No affine DPIO for cpu %d\n", cpu);
4587                         nctx->cb = NULL;
4588                         /*
4589                          * If no affine DPIO for this core, there's probably
4590                          * none available for next cores either. Signal we want
4591                          * to retry later, in case the DPIO devices weren't
4592                          * probed yet.
4593                          */
4594                         err = -EPROBE_DEFER;
4595                         goto err;
4596                 }
4597
4598                 ppriv->store = dpaa2_io_store_create(DPAA2_CAAM_STORE_SIZE,
4599                                                      dev);
4600                 if (unlikely(!ppriv->store)) {
4601                         dev_err(dev, "dpaa2_io_store_create() failed\n");
4602                         err = -ENOMEM;
4603                         goto err;
4604                 }
4605
4606                 if (++i == priv->num_pairs)
4607                         break;
4608         }
4609
4610         return 0;
4611
4612 err:
4613         for_each_online_cpu(cpu) {
4614                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
4615                 if (!ppriv->nctx.cb)
4616                         break;
4617                 dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx, dev);
4618         }
4619
4620         for_each_online_cpu(cpu) {
4621                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
4622                 if (!ppriv->store)
4623                         break;
4624                 dpaa2_io_store_destroy(ppriv->store);
4625         }
4626
4627         return err;
4628 }
4629
4630 static void __cold dpaa2_dpseci_dpio_free(struct dpaa2_caam_priv *priv)
4631 {
4632         struct dpaa2_caam_priv_per_cpu *ppriv;
4633         int i = 0, cpu;
4634
4635         for_each_online_cpu(cpu) {
4636                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
4637                 dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx,
4638                                             priv->dev);
4639                 dpaa2_io_store_destroy(ppriv->store);
4640
4641                 if (++i == priv->num_pairs)
4642                         return;
4643         }
4644 }
4645
4646 static int dpaa2_dpseci_bind(struct dpaa2_caam_priv *priv)
4647 {
4648         struct dpseci_rx_queue_cfg rx_queue_cfg;
4649         struct device *dev = priv->dev;
4650         struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
4651         struct dpaa2_caam_priv_per_cpu *ppriv;
4652         int err = 0, i = 0, cpu;
4653
4654         /* Configure Rx queues */
4655         for_each_online_cpu(cpu) {
4656                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
4657
4658                 rx_queue_cfg.options = DPSECI_QUEUE_OPT_DEST |
4659                                        DPSECI_QUEUE_OPT_USER_CTX;
4660                 rx_queue_cfg.order_preservation_en = 0;
4661                 rx_queue_cfg.dest_cfg.dest_type = DPSECI_DEST_DPIO;
4662                 rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id;
4663                 /*
4664                  * Rx priority (WQ) doesn't really matter, since we use
4665                  * pull mode, i.e. volatile dequeues from specific FQs
4666                  */
4667                 rx_queue_cfg.dest_cfg.priority = 0;
4668                 rx_queue_cfg.user_ctx = ppriv->nctx.qman64;
4669
4670                 err = dpseci_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
4671                                           &rx_queue_cfg);
4672                 if (err) {
4673                         dev_err(dev, "dpseci_set_rx_queue() failed with err %d\n",
4674                                 err);
4675                         return err;
4676                 }
4677
4678                 if (++i == priv->num_pairs)
4679                         break;
4680         }
4681
4682         return err;
4683 }
4684
4685 static void dpaa2_dpseci_congestion_free(struct dpaa2_caam_priv *priv)
4686 {
4687         struct device *dev = priv->dev;
4688
4689         if (!priv->cscn_mem)
4690                 return;
4691
4692         dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
4693         kfree(priv->cscn_mem);
4694 }
4695
4696 static void dpaa2_dpseci_free(struct dpaa2_caam_priv *priv)
4697 {
4698         struct device *dev = priv->dev;
4699         struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
4700
4701         dpaa2_dpseci_congestion_free(priv);
4702         dpseci_close(priv->mc_io, 0, ls_dev->mc_handle);
4703 }
4704
4705 static void dpaa2_caam_process_fd(struct dpaa2_caam_priv *priv,
4706                                   const struct dpaa2_fd *fd)
4707 {
4708         struct caam_request *req;
4709         u32 fd_err;
4710
4711         if (dpaa2_fd_get_format(fd) != dpaa2_fd_list) {
4712                 dev_err(priv->dev, "Only Frame List FD format is supported!\n");
4713                 return;
4714         }
4715
4716         fd_err = dpaa2_fd_get_ctrl(fd) & FD_CTRL_ERR_MASK;
4717         if (unlikely(fd_err))
4718                 dev_err_ratelimited(priv->dev, "FD error: %08x\n", fd_err);
4719
4720         /*
4721          * FD[ADDR] is guaranteed to be valid, irrespective of errors reported
4722          * in FD[ERR] or FD[FRC].
4723          */
4724         req = dpaa2_caam_iova_to_virt(priv, dpaa2_fd_get_addr(fd));
4725         dma_unmap_single(priv->dev, req->fd_flt_dma, sizeof(req->fd_flt),
4726                          DMA_BIDIRECTIONAL);
4727         req->cbk(req->ctx, dpaa2_fd_get_frc(fd));
4728 }
4729
4730 static int dpaa2_caam_pull_fq(struct dpaa2_caam_priv_per_cpu *ppriv)
4731 {
4732         int err;
4733
4734         /* Retry while portal is busy */
4735         do {
4736                 err = dpaa2_io_service_pull_fq(ppriv->dpio, ppriv->rsp_fqid,
4737                                                ppriv->store);
4738         } while (err == -EBUSY);
4739
4740         if (unlikely(err))
4741                 dev_err(ppriv->priv->dev, "dpaa2_io_service_pull err %d", err);
4742
4743         return err;
4744 }
4745
4746 static int dpaa2_caam_store_consume(struct dpaa2_caam_priv_per_cpu *ppriv)
4747 {
4748         struct dpaa2_dq *dq;
4749         int cleaned = 0, is_last;
4750
4751         do {
4752                 dq = dpaa2_io_store_next(ppriv->store, &is_last);
4753                 if (unlikely(!dq)) {
4754                         if (unlikely(!is_last)) {
4755                                 dev_dbg(ppriv->priv->dev,
4756                                         "FQ %d returned no valid frames\n",
4757                                         ppriv->rsp_fqid);
4758                                 /*
4759                                  * MUST retry until we get some sort of
4760                                  * valid response token (be it "empty dequeue"
4761                                  * or a valid frame).
4762                                  */
4763                                 continue;
4764                         }
4765                         break;
4766                 }
4767
4768                 /* Process FD */
4769                 dpaa2_caam_process_fd(ppriv->priv, dpaa2_dq_fd(dq));
4770                 cleaned++;
4771         } while (!is_last);
4772
4773         return cleaned;
4774 }
4775
4776 static int dpaa2_dpseci_poll(struct napi_struct *napi, int budget)
4777 {
4778         struct dpaa2_caam_priv_per_cpu *ppriv;
4779         struct dpaa2_caam_priv *priv;
4780         int err, cleaned = 0, store_cleaned;
4781
4782         ppriv = container_of(napi, struct dpaa2_caam_priv_per_cpu, napi);
4783         priv = ppriv->priv;
4784
4785         if (unlikely(dpaa2_caam_pull_fq(ppriv)))
4786                 return 0;
4787
4788         do {
4789                 store_cleaned = dpaa2_caam_store_consume(ppriv);
4790                 cleaned += store_cleaned;
4791
4792                 if (store_cleaned == 0 ||
4793                     cleaned > budget - DPAA2_CAAM_STORE_SIZE)
4794                         break;
4795
4796                 /* Try to dequeue some more */
4797                 err = dpaa2_caam_pull_fq(ppriv);
4798                 if (unlikely(err))
4799                         break;
4800         } while (1);
4801
4802         if (cleaned < budget) {
4803                 napi_complete_done(napi, cleaned);
4804                 err = dpaa2_io_service_rearm(ppriv->dpio, &ppriv->nctx);
4805                 if (unlikely(err))
4806                         dev_err(priv->dev, "Notification rearm failed: %d\n",
4807                                 err);
4808         }
4809
4810         return cleaned;
4811 }
4812
4813 static int dpaa2_dpseci_congestion_setup(struct dpaa2_caam_priv *priv,
4814                                          u16 token)
4815 {
4816         struct dpseci_congestion_notification_cfg cong_notif_cfg = { 0 };
4817         struct device *dev = priv->dev;
4818         int err;
4819
4820         /*
4821          * Congestion group feature supported starting with DPSECI API v5.1
4822          * and only when object has been created with this capability.
4823          */
4824         if ((DPSECI_VER(priv->major_ver, priv->minor_ver) < DPSECI_VER(5, 1)) ||
4825             !(priv->dpseci_attr.options & DPSECI_OPT_HAS_CG))
4826                 return 0;
4827
4828         priv->cscn_mem = kzalloc(DPAA2_CSCN_SIZE + DPAA2_CSCN_ALIGN,
4829                                  GFP_KERNEL | GFP_DMA);
4830         if (!priv->cscn_mem)
4831                 return -ENOMEM;
4832
4833         priv->cscn_mem_aligned = PTR_ALIGN(priv->cscn_mem, DPAA2_CSCN_ALIGN);
4834         priv->cscn_dma = dma_map_single(dev, priv->cscn_mem_aligned,
4835                                         DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
4836         if (dma_mapping_error(dev, priv->cscn_dma)) {
4837                 dev_err(dev, "Error mapping CSCN memory area\n");
4838                 err = -ENOMEM;
4839                 goto err_dma_map;
4840         }
4841
4842         cong_notif_cfg.units = DPSECI_CONGESTION_UNIT_BYTES;
4843         cong_notif_cfg.threshold_entry = DPAA2_SEC_CONG_ENTRY_THRESH;
4844         cong_notif_cfg.threshold_exit = DPAA2_SEC_CONG_EXIT_THRESH;
4845         cong_notif_cfg.message_ctx = (uintptr_t)priv;
4846         cong_notif_cfg.message_iova = priv->cscn_dma;
4847         cong_notif_cfg.notification_mode = DPSECI_CGN_MODE_WRITE_MEM_ON_ENTER |
4848                                         DPSECI_CGN_MODE_WRITE_MEM_ON_EXIT |
4849                                         DPSECI_CGN_MODE_COHERENT_WRITE;
4850
4851         err = dpseci_set_congestion_notification(priv->mc_io, 0, token,
4852                                                  &cong_notif_cfg);
4853         if (err) {
4854                 dev_err(dev, "dpseci_set_congestion_notification failed\n");
4855                 goto err_set_cong;
4856         }
4857
4858         return 0;
4859
4860 err_set_cong:
4861         dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
4862 err_dma_map:
4863         kfree(priv->cscn_mem);
4864
4865         return err;
4866 }
4867
4868 static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev)
4869 {
4870         struct device *dev = &ls_dev->dev;
4871         struct dpaa2_caam_priv *priv;
4872         struct dpaa2_caam_priv_per_cpu *ppriv;
4873         int err, cpu;
4874         u8 i;
4875
4876         priv = dev_get_drvdata(dev);
4877
4878         priv->dev = dev;
4879         priv->dpsec_id = ls_dev->obj_desc.id;
4880
4881         /* Get a handle for the DPSECI this interface is associate with */
4882         err = dpseci_open(priv->mc_io, 0, priv->dpsec_id, &ls_dev->mc_handle);
4883         if (err) {
4884                 dev_err(dev, "dpseci_open() failed: %d\n", err);
4885                 goto err_open;
4886         }
4887
4888         err = dpseci_get_api_version(priv->mc_io, 0, &priv->major_ver,
4889                                      &priv->minor_ver);
4890         if (err) {
4891                 dev_err(dev, "dpseci_get_api_version() failed\n");
4892                 goto err_get_vers;
4893         }
4894
4895         dev_info(dev, "dpseci v%d.%d\n", priv->major_ver, priv->minor_ver);
4896
4897         err = dpseci_get_attributes(priv->mc_io, 0, ls_dev->mc_handle,
4898                                     &priv->dpseci_attr);
4899         if (err) {
4900                 dev_err(dev, "dpseci_get_attributes() failed\n");
4901                 goto err_get_vers;
4902         }
4903
4904         err = dpseci_get_sec_attr(priv->mc_io, 0, ls_dev->mc_handle,
4905                                   &priv->sec_attr);
4906         if (err) {
4907                 dev_err(dev, "dpseci_get_sec_attr() failed\n");
4908                 goto err_get_vers;
4909         }
4910
4911         err = dpaa2_dpseci_congestion_setup(priv, ls_dev->mc_handle);
4912         if (err) {
4913                 dev_err(dev, "setup_congestion() failed\n");
4914                 goto err_get_vers;
4915         }
4916
4917         priv->num_pairs = min(priv->dpseci_attr.num_rx_queues,
4918                               priv->dpseci_attr.num_tx_queues);
4919         if (priv->num_pairs > num_online_cpus()) {
4920                 dev_warn(dev, "%d queues won't be used\n",
4921                          priv->num_pairs - num_online_cpus());
4922                 priv->num_pairs = num_online_cpus();
4923         }
4924
4925         for (i = 0; i < priv->dpseci_attr.num_rx_queues; i++) {
4926                 err = dpseci_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
4927                                           &priv->rx_queue_attr[i]);
4928                 if (err) {
4929                         dev_err(dev, "dpseci_get_rx_queue() failed\n");
4930                         goto err_get_rx_queue;
4931                 }
4932         }
4933
4934         for (i = 0; i < priv->dpseci_attr.num_tx_queues; i++) {
4935                 err = dpseci_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
4936                                           &priv->tx_queue_attr[i]);
4937                 if (err) {
4938                         dev_err(dev, "dpseci_get_tx_queue() failed\n");
4939                         goto err_get_rx_queue;
4940                 }
4941         }
4942
4943         i = 0;
4944         for_each_online_cpu(cpu) {
4945                 u8 j;
4946
4947                 j = i % priv->num_pairs;
4948
4949                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
4950                 ppriv->req_fqid = priv->tx_queue_attr[j].fqid;
4951
4952                 /*
4953                  * Allow all cores to enqueue, while only some of them
4954                  * will take part in dequeuing.
4955                  */
4956                 if (++i > priv->num_pairs)
4957                         continue;
4958
4959                 ppriv->rsp_fqid = priv->rx_queue_attr[j].fqid;
4960                 ppriv->prio = j;
4961
4962                 dev_dbg(dev, "pair %d: rx queue %d, tx queue %d\n", j,
4963                         priv->rx_queue_attr[j].fqid,
4964                         priv->tx_queue_attr[j].fqid);
4965
4966                 ppriv->net_dev.dev = *dev;
4967                 INIT_LIST_HEAD(&ppriv->net_dev.napi_list);
4968                 netif_napi_add(&ppriv->net_dev, &ppriv->napi, dpaa2_dpseci_poll,
4969                                DPAA2_CAAM_NAPI_WEIGHT);
4970         }
4971
4972         return 0;
4973
4974 err_get_rx_queue:
4975         dpaa2_dpseci_congestion_free(priv);
4976 err_get_vers:
4977         dpseci_close(priv->mc_io, 0, ls_dev->mc_handle);
4978 err_open:
4979         return err;
4980 }
4981
4982 static int dpaa2_dpseci_enable(struct dpaa2_caam_priv *priv)
4983 {
4984         struct device *dev = priv->dev;
4985         struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
4986         struct dpaa2_caam_priv_per_cpu *ppriv;
4987         int i;
4988
4989         for (i = 0; i < priv->num_pairs; i++) {
4990                 ppriv = per_cpu_ptr(priv->ppriv, i);
4991                 napi_enable(&ppriv->napi);
4992         }
4993
4994         return dpseci_enable(priv->mc_io, 0, ls_dev->mc_handle);
4995 }
4996
4997 static int __cold dpaa2_dpseci_disable(struct dpaa2_caam_priv *priv)
4998 {
4999         struct device *dev = priv->dev;
5000         struct dpaa2_caam_priv_per_cpu *ppriv;
5001         struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
5002         int i, err = 0, enabled;
5003
5004         err = dpseci_disable(priv->mc_io, 0, ls_dev->mc_handle);
5005         if (err) {
5006                 dev_err(dev, "dpseci_disable() failed\n");
5007                 return err;
5008         }
5009
5010         err = dpseci_is_enabled(priv->mc_io, 0, ls_dev->mc_handle, &enabled);
5011         if (err) {
5012                 dev_err(dev, "dpseci_is_enabled() failed\n");
5013                 return err;
5014         }
5015
5016         dev_dbg(dev, "disable: %s\n", enabled ? "false" : "true");
5017
5018         for (i = 0; i < priv->num_pairs; i++) {
5019                 ppriv = per_cpu_ptr(priv->ppriv, i);
5020                 napi_disable(&ppriv->napi);
5021                 netif_napi_del(&ppriv->napi);
5022         }
5023
5024         return 0;
5025 }
5026
5027 static struct list_head hash_list;
5028
5029 static int dpaa2_caam_probe(struct fsl_mc_device *dpseci_dev)
5030 {
5031         struct device *dev;
5032         struct dpaa2_caam_priv *priv;
5033         int i, err = 0;
5034         bool registered = false;
5035
5036         /*
5037          * There is no way to get CAAM endianness - there is no direct register
5038          * space access and MC f/w does not provide this attribute.
5039          * All DPAA2-based SoCs have little endian CAAM, thus hard-code this
5040          * property.
5041          */
5042         caam_little_end = true;
5043
5044         caam_imx = false;
5045
5046         dev = &dpseci_dev->dev;
5047
5048         priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
5049         if (!priv)
5050                 return -ENOMEM;
5051
5052         dev_set_drvdata(dev, priv);
5053
5054         priv->domain = iommu_get_domain_for_dev(dev);
5055
5056         qi_cache = kmem_cache_create("dpaa2_caamqicache", CAAM_QI_MEMCACHE_SIZE,
5057                                      0, SLAB_CACHE_DMA, NULL);
5058         if (!qi_cache) {
5059                 dev_err(dev, "Can't allocate SEC cache\n");
5060                 return -ENOMEM;
5061         }
5062
5063         err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(49));
5064         if (err) {
5065                 dev_err(dev, "dma_set_mask_and_coherent() failed\n");
5066                 goto err_dma_mask;
5067         }
5068
5069         /* Obtain a MC portal */
5070         err = fsl_mc_portal_allocate(dpseci_dev, 0, &priv->mc_io);
5071         if (err) {
5072                 if (err == -ENXIO)
5073                         err = -EPROBE_DEFER;
5074                 else
5075                         dev_err(dev, "MC portal allocation failed\n");
5076
5077                 goto err_dma_mask;
5078         }
5079
5080         priv->ppriv = alloc_percpu(*priv->ppriv);
5081         if (!priv->ppriv) {
5082                 dev_err(dev, "alloc_percpu() failed\n");
5083                 err = -ENOMEM;
5084                 goto err_alloc_ppriv;
5085         }
5086
5087         /* DPSECI initialization */
5088         err = dpaa2_dpseci_setup(dpseci_dev);
5089         if (err) {
5090                 dev_err(dev, "dpaa2_dpseci_setup() failed\n");
5091                 goto err_dpseci_setup;
5092         }
5093
5094         /* DPIO */
5095         err = dpaa2_dpseci_dpio_setup(priv);
5096         if (err) {
5097                 if (err != -EPROBE_DEFER)
5098                         dev_err(dev, "dpaa2_dpseci_dpio_setup() failed\n");
5099                 goto err_dpio_setup;
5100         }
5101
5102         /* DPSECI binding to DPIO */
5103         err = dpaa2_dpseci_bind(priv);
5104         if (err) {
5105                 dev_err(dev, "dpaa2_dpseci_bind() failed\n");
5106                 goto err_bind;
5107         }
5108
5109         /* DPSECI enable */
5110         err = dpaa2_dpseci_enable(priv);
5111         if (err) {
5112                 dev_err(dev, "dpaa2_dpseci_enable() failed\n");
5113                 goto err_bind;
5114         }
5115
5116         dpaa2_dpseci_debugfs_init(priv);
5117
5118         /* register crypto algorithms the device supports */
5119         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
5120                 struct caam_skcipher_alg *t_alg = driver_algs + i;
5121                 u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK;
5122
5123                 /* Skip DES algorithms if not supported by device */
5124                 if (!priv->sec_attr.des_acc_num &&
5125                     (alg_sel == OP_ALG_ALGSEL_3DES ||
5126                      alg_sel == OP_ALG_ALGSEL_DES))
5127                         continue;
5128
5129                 /* Skip AES algorithms if not supported by device */
5130                 if (!priv->sec_attr.aes_acc_num &&
5131                     alg_sel == OP_ALG_ALGSEL_AES)
5132                         continue;
5133
5134                 /* Skip CHACHA20 algorithms if not supported by device */
5135                 if (alg_sel == OP_ALG_ALGSEL_CHACHA20 &&
5136                     !priv->sec_attr.ccha_acc_num)
5137                         continue;
5138
5139                 t_alg->caam.dev = dev;
5140                 caam_skcipher_alg_init(t_alg);
5141
5142                 err = crypto_register_skcipher(&t_alg->skcipher);
5143                 if (err) {
5144                         dev_warn(dev, "%s alg registration failed: %d\n",
5145                                  t_alg->skcipher.base.cra_driver_name, err);
5146                         continue;
5147                 }
5148
5149                 t_alg->registered = true;
5150                 registered = true;
5151         }
5152
5153         for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
5154                 struct caam_aead_alg *t_alg = driver_aeads + i;
5155                 u32 c1_alg_sel = t_alg->caam.class1_alg_type &
5156                                  OP_ALG_ALGSEL_MASK;
5157                 u32 c2_alg_sel = t_alg->caam.class2_alg_type &
5158                                  OP_ALG_ALGSEL_MASK;
5159
5160                 /* Skip DES algorithms if not supported by device */
5161                 if (!priv->sec_attr.des_acc_num &&
5162                     (c1_alg_sel == OP_ALG_ALGSEL_3DES ||
5163                      c1_alg_sel == OP_ALG_ALGSEL_DES))
5164                         continue;
5165
5166                 /* Skip AES algorithms if not supported by device */
5167                 if (!priv->sec_attr.aes_acc_num &&
5168                     c1_alg_sel == OP_ALG_ALGSEL_AES)
5169                         continue;
5170
5171                 /* Skip CHACHA20 algorithms if not supported by device */
5172                 if (c1_alg_sel == OP_ALG_ALGSEL_CHACHA20 &&
5173                     !priv->sec_attr.ccha_acc_num)
5174                         continue;
5175
5176                 /* Skip POLY1305 algorithms if not supported by device */
5177                 if (c2_alg_sel == OP_ALG_ALGSEL_POLY1305 &&
5178                     !priv->sec_attr.ptha_acc_num)
5179                         continue;
5180
5181                 /*
5182                  * Skip algorithms requiring message digests
5183                  * if MD not supported by device.
5184                  */
5185                 if ((c2_alg_sel & ~OP_ALG_ALGSEL_SUBMASK) == 0x40 &&
5186                     !priv->sec_attr.md_acc_num)
5187                         continue;
5188
5189                 t_alg->caam.dev = dev;
5190                 caam_aead_alg_init(t_alg);
5191
5192                 err = crypto_register_aead(&t_alg->aead);
5193                 if (err) {
5194                         dev_warn(dev, "%s alg registration failed: %d\n",
5195                                  t_alg->aead.base.cra_driver_name, err);
5196                         continue;
5197                 }
5198
5199                 t_alg->registered = true;
5200                 registered = true;
5201         }
5202         if (registered)
5203                 dev_info(dev, "algorithms registered in /proc/crypto\n");
5204
5205         /* register hash algorithms the device supports */
5206         INIT_LIST_HEAD(&hash_list);
5207
5208         /*
5209          * Skip registration of any hashing algorithms if MD block
5210          * is not present.
5211          */
5212         if (!priv->sec_attr.md_acc_num)
5213                 return 0;
5214
5215         for (i = 0; i < ARRAY_SIZE(driver_hash); i++) {
5216                 struct caam_hash_alg *t_alg;
5217                 struct caam_hash_template *alg = driver_hash + i;
5218
5219                 /* register hmac version */
5220                 t_alg = caam_hash_alloc(dev, alg, true);
5221                 if (IS_ERR(t_alg)) {
5222                         err = PTR_ERR(t_alg);
5223                         dev_warn(dev, "%s hash alg allocation failed: %d\n",
5224                                  alg->driver_name, err);
5225                         continue;
5226                 }
5227
5228                 err = crypto_register_ahash(&t_alg->ahash_alg);
5229                 if (err) {
5230                         dev_warn(dev, "%s alg registration failed: %d\n",
5231                                  t_alg->ahash_alg.halg.base.cra_driver_name,
5232                                  err);
5233                         kfree(t_alg);
5234                 } else {
5235                         list_add_tail(&t_alg->entry, &hash_list);
5236                 }
5237
5238                 /* register unkeyed version */
5239                 t_alg = caam_hash_alloc(dev, alg, false);
5240                 if (IS_ERR(t_alg)) {
5241                         err = PTR_ERR(t_alg);
5242                         dev_warn(dev, "%s alg allocation failed: %d\n",
5243                                  alg->driver_name, err);
5244                         continue;
5245                 }
5246
5247                 err = crypto_register_ahash(&t_alg->ahash_alg);
5248                 if (err) {
5249                         dev_warn(dev, "%s alg registration failed: %d\n",
5250                                  t_alg->ahash_alg.halg.base.cra_driver_name,
5251                                  err);
5252                         kfree(t_alg);
5253                 } else {
5254                         list_add_tail(&t_alg->entry, &hash_list);
5255                 }
5256         }
5257         if (!list_empty(&hash_list))
5258                 dev_info(dev, "hash algorithms registered in /proc/crypto\n");
5259
5260         return err;
5261
5262 err_bind:
5263         dpaa2_dpseci_dpio_free(priv);
5264 err_dpio_setup:
5265         dpaa2_dpseci_free(priv);
5266 err_dpseci_setup:
5267         free_percpu(priv->ppriv);
5268 err_alloc_ppriv:
5269         fsl_mc_portal_free(priv->mc_io);
5270 err_dma_mask:
5271         kmem_cache_destroy(qi_cache);
5272
5273         return err;
5274 }
5275
5276 static int __cold dpaa2_caam_remove(struct fsl_mc_device *ls_dev)
5277 {
5278         struct device *dev;
5279         struct dpaa2_caam_priv *priv;
5280         int i;
5281
5282         dev = &ls_dev->dev;
5283         priv = dev_get_drvdata(dev);
5284
5285         dpaa2_dpseci_debugfs_exit(priv);
5286
5287         for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
5288                 struct caam_aead_alg *t_alg = driver_aeads + i;
5289
5290                 if (t_alg->registered)
5291                         crypto_unregister_aead(&t_alg->aead);
5292         }
5293
5294         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
5295                 struct caam_skcipher_alg *t_alg = driver_algs + i;
5296
5297                 if (t_alg->registered)
5298                         crypto_unregister_skcipher(&t_alg->skcipher);
5299         }
5300
5301         if (hash_list.next) {
5302                 struct caam_hash_alg *t_hash_alg, *p;
5303
5304                 list_for_each_entry_safe(t_hash_alg, p, &hash_list, entry) {
5305                         crypto_unregister_ahash(&t_hash_alg->ahash_alg);
5306                         list_del(&t_hash_alg->entry);
5307                         kfree(t_hash_alg);
5308                 }
5309         }
5310
5311         dpaa2_dpseci_disable(priv);
5312         dpaa2_dpseci_dpio_free(priv);
5313         dpaa2_dpseci_free(priv);
5314         free_percpu(priv->ppriv);
5315         fsl_mc_portal_free(priv->mc_io);
5316         kmem_cache_destroy(qi_cache);
5317
5318         return 0;
5319 }
5320
5321 int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req)
5322 {
5323         struct dpaa2_fd fd;
5324         struct dpaa2_caam_priv *priv = dev_get_drvdata(dev);
5325         struct dpaa2_caam_priv_per_cpu *ppriv;
5326         int err = 0, i;
5327
5328         if (IS_ERR(req))
5329                 return PTR_ERR(req);
5330
5331         if (priv->cscn_mem) {
5332                 dma_sync_single_for_cpu(priv->dev, priv->cscn_dma,
5333                                         DPAA2_CSCN_SIZE,
5334                                         DMA_FROM_DEVICE);
5335                 if (unlikely(dpaa2_cscn_state_congested(priv->cscn_mem_aligned))) {
5336                         dev_dbg_ratelimited(dev, "Dropping request\n");
5337                         return -EBUSY;
5338                 }
5339         }
5340
5341         dpaa2_fl_set_flc(&req->fd_flt[1], req->flc_dma);
5342
5343         req->fd_flt_dma = dma_map_single(dev, req->fd_flt, sizeof(req->fd_flt),
5344                                          DMA_BIDIRECTIONAL);
5345         if (dma_mapping_error(dev, req->fd_flt_dma)) {
5346                 dev_err(dev, "DMA mapping error for QI enqueue request\n");
5347                 goto err_out;
5348         }
5349
5350         memset(&fd, 0, sizeof(fd));
5351         dpaa2_fd_set_format(&fd, dpaa2_fd_list);
5352         dpaa2_fd_set_addr(&fd, req->fd_flt_dma);
5353         dpaa2_fd_set_len(&fd, dpaa2_fl_get_len(&req->fd_flt[1]));
5354         dpaa2_fd_set_flc(&fd, req->flc_dma);
5355
5356         ppriv = this_cpu_ptr(priv->ppriv);
5357         for (i = 0; i < (priv->dpseci_attr.num_tx_queues << 1); i++) {
5358                 err = dpaa2_io_service_enqueue_fq(ppriv->dpio, ppriv->req_fqid,
5359                                                   &fd);
5360                 if (err != -EBUSY)
5361                         break;
5362
5363                 cpu_relax();
5364         }
5365
5366         if (unlikely(err)) {
5367                 dev_err_ratelimited(dev, "Error enqueuing frame: %d\n", err);
5368                 goto err_out;
5369         }
5370
5371         return -EINPROGRESS;
5372
5373 err_out:
5374         dma_unmap_single(dev, req->fd_flt_dma, sizeof(req->fd_flt),
5375                          DMA_BIDIRECTIONAL);
5376         return -EIO;
5377 }
5378 EXPORT_SYMBOL(dpaa2_caam_enqueue);
5379
5380 static const struct fsl_mc_device_id dpaa2_caam_match_id_table[] = {
5381         {
5382                 .vendor = FSL_MC_VENDOR_FREESCALE,
5383                 .obj_type = "dpseci",
5384         },
5385         { .vendor = 0x0 }
5386 };
5387
5388 static struct fsl_mc_driver dpaa2_caam_driver = {
5389         .driver = {
5390                 .name           = KBUILD_MODNAME,
5391                 .owner          = THIS_MODULE,
5392         },
5393         .probe          = dpaa2_caam_probe,
5394         .remove         = dpaa2_caam_remove,
5395         .match_id_table = dpaa2_caam_match_id_table
5396 };
5397
5398 MODULE_LICENSE("Dual BSD/GPL");
5399 MODULE_AUTHOR("Freescale Semiconductor, Inc");
5400 MODULE_DESCRIPTION("Freescale DPAA2 CAAM Driver");
5401
5402 module_fsl_mc_driver(dpaa2_caam_driver);