drivers/crypto/vmx/aes_xts.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /**
   3  * AES XTS routines supporting VMX In-core instructions on Power 8
   4  *
   5  * Copyright (C) 2015 International Business Machines Inc.
   6  *
   7  * Author: Leonidas S. Barbosa <leosilva@linux.vnet.ibm.com>
   8  */
   9
  10 #include <asm/simd.h>
  11 #include <asm/switch_to.h>
  12 #include <crypto/aes.h>
  13 #include <crypto/internal/simd.h>
  14 #include <crypto/internal/skcipher.h>
  15 #include <crypto/xts.h>
  16
  17 #include "aesp8-ppc.h"
  18
  19 struct p8_aes_xts_ctx {
  20         struct crypto_skcipher *fallback;
  21         struct aes_key enc_key;
  22         struct aes_key dec_key;
  23         struct aes_key tweak_key;
  24 };
  25
  26 static int p8_aes_xts_init(struct crypto_skcipher *tfm)
  27 {
  28         struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
  29         struct crypto_skcipher *fallback;
  30
  31         fallback = crypto_alloc_skcipher("xts(aes)", 0,
  32                                          CRYPTO_ALG_NEED_FALLBACK |
  33                                          CRYPTO_ALG_ASYNC);
  34         if (IS_ERR(fallback)) {
  35                 pr_err("Failed to allocate xts(aes) fallback: %ld\n",
  36                        PTR_ERR(fallback));
  37                 return PTR_ERR(fallback);
  38         }
  39
  40         crypto_skcipher_set_reqsize(tfm, sizeof(struct skcipher_request) +
  41                                     crypto_skcipher_reqsize(fallback));
  42         ctx->fallback = fallback;
  43         return 0;
  44 }
  45
  46 static void p8_aes_xts_exit(struct crypto_skcipher *tfm)
  47 {
  48         struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
  49
  50         crypto_free_skcipher(ctx->fallback);
  51 }
  52
  53 static int p8_aes_xts_setkey(struct crypto_skcipher *tfm, const u8 *key,
  54                              unsigned int keylen)
  55 {
  56         struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
  57         int ret;
  58
  59         ret = xts_verify_key(tfm, key, keylen);
  60         if (ret)
  61                 return ret;
  62
  63         preempt_disable();
  64         pagefault_disable();
  65         enable_kernel_vsx();
  66         ret = aes_p8_set_encrypt_key(key + keylen/2, (keylen/2) * 8, &ctx->tweak_key);
  67         ret |= aes_p8_set_encrypt_key(key, (keylen/2) * 8, &ctx->enc_key);
  68         ret |= aes_p8_set_decrypt_key(key, (keylen/2) * 8, &ctx->dec_key);
  69         disable_kernel_vsx();
  70         pagefault_enable();
  71         preempt_enable();
  72
  73         ret |= crypto_skcipher_setkey(ctx->fallback, key, keylen);
  74
  75         return ret ? -EINVAL : 0;
  76 }
  77
  78 static int p8_aes_xts_crypt(struct skcipher_request *req, int enc)
  79 {
  80         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
  81         const struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
  82         struct skcipher_walk walk;
  83         unsigned int nbytes;
  84         u8 tweak[AES_BLOCK_SIZE];
  85         int ret;
  86
  87         if (!crypto_simd_usable() || (req->cryptlen % XTS_BLOCK_SIZE) != 0) {
  88                 struct skcipher_request *subreq = skcipher_request_ctx(req);
  89
  90                 *subreq = *req;
  91                 skcipher_request_set_tfm(subreq, ctx->fallback);
  92                 return enc ? crypto_skcipher_encrypt(subreq) :
  93                              crypto_skcipher_decrypt(subreq);
  94         }
  95
  96         ret = skcipher_walk_virt(&walk, req, false);
  97         if (ret)
  98                 return ret;
  99
 100         preempt_disable();
 101         pagefault_disable();
 102         enable_kernel_vsx();
 103
 104         aes_p8_encrypt(walk.iv, tweak, &ctx->tweak_key);
 105
 106         disable_kernel_vsx();
 107         pagefault_enable();
 108         preempt_enable();
 109
 110         while ((nbytes = walk.nbytes) != 0) {
 111                 preempt_disable();
 112                 pagefault_disable();
 113                 enable_kernel_vsx();
 114                 if (enc)
 115                         aes_p8_xts_encrypt(walk.src.virt.addr,
 116                                            walk.dst.virt.addr,
 117                                            round_down(nbytes, AES_BLOCK_SIZE),
 118                                            &ctx->enc_key, NULL, tweak);
 119                 else
 120                         aes_p8_xts_decrypt(walk.src.virt.addr,
 121                                            walk.dst.virt.addr,
 122                                            round_down(nbytes, AES_BLOCK_SIZE),
 123                                            &ctx->dec_key, NULL, tweak);
 124                 disable_kernel_vsx();
 125                 pagefault_enable();
 126                 preempt_enable();
 127
 128                 ret = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
 129         }
 130         return ret;
 131 }
 132
 133 static int p8_aes_xts_encrypt(struct skcipher_request *req)
 134 {
 135         return p8_aes_xts_crypt(req, 1);
 136 }
 137
 138 static int p8_aes_xts_decrypt(struct skcipher_request *req)
 139 {
 140         return p8_aes_xts_crypt(req, 0);
 141 }
 142
 143 struct skcipher_alg p8_aes_xts_alg = {
 144         .base.cra_name = "xts(aes)",
 145         .base.cra_driver_name = "p8_aes_xts",
 146         .base.cra_module = THIS_MODULE,
 147         .base.cra_priority = 2000,
 148         .base.cra_flags = CRYPTO_ALG_NEED_FALLBACK,
 149         .base.cra_blocksize = AES_BLOCK_SIZE,
 150         .base.cra_ctxsize = sizeof(struct p8_aes_xts_ctx),
 151         .setkey = p8_aes_xts_setkey,
 152         .encrypt = p8_aes_xts_encrypt,
 153         .decrypt = p8_aes_xts_decrypt,
 154         .init = p8_aes_xts_init,
 155         .exit = p8_aes_xts_exit,
 156         .min_keysize = 2 * AES_MIN_KEY_SIZE,
 157         .max_keysize = 2 * AES_MAX_KEY_SIZE,
 158         .ivsize = AES_BLOCK_SIZE,
 159 };