crypto: algapi - make crypto_xor() take separate dst and src arguments

There are quite a number of occurrences in the kernel of the pattern

  if (dst != src)
          memcpy(dst, src, walk.total % AES_BLOCK_SIZE);
  crypto_xor(dst, final, walk.total % AES_BLOCK_SIZE);

or

  crypto_xor(keystream, src, nbytes);
  memcpy(dst, keystream, nbytes);

where crypto_xor() is preceded or followed by a memcpy() invocation
that is only there because crypto_xor() uses its output parameter as
one of the inputs. To avoid having to add new instances of this pattern
in the arm64 code, which will be refactored to implement non-SIMD
fallbacks, add an alternative implementation called crypto_xor_cpy(),
taking separate input and output arguments. This removes the need for
the separate memcpy().

Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

diff --git a/arch/arm/crypto/aes-ce-glue.c b/arch/arm/crypto/aes-ce-glue.c
index 0f966a8ca1cef5f9b122530652d38b9a784e7876..d0a9cec73707b7be885b509e5befe393e90b902f 100644 (file)
--- a/arch/arm/crypto/aes-ce-glue.c
+++ b/arch/arm/crypto/aes-ce-glue.c
@@ -285,9 +285,7 @@ static int ctr_encrypt(struct skcipher_request *req)
 
                ce_aes_ctr_encrypt(tail, NULL, (u8 *)ctx->key_enc,
                                   num_rounds(ctx), blocks, walk.iv);
-               if (tdst != tsrc)
-                       memcpy(tdst, tsrc, nbytes);
-               crypto_xor(tdst, tail, nbytes);
+               crypto_xor_cpy(tdst, tsrc, tail, nbytes);
                err = skcipher_walk_done(&walk, 0);
        }
        kernel_neon_end();

diff --git a/arch/arm/crypto/aes-neonbs-glue.c b/arch/arm/crypto/aes-neonbs-glue.c
index c763779614442c22cbb303b9d095d30d470092ed..18768f3304495157f1ccdc411901f3ebdb4dd635 100644 (file)
--- a/arch/arm/crypto/aes-neonbs-glue.c
+++ b/arch/arm/crypto/aes-neonbs-glue.c
@@ -221,9 +221,8 @@ static int ctr_encrypt(struct skcipher_request *req)
                        u8 *dst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE;
                        u8 *src = walk.src.virt.addr + blocks * AES_BLOCK_SIZE;
 
-                       if (dst != src)
-                               memcpy(dst, src, walk.total % AES_BLOCK_SIZE);
-                       crypto_xor(dst, final, walk.total % AES_BLOCK_SIZE);
+                       crypto_xor_cpy(dst, src, final,
+                                      walk.total % AES_BLOCK_SIZE);
 
                        err = skcipher_walk_done(&walk, 0);
                        break;

diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c
index bcf596b0197ef31cfffa60e04c69cd652f5eac09..0da30e3b0e4be8c7dde5d4585bdf893202a5bfe8 100644 (file)
--- a/arch/arm64/crypto/aes-glue.c
+++ b/arch/arm64/crypto/aes-glue.c
@@ -241,9 +241,7 @@ static int ctr_encrypt(struct skcipher_request *req)
 
                aes_ctr_encrypt(tail, NULL, (u8 *)ctx->key_enc, rounds,
                                blocks, walk.iv, first);
-               if (tdst != tsrc)
-                       memcpy(tdst, tsrc, nbytes);
-               crypto_xor(tdst, tail, nbytes);
+               crypto_xor_cpy(tdst, tsrc, tail, nbytes);
                err = skcipher_walk_done(&walk, 0);
        }
        kernel_neon_end();

diff --git a/arch/arm64/crypto/aes-neonbs-glue.c b/arch/arm64/crypto/aes-neonbs-glue.c
index db2501d93550c35720c3d1adeb78fed49a9df1a2..9001aec16007bedb5212af86a2cb4f753a296df4 100644 (file)
--- a/arch/arm64/crypto/aes-neonbs-glue.c
+++ b/arch/arm64/crypto/aes-neonbs-glue.c
@@ -224,9 +224,8 @@ static int ctr_encrypt(struct skcipher_request *req)
                        u8 *dst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE;
                        u8 *src = walk.src.virt.addr + blocks * AES_BLOCK_SIZE;
 
-                       if (dst != src)
-                               memcpy(dst, src, walk.total % AES_BLOCK_SIZE);
-                       crypto_xor(dst, final, walk.total % AES_BLOCK_SIZE);
+                       crypto_xor_cpy(dst, src, final,
+                                      walk.total % AES_BLOCK_SIZE);
 
                        err = skcipher_walk_done(&walk, 0);
                        break;

diff --git a/arch/sparc/crypto/aes_glue.c b/arch/sparc/crypto/aes_glue.c
index c90930de76ba8670041598ba0d6461ef439c9539..3cd4f6b198b65665046101184f579b51138a688d 100644 (file)
--- a/arch/sparc/crypto/aes_glue.c
+++ b/arch/sparc/crypto/aes_glue.c
@@ -344,8 +344,7 @@ static void ctr_crypt_final(struct crypto_sparc64_aes_ctx *ctx,
 
        ctx->ops->ecb_encrypt(&ctx->key[0], (const u64 *)ctrblk,
                              keystream, AES_BLOCK_SIZE);
-       crypto_xor((u8 *) keystream, src, nbytes);
-       memcpy(dst, keystream, nbytes);
+       crypto_xor_cpy(dst, (u8 *) keystream, src, nbytes);
        crypto_inc(ctrblk, AES_BLOCK_SIZE);
 }
 

diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index 4a55cdcdc0082790bcf6457ab43e1a84caea7388..5c15d6b573299e75c2eec54d98bbde9f48e67637 100644 (file)
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -475,8 +475,8 @@ static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
        unsigned int nbytes = walk->nbytes;
 
        aesni_enc(ctx, keystream, ctrblk);
-       crypto_xor(keystream, src, nbytes);
-       memcpy(dst, keystream, nbytes);
+       crypto_xor_cpy(dst, keystream, src, nbytes);
+
        crypto_inc(ctrblk, AES_BLOCK_SIZE);
 }
 

diff --git a/arch/x86/crypto/blowfish_glue.c b/arch/x86/crypto/blowfish_glue.c
index 17c05531dfd1752234eb9b6835f848b12ed07817..f9eca34301e20fccec4b8daa5a0cc98710fa3094 100644 (file)
--- a/arch/x86/crypto/blowfish_glue.c
+++ b/arch/x86/crypto/blowfish_glue.c
@@ -271,8 +271,7 @@ static void ctr_crypt_final(struct bf_ctx *ctx, struct blkcipher_walk *walk)
        unsigned int nbytes = walk->nbytes;
 
        blowfish_enc_blk(ctx, keystream, ctrblk);
-       crypto_xor(keystream, src, nbytes);
-       memcpy(dst, keystream, nbytes);
+       crypto_xor_cpy(dst, keystream, src, nbytes);
 
        crypto_inc(ctrblk, BF_BLOCK_SIZE);
 }

diff --git a/arch/x86/crypto/cast5_avx_glue.c b/arch/x86/crypto/cast5_avx_glue.c
index 8648158f39166f1cf1603fc7f139af629e163bde..dbea6020ffe7d462bd93bf703ccecbe1f817d3f4 100644 (file)
--- a/arch/x86/crypto/cast5_avx_glue.c
+++ b/arch/x86/crypto/cast5_avx_glue.c
@@ -256,8 +256,7 @@ static void ctr_crypt_final(struct blkcipher_desc *desc,
        unsigned int nbytes = walk->nbytes;
 
        __cast5_encrypt(ctx, keystream, ctrblk);
-       crypto_xor(keystream, src, nbytes);
-       memcpy(dst, keystream, nbytes);
+       crypto_xor_cpy(dst, keystream, src, nbytes);
 
        crypto_inc(ctrblk, CAST5_BLOCK_SIZE);
 }

diff --git a/arch/x86/crypto/des3_ede_glue.c b/arch/x86/crypto/des3_ede_glue.c
index d6fc59aaaadfbc0461a9aaab1557eb27f7841402..30c0a37f488264651460d5de9df30a097164ed88 100644 (file)
--- a/arch/x86/crypto/des3_ede_glue.c
+++ b/arch/x86/crypto/des3_ede_glue.c
@@ -277,8 +277,7 @@ static void ctr_crypt_final(struct des3_ede_x86_ctx *ctx,
        unsigned int nbytes = walk->nbytes;
 
        des3_ede_enc_blk(ctx, keystream, ctrblk);
-       crypto_xor(keystream, src, nbytes);
-       memcpy(dst, keystream, nbytes);
+       crypto_xor_cpy(dst, keystream, src, nbytes);
 
        crypto_inc(ctrblk, DES3_EDE_BLOCK_SIZE);
 }

diff --git a/crypto/ctr.c b/crypto/ctr.c
index 477d9226ccaac99c087bf0df5f51597a34518100..854d924f9d8e6fad3c7653ca5ba0ea5d83a9d425 100644 (file)
--- a/crypto/ctr.c
+++ b/crypto/ctr.c
@@ -65,8 +65,7 @@ static void crypto_ctr_crypt_final(struct blkcipher_walk *walk,
        unsigned int nbytes = walk->nbytes;
 
        crypto_cipher_encrypt_one(tfm, keystream, ctrblk);
-       crypto_xor(keystream, src, nbytes);
-       memcpy(dst, keystream, nbytes);
+       crypto_xor_cpy(dst, keystream, src, nbytes);
 
        crypto_inc(ctrblk, bsize);
 }

diff --git a/crypto/pcbc.c b/crypto/pcbc.c
index 29dd2b4a3b85b4021592facbdd019cc1c14938f9..d9e45a9587201d7760084bd81a024c11d743dc75 100644 (file)
--- a/crypto/pcbc.c
+++ b/crypto/pcbc.c
@@ -55,8 +55,7 @@ static int crypto_pcbc_encrypt_segment(struct skcipher_request *req,
        do {
                crypto_xor(iv, src, bsize);
                crypto_cipher_encrypt_one(tfm, dst, iv);
-               memcpy(iv, dst, bsize);
-               crypto_xor(iv, src, bsize);
+               crypto_xor_cpy(iv, dst, src, bsize);
 
                src += bsize;
                dst += bsize;
@@ -79,8 +78,7 @@ static int crypto_pcbc_encrypt_inplace(struct skcipher_request *req,
                memcpy(tmpbuf, src, bsize);
                crypto_xor(iv, src, bsize);
                crypto_cipher_encrypt_one(tfm, src, iv);
-               memcpy(iv, tmpbuf, bsize);
-               crypto_xor(iv, src, bsize);
+               crypto_xor_cpy(iv, tmpbuf, src, bsize);
 
                src += bsize;
        } while ((nbytes -= bsize) >= bsize);
@@ -127,8 +125,7 @@ static int crypto_pcbc_decrypt_segment(struct skcipher_request *req,
        do {
                crypto_cipher_decrypt_one(tfm, dst, src);
                crypto_xor(dst, iv, bsize);
-               memcpy(iv, src, bsize);
-               crypto_xor(iv, dst, bsize);
+               crypto_xor_cpy(iv, dst, src, bsize);
 
                src += bsize;
                dst += bsize;
@@ -153,8 +150,7 @@ static int crypto_pcbc_decrypt_inplace(struct skcipher_request *req,
                memcpy(tmpbuf, src, bsize);
                crypto_cipher_decrypt_one(tfm, src, src);
                crypto_xor(src, iv, bsize);
-               memcpy(iv, tmpbuf, bsize);
-               crypto_xor(iv, src, bsize);
+               crypto_xor_cpy(iv, src, tmpbuf, bsize);
 
                src += bsize;
        } while ((nbytes -= bsize) >= bsize);

diff --git a/drivers/crypto/vmx/aes_ctr.c b/drivers/crypto/vmx/aes_ctr.c
index 9c26d9e8dbea76ed49ac55bab94984c4a55767b9..17d84217dd765a29d8d9e69921820ebd52823a65 100644 (file)
--- a/drivers/crypto/vmx/aes_ctr.c
+++ b/drivers/crypto/vmx/aes_ctr.c
@@ -104,8 +104,7 @@ static void p8_aes_ctr_final(struct p8_aes_ctr_ctx *ctx,
        pagefault_enable();
        preempt_enable();
 
-       crypto_xor(keystream, src, nbytes);
-       memcpy(dst, keystream, nbytes);
+       crypto_xor_cpy(dst, keystream, src, nbytes);
        crypto_inc(ctrblk, AES_BLOCK_SIZE);
 }
 

diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c
index cdf6b1e124604b7bd5e2fee7743a9435f1e79032..fa17e545279686004d44baac196d48631a5b0d85 100644 (file)
--- a/drivers/md/dm-crypt.c
+++ b/drivers/md/dm-crypt.c
@@ -758,9 +758,8 @@ static int crypt_iv_tcw_whitening(struct crypt_config *cc,
        int i, r;
 
        /* xor whitening with sector number */
-       memcpy(buf, tcw->whitening, TCW_WHITENING_SIZE);
-       crypto_xor(buf, (u8 *)&sector, 8);
-       crypto_xor(&buf[8], (u8 *)&sector, 8);
+       crypto_xor_cpy(buf, tcw->whitening, (u8 *)&sector, 8);
+       crypto_xor_cpy(&buf[8], tcw->whitening + 8, (u8 *)&sector, 8);
 
        /* calculate crc32 for every 32bit part and xor it */
        desc->tfm = tcw->crc32_tfm;
@@ -805,10 +804,10 @@ static int crypt_iv_tcw_gen(struct crypt_config *cc, u8 *iv,
        }
 
        /* Calculate IV */
-       memcpy(iv, tcw->iv_seed, cc->iv_size);
-       crypto_xor(iv, (u8 *)&sector, 8);
+       crypto_xor_cpy(iv, tcw->iv_seed, (u8 *)&sector, 8);
        if (cc->iv_size > 8)
-               crypto_xor(&iv[8], (u8 *)&sector, cc->iv_size - 8);
+               crypto_xor_cpy(&iv[8], tcw->iv_seed + 8, (u8 *)&sector,
+                              cc->iv_size - 8);
 
        return r;
 }

diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h
index fd547f946bf8e0ada50ce9d1d9af98dbd29346dc..e3cebf640c00070e7fcc436e59cb1c59c2d57379 100644 (file)
--- a/include/crypto/algapi.h
+++ b/include/crypto/algapi.h
@@ -211,6 +211,25 @@ static inline void crypto_xor(u8 *dst, const u8 *src, unsigned int size)
        }
 }
 
+static inline void crypto_xor_cpy(u8 *dst, const u8 *src1, const u8 *src2,
+                                 unsigned int size)
+{
+       if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) &&
+           __builtin_constant_p(size) &&
+           (size % sizeof(unsigned long)) == 0) {
+               unsigned long *d = (unsigned long *)dst;
+               unsigned long *s1 = (unsigned long *)src1;
+               unsigned long *s2 = (unsigned long *)src2;
+
+               while (size > 0) {
+                       *d++ = *s1++ ^ *s2++;
+                       size -= sizeof(unsigned long);
+               }
+       } else {
+               __crypto_xor(dst, src1, src2, size);
+       }
+}
+
 int blkcipher_walk_done(struct blkcipher_desc *desc,
                        struct blkcipher_walk *walk, int err);
 int blkcipher_walk_virt(struct blkcipher_desc *desc,