crypto: powerpc - convert to use crypto_simd_usable()

Replace all calls to in_interrupt() in the PowerPC crypto code with
!crypto_simd_usable().  This causes the crypto self-tests to test the
no-SIMD code paths when CONFIG_CRYPTO_MANAGER_EXTRA_TESTS=y.

The p8_ghash algorithm is currently failing and needs to be fixed, as it
produces the wrong digest when no-SIMD updates are mixed with SIMD ones.

Signed-off-by: Eric Biggers <ebiggers@google.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

diff --git a/arch/powerpc/crypto/crc32c-vpmsum_glue.c b/arch/powerpc/crypto/crc32c-vpmsum_glue.c
index fd1d6c83f0c02df09c4d097b000289c43b323c69..c4fa242dd652d806aca6493850d763a69a52d2af 100644 (file)
--- a/arch/powerpc/crypto/crc32c-vpmsum_glue.c
+++ b/arch/powerpc/crypto/crc32c-vpmsum_glue.c
@@ -1,10 +1,12 @@
 #include <linux/crc32.h>
 #include <crypto/internal/hash.h>
 #include <linux/crc32.h>
 #include <crypto/internal/hash.h>

+#include <crypto/internal/simd.h>

 #include <linux/init.h>
 #include <linux/module.h>
 #include <linux/string.h>
 #include <linux/kernel.h>
 #include <linux/cpufeature.h>
 #include <linux/init.h>
 #include <linux/module.h>
 #include <linux/string.h>
 #include <linux/kernel.h>
 #include <linux/cpufeature.h>

+#include <asm/simd.h>

 #include <asm/switch_to.h>
 
 #define CHKSUM_BLOCK_SIZE      1
 #include <asm/switch_to.h>
 
 #define CHKSUM_BLOCK_SIZE      1


@@ -22,7 +24,7 @@ static u32 crc32c_vpmsum(u32 crc, unsigned char const *p, size_t len)
        unsigned int prealign;
        unsigned int tail;
 
        unsigned int prealign;
        unsigned int tail;
 

-       if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) || in_interrupt())
+       if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) || !crypto_simd_usable())

                return __crc32c_le(crc, p, len);
 
        if ((unsigned long)p & VMX_ALIGN_MASK) {
                return __crc32c_le(crc, p, len);
 
        if ((unsigned long)p & VMX_ALIGN_MASK) {

diff --git a/arch/powerpc/crypto/crct10dif-vpmsum_glue.c b/arch/powerpc/crypto/crct10dif-vpmsum_glue.c
index 02ea277863d15001be3c919aa17c7f2ccd73aa1a..e27ff16573b5bf359da957e7a7c6be8d115fa002 100644 (file)
--- a/arch/powerpc/crypto/crct10dif-vpmsum_glue.c
+++ b/arch/powerpc/crypto/crct10dif-vpmsum_glue.c
@@ -12,11 +12,13 @@
 
 #include <linux/crc-t10dif.h>
 #include <crypto/internal/hash.h>
 
 #include <linux/crc-t10dif.h>
 #include <crypto/internal/hash.h>

+#include <crypto/internal/simd.h>

 #include <linux/init.h>
 #include <linux/module.h>
 #include <linux/string.h>
 #include <linux/kernel.h>
 #include <linux/cpufeature.h>
 #include <linux/init.h>
 #include <linux/module.h>
 #include <linux/string.h>
 #include <linux/kernel.h>
 #include <linux/cpufeature.h>

+#include <asm/simd.h>

 #include <asm/switch_to.h>
 
 #define VMX_ALIGN              16
 #include <asm/switch_to.h>
 
 #define VMX_ALIGN              16


@@ -32,7 +34,7 @@ static u16 crct10dif_vpmsum(u16 crci, unsigned char const *p, size_t len)
        unsigned int tail;
        u32 crc = crci;
 
        unsigned int tail;
        u32 crc = crci;
 

-       if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) || in_interrupt())
+       if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) || !crypto_simd_usable())

                return crc_t10dif_generic(crc, p, len);
 
        if ((unsigned long)p & VMX_ALIGN_MASK) {
                return crc_t10dif_generic(crc, p, len);
 
        if ((unsigned long)p & VMX_ALIGN_MASK) {

diff --git a/arch/powerpc/include/asm/Kbuild b/arch/powerpc/include/asm/Kbuild
index a0c132bedfae86965c2f7c850098b65420c2c5fc..5ac3dead69523e1d6081d44429fbca14e05e9aeb 100644 (file)
--- a/arch/powerpc/include/asm/Kbuild
+++ b/arch/powerpc/include/asm/Kbuild
@@ -11,3 +11,4 @@ generic-y += preempt.h
 generic-y += rwsem.h
 generic-y += vtime.h
 generic-y += msi.h
 generic-y += rwsem.h
 generic-y += vtime.h
 generic-y += msi.h

+generic-y += simd.h

diff --git a/drivers/crypto/vmx/aes.c b/drivers/crypto/vmx/aes.c
index b00d6947e02f4c5c283fe0d8cdd375754d8ef3df..603a620819941957cc3ce4c2bccf7f4abb829be2 100644 (file)
--- a/drivers/crypto/vmx/aes.c
+++ b/drivers/crypto/vmx/aes.c
@@ -23,9 +23,10 @@
 #include <linux/err.h>
 #include <linux/crypto.h>
 #include <linux/delay.h>
 #include <linux/err.h>
 #include <linux/crypto.h>
 #include <linux/delay.h>

-#include <linux/hardirq.h>
+#include <asm/simd.h>

 #include <asm/switch_to.h>
 #include <crypto/aes.h>
 #include <asm/switch_to.h>
 #include <crypto/aes.h>

+#include <crypto/internal/simd.h>

 
 #include "aesp8-ppc.h"
 
 
 #include "aesp8-ppc.h"
 


@@ -92,7 +93,7 @@ static void p8_aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 {
        struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm);
 
 {
        struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm);
 

-       if (in_interrupt()) {
+       if (!crypto_simd_usable()) {

                crypto_cipher_encrypt_one(ctx->fallback, dst, src);
        } else {
                preempt_disable();
                crypto_cipher_encrypt_one(ctx->fallback, dst, src);
        } else {
                preempt_disable();


@@ -109,7 +110,7 @@ static void p8_aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 {
        struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm);
 
 {
        struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm);
 

-       if (in_interrupt()) {
+       if (!crypto_simd_usable()) {

                crypto_cipher_decrypt_one(ctx->fallback, dst, src);
        } else {
                preempt_disable();
                crypto_cipher_decrypt_one(ctx->fallback, dst, src);
        } else {
                preempt_disable();

diff --git a/drivers/crypto/vmx/aes_cbc.c b/drivers/crypto/vmx/aes_cbc.c
index fbe882ef1bc5dd88be2ccce06d7e16912554bee0..a1a9a6f0d42cf0ae3868fe3ff53249ad4753dcd8 100644 (file)
--- a/drivers/crypto/vmx/aes_cbc.c
+++ b/drivers/crypto/vmx/aes_cbc.c
@@ -23,9 +23,10 @@
 #include <linux/err.h>
 #include <linux/crypto.h>
 #include <linux/delay.h>
 #include <linux/err.h>
 #include <linux/crypto.h>
 #include <linux/delay.h>

-#include <linux/hardirq.h>
+#include <asm/simd.h>

 #include <asm/switch_to.h>
 #include <crypto/aes.h>
 #include <asm/switch_to.h>
 #include <crypto/aes.h>

+#include <crypto/internal/simd.h>

 #include <crypto/scatterwalk.h>
 #include <crypto/skcipher.h>
 
 #include <crypto/scatterwalk.h>
 #include <crypto/skcipher.h>
 


@@ -100,7 +101,7 @@ static int p8_aes_cbc_encrypt(struct blkcipher_desc *desc,
        struct p8_aes_cbc_ctx *ctx =
                crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm));
 
        struct p8_aes_cbc_ctx *ctx =
                crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm));
 

-       if (in_interrupt()) {
+       if (!crypto_simd_usable()) {

                SYNC_SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback);
                skcipher_request_set_sync_tfm(req, ctx->fallback);
                skcipher_request_set_callback(req, desc->flags, NULL, NULL);
                SYNC_SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback);
                skcipher_request_set_sync_tfm(req, ctx->fallback);
                skcipher_request_set_callback(req, desc->flags, NULL, NULL);


@@ -139,7 +140,7 @@ static int p8_aes_cbc_decrypt(struct blkcipher_desc *desc,
        struct p8_aes_cbc_ctx *ctx =
                crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm));
 
        struct p8_aes_cbc_ctx *ctx =
                crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm));
 

-       if (in_interrupt()) {
+       if (!crypto_simd_usable()) {

                SYNC_SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback);
                skcipher_request_set_sync_tfm(req, ctx->fallback);
                skcipher_request_set_callback(req, desc->flags, NULL, NULL);
                SYNC_SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback);
                skcipher_request_set_sync_tfm(req, ctx->fallback);
                skcipher_request_set_callback(req, desc->flags, NULL, NULL);

diff --git a/drivers/crypto/vmx/aes_ctr.c b/drivers/crypto/vmx/aes_ctr.c
index 214c69db9ebdf8cdeaf033596f040065ea87d51c..192a53512f5e8f4b216aaa1f532b5064c34af145 100644 (file)
--- a/drivers/crypto/vmx/aes_ctr.c
+++ b/drivers/crypto/vmx/aes_ctr.c
@@ -23,9 +23,10 @@
 #include <linux/err.h>
 #include <linux/crypto.h>
 #include <linux/delay.h>
 #include <linux/err.h>
 #include <linux/crypto.h>
 #include <linux/delay.h>

-#include <linux/hardirq.h>
+#include <asm/simd.h>

 #include <asm/switch_to.h>
 #include <crypto/aes.h>
 #include <asm/switch_to.h>
 #include <crypto/aes.h>

+#include <crypto/internal/simd.h>

 #include <crypto/scatterwalk.h>
 #include <crypto/skcipher.h>
 
 #include <crypto/scatterwalk.h>
 #include <crypto/skcipher.h>
 


@@ -119,7 +120,7 @@ static int p8_aes_ctr_crypt(struct blkcipher_desc *desc,
        struct p8_aes_ctr_ctx *ctx =
                crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm));
 
        struct p8_aes_ctr_ctx *ctx =
                crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm));
 

-       if (in_interrupt()) {
+       if (!crypto_simd_usable()) {

                SYNC_SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback);
                skcipher_request_set_sync_tfm(req, ctx->fallback);
                skcipher_request_set_callback(req, desc->flags, NULL, NULL);
                SYNC_SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback);
                skcipher_request_set_sync_tfm(req, ctx->fallback);
                skcipher_request_set_callback(req, desc->flags, NULL, NULL);

diff --git a/drivers/crypto/vmx/aes_xts.c b/drivers/crypto/vmx/aes_xts.c
index 5bf4c3856650211bc396708d7946421551794b2f..00d412d811ae6010fd775d9c51b8af3516041a6c 100644 (file)
--- a/drivers/crypto/vmx/aes_xts.c
+++ b/drivers/crypto/vmx/aes_xts.c
@@ -23,9 +23,10 @@
 #include <linux/err.h>
 #include <linux/crypto.h>
 #include <linux/delay.h>
 #include <linux/err.h>
 #include <linux/crypto.h>
 #include <linux/delay.h>

-#include <linux/hardirq.h>
+#include <asm/simd.h>

 #include <asm/switch_to.h>
 #include <crypto/aes.h>
 #include <asm/switch_to.h>
 #include <crypto/aes.h>

+#include <crypto/internal/simd.h>

 #include <crypto/scatterwalk.h>
 #include <crypto/xts.h>
 #include <crypto/skcipher.h>
 #include <crypto/scatterwalk.h>
 #include <crypto/xts.h>
 #include <crypto/skcipher.h>


@@ -109,7 +110,7 @@ static int p8_aes_xts_crypt(struct blkcipher_desc *desc,
        struct p8_aes_xts_ctx *ctx =
                crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm));
 
        struct p8_aes_xts_ctx *ctx =
                crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm));
 

-       if (in_interrupt()) {
+       if (!crypto_simd_usable()) {

                SYNC_SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback);
                skcipher_request_set_sync_tfm(req, ctx->fallback);
                skcipher_request_set_callback(req, desc->flags, NULL, NULL);
                SYNC_SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback);
                skcipher_request_set_sync_tfm(req, ctx->fallback);
                skcipher_request_set_callback(req, desc->flags, NULL, NULL);

diff --git a/drivers/crypto/vmx/ghash.c b/drivers/crypto/vmx/ghash.c
index dd8b8716467a2cdd314683541436de0742b0106e..611ff591410ead8de61bdb39042b6275ed881eb6 100644 (file)
--- a/drivers/crypto/vmx/ghash.c
+++ b/drivers/crypto/vmx/ghash.c
@@ -23,16 +23,15 @@
 #include <linux/err.h>
 #include <linux/crypto.h>
 #include <linux/delay.h>
 #include <linux/err.h>
 #include <linux/crypto.h>
 #include <linux/delay.h>

-#include <linux/hardirq.h>
+#include <asm/simd.h>

 #include <asm/switch_to.h>
 #include <crypto/aes.h>
 #include <crypto/ghash.h>
 #include <crypto/scatterwalk.h>
 #include <crypto/internal/hash.h>
 #include <asm/switch_to.h>
 #include <crypto/aes.h>
 #include <crypto/ghash.h>
 #include <crypto/scatterwalk.h>
 #include <crypto/internal/hash.h>

+#include <crypto/internal/simd.h>

 #include <crypto/b128ops.h>
 
 #include <crypto/b128ops.h>
 

-#define IN_INTERRUPT in_interrupt()
-

 void gcm_init_p8(u128 htable[16], const u64 Xi[2]);
 void gcm_gmult_p8(u64 Xi[2], const u128 htable[16]);
 void gcm_ghash_p8(u64 Xi[2], const u128 htable[16],
 void gcm_init_p8(u128 htable[16], const u64 Xi[2]);
 void gcm_gmult_p8(u64 Xi[2], const u128 htable[16]);
 void gcm_ghash_p8(u64 Xi[2], const u128 htable[16],


@@ -131,7 +130,7 @@ static int p8_ghash_update(struct shash_desc *desc,
        struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm));
        struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
 
        struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm));
        struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
 

-       if (IN_INTERRUPT) {
+       if (!crypto_simd_usable()) {

                return crypto_shash_update(&dctx->fallback_desc, src,
                                           srclen);
        } else {
                return crypto_shash_update(&dctx->fallback_desc, src,
                                           srclen);
        } else {


@@ -182,7 +181,7 @@ static int p8_ghash_final(struct shash_desc *desc, u8 *out)
        struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm));
        struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
 
        struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm));
        struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
 

-       if (IN_INTERRUPT) {
+       if (!crypto_simd_usable()) {

                return crypto_shash_final(&dctx->fallback_desc, out);
        } else {
                if (dctx->bytes) {
                return crypto_shash_final(&dctx->fallback_desc, out);
        } else {
                if (dctx->bytes) {