1 // SPDX-License-Identifier: GPL-2.0 OR MIT
3 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
5 * This is an implementation of the ChaCha20Poly1305 AEAD construction.
7 * Information: https://tools.ietf.org/html/rfc8439
10 #include <crypto/algapi.h>
11 #include <crypto/chacha20poly1305.h>
12 #include <crypto/chacha.h>
13 #include <crypto/poly1305.h>
14 #include <crypto/scatterwalk.h>
16 #include <asm/unaligned.h>
17 #include <linux/kernel.h>
18 #include <linux/init.h>
20 #include <linux/module.h>
22 #define CHACHA_KEY_WORDS (CHACHA_KEY_SIZE / sizeof(u32))
24 bool __init chacha20poly1305_selftest(void);
26 static void chacha_load_key(u32 *k, const u8 *in)
28 k[0] = get_unaligned_le32(in);
29 k[1] = get_unaligned_le32(in + 4);
30 k[2] = get_unaligned_le32(in + 8);
31 k[3] = get_unaligned_le32(in + 12);
32 k[4] = get_unaligned_le32(in + 16);
33 k[5] = get_unaligned_le32(in + 20);
34 k[6] = get_unaligned_le32(in + 24);
35 k[7] = get_unaligned_le32(in + 28);
38 static void xchacha_init(u32 *chacha_state, const u8 *key, const u8 *nonce)
40 u32 k[CHACHA_KEY_WORDS];
41 u8 iv[CHACHA_IV_SIZE];
44 memcpy(iv + 8, nonce + 16, 8);
46 chacha_load_key(k, key);
48 /* Compute the subkey given the original key and first 128 nonce bits */
49 chacha_init(chacha_state, k, nonce);
50 hchacha_block(chacha_state, k, 20);
52 chacha_init(chacha_state, k, iv);
54 memzero_explicit(k, sizeof(k));
55 memzero_explicit(iv, sizeof(iv));
59 __chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
60 const u8 *ad, const size_t ad_len, u32 *chacha_state)
62 const u8 *pad0 = page_address(ZERO_PAGE(0));
63 struct poly1305_desc_ctx poly1305_state;
65 u8 block0[POLY1305_KEY_SIZE];
69 chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
70 poly1305_init(&poly1305_state, b.block0);
72 poly1305_update(&poly1305_state, ad, ad_len);
74 poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
76 chacha20_crypt(chacha_state, dst, src, src_len);
78 poly1305_update(&poly1305_state, dst, src_len);
80 poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));
82 b.lens[0] = cpu_to_le64(ad_len);
83 b.lens[1] = cpu_to_le64(src_len);
84 poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
86 poly1305_final(&poly1305_state, dst + src_len);
88 memzero_explicit(chacha_state, CHACHA_STATE_WORDS * sizeof(u32));
89 memzero_explicit(&b, sizeof(b));
92 void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
93 const u8 *ad, const size_t ad_len,
95 const u8 key[CHACHA20POLY1305_KEY_SIZE])
97 u32 chacha_state[CHACHA_STATE_WORDS];
98 u32 k[CHACHA_KEY_WORDS];
101 chacha_load_key(k, key);
104 iv[1] = cpu_to_le64(nonce);
106 chacha_init(chacha_state, k, (u8 *)iv);
107 __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state);
109 memzero_explicit(iv, sizeof(iv));
110 memzero_explicit(k, sizeof(k));
112 EXPORT_SYMBOL(chacha20poly1305_encrypt);
114 void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
115 const u8 *ad, const size_t ad_len,
116 const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
117 const u8 key[CHACHA20POLY1305_KEY_SIZE])
119 u32 chacha_state[CHACHA_STATE_WORDS];
121 xchacha_init(chacha_state, key, nonce);
122 __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state);
124 EXPORT_SYMBOL(xchacha20poly1305_encrypt);
127 __chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
128 const u8 *ad, const size_t ad_len, u32 *chacha_state)
130 const u8 *pad0 = page_address(ZERO_PAGE(0));
131 struct poly1305_desc_ctx poly1305_state;
135 u8 block0[POLY1305_KEY_SIZE];
136 u8 mac[POLY1305_DIGEST_SIZE];
140 if (unlikely(src_len < POLY1305_DIGEST_SIZE))
143 chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
144 poly1305_init(&poly1305_state, b.block0);
146 poly1305_update(&poly1305_state, ad, ad_len);
148 poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
150 dst_len = src_len - POLY1305_DIGEST_SIZE;
151 poly1305_update(&poly1305_state, src, dst_len);
153 poly1305_update(&poly1305_state, pad0, 0x10 - (dst_len & 0xf));
155 b.lens[0] = cpu_to_le64(ad_len);
156 b.lens[1] = cpu_to_le64(dst_len);
157 poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
159 poly1305_final(&poly1305_state, b.mac);
161 ret = crypto_memneq(b.mac, src + dst_len, POLY1305_DIGEST_SIZE);
163 chacha20_crypt(chacha_state, dst, src, dst_len);
165 memzero_explicit(&b, sizeof(b));
170 bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
171 const u8 *ad, const size_t ad_len,
173 const u8 key[CHACHA20POLY1305_KEY_SIZE])
175 u32 chacha_state[CHACHA_STATE_WORDS];
176 u32 k[CHACHA_KEY_WORDS];
180 chacha_load_key(k, key);
183 iv[1] = cpu_to_le64(nonce);
185 chacha_init(chacha_state, k, (u8 *)iv);
186 ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
189 memzero_explicit(chacha_state, sizeof(chacha_state));
190 memzero_explicit(iv, sizeof(iv));
191 memzero_explicit(k, sizeof(k));
194 EXPORT_SYMBOL(chacha20poly1305_decrypt);
196 bool xchacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
197 const u8 *ad, const size_t ad_len,
198 const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
199 const u8 key[CHACHA20POLY1305_KEY_SIZE])
201 u32 chacha_state[CHACHA_STATE_WORDS];
203 xchacha_init(chacha_state, key, nonce);
204 return __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
207 EXPORT_SYMBOL(xchacha20poly1305_decrypt);
210 bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src,
211 const size_t src_len,
212 const u8 *ad, const size_t ad_len,
214 const u8 key[CHACHA20POLY1305_KEY_SIZE],
217 const u8 *pad0 = page_address(ZERO_PAGE(0));
218 struct poly1305_desc_ctx poly1305_state;
219 u32 chacha_state[CHACHA_STATE_WORDS];
220 struct sg_mapping_iter miter;
227 u32 k[CHACHA_KEY_WORDS];
230 u8 block0[POLY1305_KEY_SIZE];
231 u8 chacha_stream[CHACHA_BLOCK_SIZE];
233 u8 mac[2][POLY1305_DIGEST_SIZE];
238 if (WARN_ON(src_len > INT_MAX))
241 chacha_load_key(b.k, key);
244 b.iv[1] = cpu_to_le64(nonce);
246 chacha_init(chacha_state, b.k, (u8 *)b.iv);
247 chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
248 poly1305_init(&poly1305_state, b.block0);
250 if (unlikely(ad_len)) {
251 poly1305_update(&poly1305_state, ad, ad_len);
253 poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
256 flags = SG_MITER_TO_SG;
258 flags |= SG_MITER_ATOMIC;
260 sg_miter_start(&miter, src, sg_nents(src), flags);
262 for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) {
263 u8 *addr = miter.addr;
264 size_t length = min_t(size_t, sl, miter.length);
267 poly1305_update(&poly1305_state, addr, length);
269 if (unlikely(partial)) {
270 size_t l = min(length, CHACHA_BLOCK_SIZE - partial);
272 crypto_xor(addr, b.chacha_stream + partial, l);
273 partial = (partial + l) & (CHACHA_BLOCK_SIZE - 1);
279 if (likely(length >= CHACHA_BLOCK_SIZE || length == sl)) {
282 if (unlikely(length < sl))
283 l &= ~(CHACHA_BLOCK_SIZE - 1);
284 chacha20_crypt(chacha_state, addr, addr, l);
289 if (unlikely(length > 0)) {
290 chacha20_crypt(chacha_state, b.chacha_stream, pad0,
292 crypto_xor(addr, b.chacha_stream, length);
297 poly1305_update(&poly1305_state, miter.addr,
298 min_t(size_t, sl, miter.length));
302 poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));
304 b.lens[0] = cpu_to_le64(ad_len);
305 b.lens[1] = cpu_to_le64(src_len);
306 poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
308 if (likely(sl <= -POLY1305_DIGEST_SIZE)) {
310 poly1305_final(&poly1305_state,
311 miter.addr + miter.length + sl);
314 poly1305_final(&poly1305_state, b.mac[0]);
315 ret = !crypto_memneq(b.mac[0],
316 miter.addr + miter.length + sl,
317 POLY1305_DIGEST_SIZE);
321 sg_miter_stop(&miter);
323 if (unlikely(sl > -POLY1305_DIGEST_SIZE)) {
324 poly1305_final(&poly1305_state, b.mac[1]);
325 scatterwalk_map_and_copy(b.mac[encrypt], src, src_len,
326 sizeof(b.mac[1]), encrypt);
328 !crypto_memneq(b.mac[0], b.mac[1], POLY1305_DIGEST_SIZE);
331 memzero_explicit(chacha_state, sizeof(chacha_state));
332 memzero_explicit(&b, sizeof(b));
337 bool chacha20poly1305_encrypt_sg_inplace(struct scatterlist *src, size_t src_len,
338 const u8 *ad, const size_t ad_len,
340 const u8 key[CHACHA20POLY1305_KEY_SIZE])
342 return chacha20poly1305_crypt_sg_inplace(src, src_len, ad, ad_len,
345 EXPORT_SYMBOL(chacha20poly1305_encrypt_sg_inplace);
347 bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len,
348 const u8 *ad, const size_t ad_len,
350 const u8 key[CHACHA20POLY1305_KEY_SIZE])
352 if (unlikely(src_len < POLY1305_DIGEST_SIZE))
355 return chacha20poly1305_crypt_sg_inplace(src,
356 src_len - POLY1305_DIGEST_SIZE,
357 ad, ad_len, nonce, key, 0);
359 EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace);
361 static int __init mod_init(void)
363 if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) &&
364 WARN_ON(!chacha20poly1305_selftest()))
369 module_init(mod_init);
370 MODULE_LICENSE("GPL v2");
371 MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction");
372 MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>");