2 * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved.
3 * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <linux/module.h>
37 #include <net/inet_common.h>
38 #include <linux/highmem.h>
39 #include <linux/netdevice.h>
40 #include <linux/sched/signal.h>
41 #include <linux/inetdevice.h>
45 MODULE_AUTHOR("Mellanox Technologies");
46 MODULE_DESCRIPTION("Transport Layer Security Support");
47 MODULE_LICENSE("Dual BSD/GPL");
48 MODULE_ALIAS_TCP_ULP("tls");
56 static struct proto *saved_tcpv6_prot;
57 static DEFINE_MUTEX(tcpv6_prot_mutex);
58 static LIST_HEAD(device_list);
59 static DEFINE_MUTEX(device_mutex);
60 static struct proto tls_prots[TLS_NUM_PROTS][TLS_NUM_CONFIG][TLS_NUM_CONFIG];
61 static struct proto_ops tls_sw_proto_ops;
63 static void update_sk_prot(struct sock *sk, struct tls_context *ctx)
65 int ip_ver = sk->sk_family == AF_INET6 ? TLSV6 : TLSV4;
67 sk->sk_prot = &tls_prots[ip_ver][ctx->tx_conf][ctx->rx_conf];
70 int wait_on_pending_writer(struct sock *sk, long *timeo)
73 DEFINE_WAIT_FUNC(wait, woken_wake_function);
75 add_wait_queue(sk_sleep(sk), &wait);
82 if (signal_pending(current)) {
83 rc = sock_intr_errno(*timeo);
87 if (sk_wait_event(sk, timeo, !sk->sk_write_pending, &wait))
90 remove_wait_queue(sk_sleep(sk), &wait);
94 int tls_push_sg(struct sock *sk,
95 struct tls_context *ctx,
96 struct scatterlist *sg,
100 int sendpage_flags = flags | MSG_SENDPAGE_NOTLAST;
104 int offset = first_offset;
106 size = sg->length - offset;
107 offset += sg->offset;
109 ctx->in_tcp_sendpages = true;
112 sendpage_flags = flags;
114 /* is sending application-limited? */
115 tcp_rate_check_app_limited(sk);
118 ret = do_tcp_sendpages(sk, p, offset, size, sendpage_flags);
127 offset -= sg->offset;
128 ctx->partially_sent_offset = offset;
129 ctx->partially_sent_record = (void *)sg;
130 ctx->in_tcp_sendpages = false;
135 sk_mem_uncharge(sk, sg->length);
144 clear_bit(TLS_PENDING_CLOSED_RECORD, &ctx->flags);
145 ctx->in_tcp_sendpages = false;
146 ctx->sk_write_space(sk);
151 static int tls_handle_open_record(struct sock *sk, int flags)
153 struct tls_context *ctx = tls_get_ctx(sk);
155 if (tls_is_pending_open_record(ctx))
156 return ctx->push_pending_record(sk, flags);
161 int tls_proccess_cmsg(struct sock *sk, struct msghdr *msg,
162 unsigned char *record_type)
164 struct cmsghdr *cmsg;
167 for_each_cmsghdr(cmsg, msg) {
168 if (!CMSG_OK(msg, cmsg))
170 if (cmsg->cmsg_level != SOL_TLS)
173 switch (cmsg->cmsg_type) {
174 case TLS_SET_RECORD_TYPE:
175 if (cmsg->cmsg_len < CMSG_LEN(sizeof(*record_type)))
178 if (msg->msg_flags & MSG_MORE)
181 rc = tls_handle_open_record(sk, msg->msg_flags);
185 *record_type = *(unsigned char *)CMSG_DATA(cmsg);
196 int tls_push_pending_closed_record(struct sock *sk, struct tls_context *ctx,
197 int flags, long *timeo)
199 struct scatterlist *sg;
202 if (!tls_is_partially_sent_record(ctx))
203 return ctx->push_pending_record(sk, flags);
205 sg = ctx->partially_sent_record;
206 offset = ctx->partially_sent_offset;
208 ctx->partially_sent_record = NULL;
209 return tls_push_sg(sk, ctx, sg, offset, flags);
212 static void tls_write_space(struct sock *sk)
214 struct tls_context *ctx = tls_get_ctx(sk);
216 /* We are already sending pages, ignore notification */
217 if (ctx->in_tcp_sendpages)
220 if (!sk->sk_write_pending && tls_is_pending_closed_record(ctx)) {
221 gfp_t sk_allocation = sk->sk_allocation;
225 sk->sk_allocation = GFP_ATOMIC;
226 rc = tls_push_pending_closed_record(sk, ctx,
230 sk->sk_allocation = sk_allocation;
236 ctx->sk_write_space(sk);
239 static void tls_sk_proto_close(struct sock *sk, long timeout)
241 struct tls_context *ctx = tls_get_ctx(sk);
242 long timeo = sock_sndtimeo(sk, 0);
243 void (*sk_proto_close)(struct sock *sk, long timeout);
244 bool free_ctx = false;
247 sk_proto_close = ctx->sk_proto_close;
249 if ((ctx->tx_conf == TLS_HW_RECORD && ctx->rx_conf == TLS_HW_RECORD) ||
250 (ctx->tx_conf == TLS_BASE && ctx->rx_conf == TLS_BASE)) {
252 goto skip_tx_cleanup;
255 if (!tls_complete_pending_work(sk, ctx, 0, &timeo))
256 tls_handle_open_record(sk, 0);
258 if (ctx->partially_sent_record) {
259 struct scatterlist *sg = ctx->partially_sent_record;
262 put_page(sg_page(sg));
263 sk_mem_uncharge(sk, sg->length);
271 /* We need these for tls_sw_fallback handling of other packets */
272 if (ctx->tx_conf == TLS_SW) {
273 kfree(ctx->tx.rec_seq);
275 tls_sw_free_resources_tx(sk);
278 if (ctx->rx_conf == TLS_SW) {
279 kfree(ctx->rx.rec_seq);
281 tls_sw_free_resources_rx(sk);
284 #ifdef CONFIG_TLS_DEVICE
285 if (ctx->rx_conf == TLS_HW)
286 tls_device_offload_cleanup_rx(sk);
288 if (ctx->tx_conf != TLS_HW && ctx->rx_conf != TLS_HW) {
298 sk_proto_close(sk, timeout);
299 /* free ctx for TLS_HW_RECORD, used by tcp_set_state
300 * for sk->sk_prot->unhash [tls_hw_unhash]
306 static int do_tls_getsockopt_tx(struct sock *sk, char __user *optval,
310 struct tls_context *ctx = tls_get_ctx(sk);
311 struct tls_crypto_info *crypto_info;
314 if (get_user(len, optlen))
317 if (!optval || (len < sizeof(*crypto_info))) {
327 /* get user crypto info */
328 crypto_info = &ctx->crypto_send;
330 if (!TLS_CRYPTO_INFO_READY(crypto_info)) {
335 if (len == sizeof(*crypto_info)) {
336 if (copy_to_user(optval, crypto_info, sizeof(*crypto_info)))
341 switch (crypto_info->cipher_type) {
342 case TLS_CIPHER_AES_GCM_128: {
343 struct tls12_crypto_info_aes_gcm_128 *
344 crypto_info_aes_gcm_128 =
345 container_of(crypto_info,
346 struct tls12_crypto_info_aes_gcm_128,
349 if (len != sizeof(*crypto_info_aes_gcm_128)) {
354 memcpy(crypto_info_aes_gcm_128->iv,
355 ctx->tx.iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE,
356 TLS_CIPHER_AES_GCM_128_IV_SIZE);
357 memcpy(crypto_info_aes_gcm_128->rec_seq, ctx->tx.rec_seq,
358 TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE);
360 if (copy_to_user(optval,
361 crypto_info_aes_gcm_128,
362 sizeof(*crypto_info_aes_gcm_128)))
374 static int do_tls_getsockopt(struct sock *sk, int optname,
375 char __user *optval, int __user *optlen)
381 rc = do_tls_getsockopt_tx(sk, optval, optlen);
390 static int tls_getsockopt(struct sock *sk, int level, int optname,
391 char __user *optval, int __user *optlen)
393 struct tls_context *ctx = tls_get_ctx(sk);
395 if (level != SOL_TLS)
396 return ctx->getsockopt(sk, level, optname, optval, optlen);
398 return do_tls_getsockopt(sk, optname, optval, optlen);
401 static int do_tls_setsockopt_conf(struct sock *sk, char __user *optval,
402 unsigned int optlen, int tx)
404 struct tls_crypto_info *crypto_info;
405 struct tls_context *ctx = tls_get_ctx(sk);
409 if (!optval || (optlen < sizeof(*crypto_info))) {
415 crypto_info = &ctx->crypto_send;
417 crypto_info = &ctx->crypto_recv;
419 /* Currently we don't support set crypto info more than one time */
420 if (TLS_CRYPTO_INFO_READY(crypto_info)) {
425 rc = copy_from_user(crypto_info, optval, sizeof(*crypto_info));
428 goto err_crypto_info;
432 if (crypto_info->version != TLS_1_2_VERSION) {
434 goto err_crypto_info;
437 switch (crypto_info->cipher_type) {
438 case TLS_CIPHER_AES_GCM_128: {
439 if (optlen != sizeof(struct tls12_crypto_info_aes_gcm_128)) {
441 goto err_crypto_info;
443 rc = copy_from_user(crypto_info + 1, optval + sizeof(*crypto_info),
444 optlen - sizeof(*crypto_info));
447 goto err_crypto_info;
453 goto err_crypto_info;
457 #ifdef CONFIG_TLS_DEVICE
458 rc = tls_set_device_offload(sk, ctx);
464 rc = tls_set_sw_offload(sk, ctx, 1);
468 #ifdef CONFIG_TLS_DEVICE
469 rc = tls_set_device_offload_rx(sk, ctx);
475 rc = tls_set_sw_offload(sk, ctx, 0);
481 goto err_crypto_info;
487 update_sk_prot(sk, ctx);
489 ctx->sk_write_space = sk->sk_write_space;
490 sk->sk_write_space = tls_write_space;
492 sk->sk_socket->ops = &tls_sw_proto_ops;
497 memset(crypto_info, 0, sizeof(*crypto_info));
502 static int do_tls_setsockopt(struct sock *sk, int optname,
503 char __user *optval, unsigned int optlen)
511 rc = do_tls_setsockopt_conf(sk, optval, optlen,
522 static int tls_setsockopt(struct sock *sk, int level, int optname,
523 char __user *optval, unsigned int optlen)
525 struct tls_context *ctx = tls_get_ctx(sk);
527 if (level != SOL_TLS)
528 return ctx->setsockopt(sk, level, optname, optval, optlen);
530 return do_tls_setsockopt(sk, optname, optval, optlen);
533 static struct tls_context *create_ctx(struct sock *sk)
535 struct inet_connection_sock *icsk = inet_csk(sk);
536 struct tls_context *ctx;
538 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
542 icsk->icsk_ulp_data = ctx;
546 static int tls_hw_prot(struct sock *sk)
548 struct tls_context *ctx;
549 struct tls_device *dev;
552 mutex_lock(&device_mutex);
553 list_for_each_entry(dev, &device_list, dev_list) {
554 if (dev->feature && dev->feature(dev)) {
555 ctx = create_ctx(sk);
559 ctx->hash = sk->sk_prot->hash;
560 ctx->unhash = sk->sk_prot->unhash;
561 ctx->sk_proto_close = sk->sk_prot->close;
562 ctx->rx_conf = TLS_HW_RECORD;
563 ctx->tx_conf = TLS_HW_RECORD;
564 update_sk_prot(sk, ctx);
570 mutex_unlock(&device_mutex);
574 static void tls_hw_unhash(struct sock *sk)
576 struct tls_context *ctx = tls_get_ctx(sk);
577 struct tls_device *dev;
579 mutex_lock(&device_mutex);
580 list_for_each_entry(dev, &device_list, dev_list) {
582 dev->unhash(dev, sk);
584 mutex_unlock(&device_mutex);
588 static int tls_hw_hash(struct sock *sk)
590 struct tls_context *ctx = tls_get_ctx(sk);
591 struct tls_device *dev;
595 mutex_lock(&device_mutex);
596 list_for_each_entry(dev, &device_list, dev_list) {
598 err |= dev->hash(dev, sk);
600 mutex_unlock(&device_mutex);
607 static void build_protos(struct proto prot[TLS_NUM_CONFIG][TLS_NUM_CONFIG],
610 prot[TLS_BASE][TLS_BASE] = *base;
611 prot[TLS_BASE][TLS_BASE].setsockopt = tls_setsockopt;
612 prot[TLS_BASE][TLS_BASE].getsockopt = tls_getsockopt;
613 prot[TLS_BASE][TLS_BASE].close = tls_sk_proto_close;
615 prot[TLS_SW][TLS_BASE] = prot[TLS_BASE][TLS_BASE];
616 prot[TLS_SW][TLS_BASE].sendmsg = tls_sw_sendmsg;
617 prot[TLS_SW][TLS_BASE].sendpage = tls_sw_sendpage;
619 prot[TLS_BASE][TLS_SW] = prot[TLS_BASE][TLS_BASE];
620 prot[TLS_BASE][TLS_SW].recvmsg = tls_sw_recvmsg;
621 prot[TLS_BASE][TLS_SW].close = tls_sk_proto_close;
623 prot[TLS_SW][TLS_SW] = prot[TLS_SW][TLS_BASE];
624 prot[TLS_SW][TLS_SW].recvmsg = tls_sw_recvmsg;
625 prot[TLS_SW][TLS_SW].close = tls_sk_proto_close;
627 #ifdef CONFIG_TLS_DEVICE
628 prot[TLS_HW][TLS_BASE] = prot[TLS_BASE][TLS_BASE];
629 prot[TLS_HW][TLS_BASE].sendmsg = tls_device_sendmsg;
630 prot[TLS_HW][TLS_BASE].sendpage = tls_device_sendpage;
632 prot[TLS_HW][TLS_SW] = prot[TLS_BASE][TLS_SW];
633 prot[TLS_HW][TLS_SW].sendmsg = tls_device_sendmsg;
634 prot[TLS_HW][TLS_SW].sendpage = tls_device_sendpage;
636 prot[TLS_BASE][TLS_HW] = prot[TLS_BASE][TLS_SW];
638 prot[TLS_SW][TLS_HW] = prot[TLS_SW][TLS_SW];
640 prot[TLS_HW][TLS_HW] = prot[TLS_HW][TLS_SW];
643 prot[TLS_HW_RECORD][TLS_HW_RECORD] = *base;
644 prot[TLS_HW_RECORD][TLS_HW_RECORD].hash = tls_hw_hash;
645 prot[TLS_HW_RECORD][TLS_HW_RECORD].unhash = tls_hw_unhash;
646 prot[TLS_HW_RECORD][TLS_HW_RECORD].close = tls_sk_proto_close;
649 static int tls_init(struct sock *sk)
651 int ip_ver = sk->sk_family == AF_INET6 ? TLSV6 : TLSV4;
652 struct tls_context *ctx;
658 /* The TLS ulp is currently supported only for TCP sockets
659 * in ESTABLISHED state.
660 * Supporting sockets in LISTEN state will require us
661 * to modify the accept implementation to clone rather then
662 * share the ulp context.
664 if (sk->sk_state != TCP_ESTABLISHED)
667 /* allocate tls context */
668 ctx = create_ctx(sk);
673 ctx->setsockopt = sk->sk_prot->setsockopt;
674 ctx->getsockopt = sk->sk_prot->getsockopt;
675 ctx->sk_proto_close = sk->sk_prot->close;
677 /* Build IPv6 TLS whenever the address of tcpv6 _prot changes */
678 if (ip_ver == TLSV6 &&
679 unlikely(sk->sk_prot != smp_load_acquire(&saved_tcpv6_prot))) {
680 mutex_lock(&tcpv6_prot_mutex);
681 if (likely(sk->sk_prot != saved_tcpv6_prot)) {
682 build_protos(tls_prots[TLSV6], sk->sk_prot);
683 smp_store_release(&saved_tcpv6_prot, sk->sk_prot);
685 mutex_unlock(&tcpv6_prot_mutex);
688 ctx->tx_conf = TLS_BASE;
689 ctx->rx_conf = TLS_BASE;
690 update_sk_prot(sk, ctx);
695 void tls_register_device(struct tls_device *device)
697 mutex_lock(&device_mutex);
698 list_add_tail(&device->dev_list, &device_list);
699 mutex_unlock(&device_mutex);
701 EXPORT_SYMBOL(tls_register_device);
703 void tls_unregister_device(struct tls_device *device)
705 mutex_lock(&device_mutex);
706 list_del(&device->dev_list);
707 mutex_unlock(&device_mutex);
709 EXPORT_SYMBOL(tls_unregister_device);
711 static struct tcp_ulp_ops tcp_tls_ulp_ops __read_mostly = {
714 .user_visible = true,
715 .owner = THIS_MODULE,
719 static int __init tls_register(void)
721 build_protos(tls_prots[TLSV4], &tcp_prot);
723 tls_sw_proto_ops = inet_stream_ops;
724 tls_sw_proto_ops.poll = tls_sw_poll;
725 tls_sw_proto_ops.splice_read = tls_sw_splice_read;
727 #ifdef CONFIG_TLS_DEVICE
730 tcp_register_ulp(&tcp_tls_ulp_ops);
735 static void __exit tls_unregister(void)
737 tcp_unregister_ulp(&tcp_tls_ulp_ops);
738 #ifdef CONFIG_TLS_DEVICE
739 tls_device_cleanup();
743 module_init(tls_register);
744 module_exit(tls_unregister);