2 * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved.
3 * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <linux/module.h>
37 #include <net/inet_common.h>
38 #include <linux/highmem.h>
39 #include <linux/netdevice.h>
40 #include <linux/sched/signal.h>
41 #include <linux/inetdevice.h>
45 MODULE_AUTHOR("Mellanox Technologies");
46 MODULE_DESCRIPTION("Transport Layer Security Support");
47 MODULE_LICENSE("Dual BSD/GPL");
48 MODULE_ALIAS_TCP_ULP("tls");
56 static struct proto
*saved_tcpv6_prot
;
57 static DEFINE_MUTEX(tcpv6_prot_mutex
);
58 static LIST_HEAD(device_list
);
59 static DEFINE_MUTEX(device_mutex
);
60 static struct proto tls_prots
[TLS_NUM_PROTS
][TLS_NUM_CONFIG
][TLS_NUM_CONFIG
];
61 static struct proto_ops tls_sw_proto_ops
;
63 static void update_sk_prot(struct sock
*sk
, struct tls_context
*ctx
)
65 int ip_ver
= sk
->sk_family
== AF_INET6
? TLSV6
: TLSV4
;
67 sk
->sk_prot
= &tls_prots
[ip_ver
][ctx
->tx_conf
][ctx
->rx_conf
];
70 int wait_on_pending_writer(struct sock
*sk
, long *timeo
)
73 DEFINE_WAIT_FUNC(wait
, woken_wake_function
);
75 add_wait_queue(sk_sleep(sk
), &wait
);
82 if (signal_pending(current
)) {
83 rc
= sock_intr_errno(*timeo
);
87 if (sk_wait_event(sk
, timeo
, !sk
->sk_write_pending
, &wait
))
90 remove_wait_queue(sk_sleep(sk
), &wait
);
94 int tls_push_sg(struct sock
*sk
,
95 struct tls_context
*ctx
,
96 struct scatterlist
*sg
,
100 int sendpage_flags
= flags
| MSG_SENDPAGE_NOTLAST
;
104 int offset
= first_offset
;
106 size
= sg
->length
- offset
;
107 offset
+= sg
->offset
;
109 ctx
->in_tcp_sendpages
= true;
112 sendpage_flags
= flags
;
114 /* is sending application-limited? */
115 tcp_rate_check_app_limited(sk
);
118 ret
= do_tcp_sendpages(sk
, p
, offset
, size
, sendpage_flags
);
127 offset
-= sg
->offset
;
128 ctx
->partially_sent_offset
= offset
;
129 ctx
->partially_sent_record
= (void *)sg
;
130 ctx
->in_tcp_sendpages
= false;
135 sk_mem_uncharge(sk
, sg
->length
);
144 clear_bit(TLS_PENDING_CLOSED_RECORD
, &ctx
->flags
);
145 ctx
->in_tcp_sendpages
= false;
146 ctx
->sk_write_space(sk
);
151 static int tls_handle_open_record(struct sock
*sk
, int flags
)
153 struct tls_context
*ctx
= tls_get_ctx(sk
);
155 if (tls_is_pending_open_record(ctx
))
156 return ctx
->push_pending_record(sk
, flags
);
161 int tls_proccess_cmsg(struct sock
*sk
, struct msghdr
*msg
,
162 unsigned char *record_type
)
164 struct cmsghdr
*cmsg
;
167 for_each_cmsghdr(cmsg
, msg
) {
168 if (!CMSG_OK(msg
, cmsg
))
170 if (cmsg
->cmsg_level
!= SOL_TLS
)
173 switch (cmsg
->cmsg_type
) {
174 case TLS_SET_RECORD_TYPE
:
175 if (cmsg
->cmsg_len
< CMSG_LEN(sizeof(*record_type
)))
178 if (msg
->msg_flags
& MSG_MORE
)
181 rc
= tls_handle_open_record(sk
, msg
->msg_flags
);
185 *record_type
= *(unsigned char *)CMSG_DATA(cmsg
);
196 int tls_push_pending_closed_record(struct sock
*sk
, struct tls_context
*ctx
,
197 int flags
, long *timeo
)
199 struct scatterlist
*sg
;
202 if (!tls_is_partially_sent_record(ctx
))
203 return ctx
->push_pending_record(sk
, flags
);
205 sg
= ctx
->partially_sent_record
;
206 offset
= ctx
->partially_sent_offset
;
208 ctx
->partially_sent_record
= NULL
;
209 return tls_push_sg(sk
, ctx
, sg
, offset
, flags
);
212 static void tls_write_space(struct sock
*sk
)
214 struct tls_context
*ctx
= tls_get_ctx(sk
);
216 /* If in_tcp_sendpages call lower protocol write space handler
217 * to ensure we wake up any waiting operations there. For example
218 * if do_tcp_sendpages where to call sk_wait_event.
220 if (ctx
->in_tcp_sendpages
) {
221 ctx
->sk_write_space(sk
);
225 if (!sk
->sk_write_pending
&& tls_is_pending_closed_record(ctx
)) {
226 gfp_t sk_allocation
= sk
->sk_allocation
;
230 sk
->sk_allocation
= GFP_ATOMIC
;
231 rc
= tls_push_pending_closed_record(sk
, ctx
,
235 sk
->sk_allocation
= sk_allocation
;
241 ctx
->sk_write_space(sk
);
244 static void tls_ctx_free(struct tls_context
*ctx
)
249 memzero_explicit(&ctx
->crypto_send
, sizeof(ctx
->crypto_send
));
250 memzero_explicit(&ctx
->crypto_recv
, sizeof(ctx
->crypto_recv
));
254 static void tls_sk_proto_close(struct sock
*sk
, long timeout
)
256 struct tls_context
*ctx
= tls_get_ctx(sk
);
257 long timeo
= sock_sndtimeo(sk
, 0);
258 void (*sk_proto_close
)(struct sock
*sk
, long timeout
);
259 bool free_ctx
= false;
262 sk_proto_close
= ctx
->sk_proto_close
;
264 if ((ctx
->tx_conf
== TLS_HW_RECORD
&& ctx
->rx_conf
== TLS_HW_RECORD
) ||
265 (ctx
->tx_conf
== TLS_BASE
&& ctx
->rx_conf
== TLS_BASE
)) {
267 goto skip_tx_cleanup
;
270 if (!tls_complete_pending_work(sk
, ctx
, 0, &timeo
))
271 tls_handle_open_record(sk
, 0);
273 if (ctx
->partially_sent_record
) {
274 struct scatterlist
*sg
= ctx
->partially_sent_record
;
277 put_page(sg_page(sg
));
278 sk_mem_uncharge(sk
, sg
->length
);
286 /* We need these for tls_sw_fallback handling of other packets */
287 if (ctx
->tx_conf
== TLS_SW
) {
288 kfree(ctx
->tx
.rec_seq
);
290 tls_sw_free_resources_tx(sk
);
293 if (ctx
->rx_conf
== TLS_SW
) {
294 kfree(ctx
->rx
.rec_seq
);
296 tls_sw_free_resources_rx(sk
);
299 #ifdef CONFIG_TLS_DEVICE
300 if (ctx
->rx_conf
== TLS_HW
)
301 tls_device_offload_cleanup_rx(sk
);
303 if (ctx
->tx_conf
!= TLS_HW
&& ctx
->rx_conf
!= TLS_HW
) {
313 sk_proto_close(sk
, timeout
);
314 /* free ctx for TLS_HW_RECORD, used by tcp_set_state
315 * for sk->sk_prot->unhash [tls_hw_unhash]
321 static int do_tls_getsockopt_tx(struct sock
*sk
, char __user
*optval
,
325 struct tls_context
*ctx
= tls_get_ctx(sk
);
326 struct tls_crypto_info
*crypto_info
;
329 if (get_user(len
, optlen
))
332 if (!optval
|| (len
< sizeof(*crypto_info
))) {
342 /* get user crypto info */
343 crypto_info
= &ctx
->crypto_send
.info
;
345 if (!TLS_CRYPTO_INFO_READY(crypto_info
)) {
350 if (len
== sizeof(*crypto_info
)) {
351 if (copy_to_user(optval
, crypto_info
, sizeof(*crypto_info
)))
356 switch (crypto_info
->cipher_type
) {
357 case TLS_CIPHER_AES_GCM_128
: {
358 struct tls12_crypto_info_aes_gcm_128
*
359 crypto_info_aes_gcm_128
=
360 container_of(crypto_info
,
361 struct tls12_crypto_info_aes_gcm_128
,
364 if (len
!= sizeof(*crypto_info_aes_gcm_128
)) {
369 memcpy(crypto_info_aes_gcm_128
->iv
,
370 ctx
->tx
.iv
+ TLS_CIPHER_AES_GCM_128_SALT_SIZE
,
371 TLS_CIPHER_AES_GCM_128_IV_SIZE
);
372 memcpy(crypto_info_aes_gcm_128
->rec_seq
, ctx
->tx
.rec_seq
,
373 TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE
);
375 if (copy_to_user(optval
,
376 crypto_info_aes_gcm_128
,
377 sizeof(*crypto_info_aes_gcm_128
)))
389 static int do_tls_getsockopt(struct sock
*sk
, int optname
,
390 char __user
*optval
, int __user
*optlen
)
396 rc
= do_tls_getsockopt_tx(sk
, optval
, optlen
);
405 static int tls_getsockopt(struct sock
*sk
, int level
, int optname
,
406 char __user
*optval
, int __user
*optlen
)
408 struct tls_context
*ctx
= tls_get_ctx(sk
);
410 if (level
!= SOL_TLS
)
411 return ctx
->getsockopt(sk
, level
, optname
, optval
, optlen
);
413 return do_tls_getsockopt(sk
, optname
, optval
, optlen
);
416 static int do_tls_setsockopt_conf(struct sock
*sk
, char __user
*optval
,
417 unsigned int optlen
, int tx
)
419 struct tls_crypto_info
*crypto_info
;
420 struct tls_context
*ctx
= tls_get_ctx(sk
);
424 if (!optval
|| (optlen
< sizeof(*crypto_info
))) {
430 crypto_info
= &ctx
->crypto_send
.info
;
432 crypto_info
= &ctx
->crypto_recv
.info
;
434 /* Currently we don't support set crypto info more than one time */
435 if (TLS_CRYPTO_INFO_READY(crypto_info
)) {
440 rc
= copy_from_user(crypto_info
, optval
, sizeof(*crypto_info
));
443 goto err_crypto_info
;
447 if (crypto_info
->version
!= TLS_1_2_VERSION
) {
449 goto err_crypto_info
;
452 switch (crypto_info
->cipher_type
) {
453 case TLS_CIPHER_AES_GCM_128
: {
454 if (optlen
!= sizeof(struct tls12_crypto_info_aes_gcm_128
)) {
456 goto err_crypto_info
;
458 rc
= copy_from_user(crypto_info
+ 1, optval
+ sizeof(*crypto_info
),
459 optlen
- sizeof(*crypto_info
));
462 goto err_crypto_info
;
468 goto err_crypto_info
;
472 #ifdef CONFIG_TLS_DEVICE
473 rc
= tls_set_device_offload(sk
, ctx
);
479 rc
= tls_set_sw_offload(sk
, ctx
, 1);
483 #ifdef CONFIG_TLS_DEVICE
484 rc
= tls_set_device_offload_rx(sk
, ctx
);
490 rc
= tls_set_sw_offload(sk
, ctx
, 0);
496 goto err_crypto_info
;
502 update_sk_prot(sk
, ctx
);
504 ctx
->sk_write_space
= sk
->sk_write_space
;
505 sk
->sk_write_space
= tls_write_space
;
507 sk
->sk_socket
->ops
= &tls_sw_proto_ops
;
512 memzero_explicit(crypto_info
, sizeof(union tls_crypto_context
));
517 static int do_tls_setsockopt(struct sock
*sk
, int optname
,
518 char __user
*optval
, unsigned int optlen
)
526 rc
= do_tls_setsockopt_conf(sk
, optval
, optlen
,
537 static int tls_setsockopt(struct sock
*sk
, int level
, int optname
,
538 char __user
*optval
, unsigned int optlen
)
540 struct tls_context
*ctx
= tls_get_ctx(sk
);
542 if (level
!= SOL_TLS
)
543 return ctx
->setsockopt(sk
, level
, optname
, optval
, optlen
);
545 return do_tls_setsockopt(sk
, optname
, optval
, optlen
);
548 static struct tls_context
*create_ctx(struct sock
*sk
)
550 struct inet_connection_sock
*icsk
= inet_csk(sk
);
551 struct tls_context
*ctx
;
553 ctx
= kzalloc(sizeof(*ctx
), GFP_KERNEL
);
557 icsk
->icsk_ulp_data
= ctx
;
561 static int tls_hw_prot(struct sock
*sk
)
563 struct tls_context
*ctx
;
564 struct tls_device
*dev
;
567 mutex_lock(&device_mutex
);
568 list_for_each_entry(dev
, &device_list
, dev_list
) {
569 if (dev
->feature
&& dev
->feature(dev
)) {
570 ctx
= create_ctx(sk
);
574 ctx
->hash
= sk
->sk_prot
->hash
;
575 ctx
->unhash
= sk
->sk_prot
->unhash
;
576 ctx
->sk_proto_close
= sk
->sk_prot
->close
;
577 ctx
->rx_conf
= TLS_HW_RECORD
;
578 ctx
->tx_conf
= TLS_HW_RECORD
;
579 update_sk_prot(sk
, ctx
);
585 mutex_unlock(&device_mutex
);
589 static void tls_hw_unhash(struct sock
*sk
)
591 struct tls_context
*ctx
= tls_get_ctx(sk
);
592 struct tls_device
*dev
;
594 mutex_lock(&device_mutex
);
595 list_for_each_entry(dev
, &device_list
, dev_list
) {
597 dev
->unhash(dev
, sk
);
599 mutex_unlock(&device_mutex
);
603 static int tls_hw_hash(struct sock
*sk
)
605 struct tls_context
*ctx
= tls_get_ctx(sk
);
606 struct tls_device
*dev
;
610 mutex_lock(&device_mutex
);
611 list_for_each_entry(dev
, &device_list
, dev_list
) {
613 err
|= dev
->hash(dev
, sk
);
615 mutex_unlock(&device_mutex
);
622 static void build_protos(struct proto prot
[TLS_NUM_CONFIG
][TLS_NUM_CONFIG
],
625 prot
[TLS_BASE
][TLS_BASE
] = *base
;
626 prot
[TLS_BASE
][TLS_BASE
].setsockopt
= tls_setsockopt
;
627 prot
[TLS_BASE
][TLS_BASE
].getsockopt
= tls_getsockopt
;
628 prot
[TLS_BASE
][TLS_BASE
].close
= tls_sk_proto_close
;
630 prot
[TLS_SW
][TLS_BASE
] = prot
[TLS_BASE
][TLS_BASE
];
631 prot
[TLS_SW
][TLS_BASE
].sendmsg
= tls_sw_sendmsg
;
632 prot
[TLS_SW
][TLS_BASE
].sendpage
= tls_sw_sendpage
;
634 prot
[TLS_BASE
][TLS_SW
] = prot
[TLS_BASE
][TLS_BASE
];
635 prot
[TLS_BASE
][TLS_SW
].recvmsg
= tls_sw_recvmsg
;
636 prot
[TLS_BASE
][TLS_SW
].close
= tls_sk_proto_close
;
638 prot
[TLS_SW
][TLS_SW
] = prot
[TLS_SW
][TLS_BASE
];
639 prot
[TLS_SW
][TLS_SW
].recvmsg
= tls_sw_recvmsg
;
640 prot
[TLS_SW
][TLS_SW
].close
= tls_sk_proto_close
;
642 #ifdef CONFIG_TLS_DEVICE
643 prot
[TLS_HW
][TLS_BASE
] = prot
[TLS_BASE
][TLS_BASE
];
644 prot
[TLS_HW
][TLS_BASE
].sendmsg
= tls_device_sendmsg
;
645 prot
[TLS_HW
][TLS_BASE
].sendpage
= tls_device_sendpage
;
647 prot
[TLS_HW
][TLS_SW
] = prot
[TLS_BASE
][TLS_SW
];
648 prot
[TLS_HW
][TLS_SW
].sendmsg
= tls_device_sendmsg
;
649 prot
[TLS_HW
][TLS_SW
].sendpage
= tls_device_sendpage
;
651 prot
[TLS_BASE
][TLS_HW
] = prot
[TLS_BASE
][TLS_SW
];
653 prot
[TLS_SW
][TLS_HW
] = prot
[TLS_SW
][TLS_SW
];
655 prot
[TLS_HW
][TLS_HW
] = prot
[TLS_HW
][TLS_SW
];
658 prot
[TLS_HW_RECORD
][TLS_HW_RECORD
] = *base
;
659 prot
[TLS_HW_RECORD
][TLS_HW_RECORD
].hash
= tls_hw_hash
;
660 prot
[TLS_HW_RECORD
][TLS_HW_RECORD
].unhash
= tls_hw_unhash
;
661 prot
[TLS_HW_RECORD
][TLS_HW_RECORD
].close
= tls_sk_proto_close
;
664 static int tls_init(struct sock
*sk
)
666 int ip_ver
= sk
->sk_family
== AF_INET6
? TLSV6
: TLSV4
;
667 struct tls_context
*ctx
;
673 /* The TLS ulp is currently supported only for TCP sockets
674 * in ESTABLISHED state.
675 * Supporting sockets in LISTEN state will require us
676 * to modify the accept implementation to clone rather then
677 * share the ulp context.
679 if (sk
->sk_state
!= TCP_ESTABLISHED
)
682 /* allocate tls context */
683 ctx
= create_ctx(sk
);
688 ctx
->setsockopt
= sk
->sk_prot
->setsockopt
;
689 ctx
->getsockopt
= sk
->sk_prot
->getsockopt
;
690 ctx
->sk_proto_close
= sk
->sk_prot
->close
;
692 /* Build IPv6 TLS whenever the address of tcpv6 _prot changes */
693 if (ip_ver
== TLSV6
&&
694 unlikely(sk
->sk_prot
!= smp_load_acquire(&saved_tcpv6_prot
))) {
695 mutex_lock(&tcpv6_prot_mutex
);
696 if (likely(sk
->sk_prot
!= saved_tcpv6_prot
)) {
697 build_protos(tls_prots
[TLSV6
], sk
->sk_prot
);
698 smp_store_release(&saved_tcpv6_prot
, sk
->sk_prot
);
700 mutex_unlock(&tcpv6_prot_mutex
);
703 ctx
->tx_conf
= TLS_BASE
;
704 ctx
->rx_conf
= TLS_BASE
;
705 update_sk_prot(sk
, ctx
);
710 void tls_register_device(struct tls_device
*device
)
712 mutex_lock(&device_mutex
);
713 list_add_tail(&device
->dev_list
, &device_list
);
714 mutex_unlock(&device_mutex
);
716 EXPORT_SYMBOL(tls_register_device
);
718 void tls_unregister_device(struct tls_device
*device
)
720 mutex_lock(&device_mutex
);
721 list_del(&device
->dev_list
);
722 mutex_unlock(&device_mutex
);
724 EXPORT_SYMBOL(tls_unregister_device
);
726 static struct tcp_ulp_ops tcp_tls_ulp_ops __read_mostly
= {
729 .user_visible
= true,
730 .owner
= THIS_MODULE
,
734 static int __init
tls_register(void)
736 build_protos(tls_prots
[TLSV4
], &tcp_prot
);
738 tls_sw_proto_ops
= inet_stream_ops
;
739 tls_sw_proto_ops
.poll
= tls_sw_poll
;
740 tls_sw_proto_ops
.splice_read
= tls_sw_splice_read
;
742 #ifdef CONFIG_TLS_DEVICE
745 tcp_register_ulp(&tcp_tls_ulp_ops
);
750 static void __exit
tls_unregister(void)
752 tcp_unregister_ulp(&tcp_tls_ulp_ops
);
753 #ifdef CONFIG_TLS_DEVICE
754 tls_device_cleanup();
758 module_init(tls_register
);
759 module_exit(tls_unregister
);