1 /* SPDX-License-Identifier: GPL-2.0 */
6 #include <linux/scatterlist.h>
7 #include <linux/highmem.h>
8 #include <rdma/rdma_cm.h>
9 #include <linux/mutex.h>
10 #include <linux/rds.h>
11 #include <linux/rhashtable.h>
12 #include <linux/refcount.h>
17 * RDS Network protocol version
19 #define RDS_PROTOCOL_3_0 0x0300
20 #define RDS_PROTOCOL_3_1 0x0301
21 #define RDS_PROTOCOL_VERSION RDS_PROTOCOL_3_1
22 #define RDS_PROTOCOL_MAJOR(v) ((v) >> 8)
23 #define RDS_PROTOCOL_MINOR(v) ((v) & 255)
24 #define RDS_PROTOCOL(maj, min) (((maj) << 8) | min)
27 * XXX randomly chosen, but at least seems to be unused:
28 * # 18464-18768 Unassigned
29 * We should do better. We want a reserved port to discourage unpriv'ed
30 * userspace from listening.
32 #define RDS_PORT 18634
35 #define KERNEL_HAS_ATOMIC64
39 #define rdsdebug(fmt, args...) pr_debug("%s(): " fmt, __func__ , ##args)
41 /* sigh, pr_debug() causes unused variable warnings */
42 static inline __printf(1, 2)
43 void rdsdebug(char *fmt, ...)
48 /* XXX is there one of these somewhere? */
50 ({ unsigned long __x = (x), __y = (y); (__x + __y - 1) / __y; })
52 #define RDS_FRAG_SHIFT 12
53 #define RDS_FRAG_SIZE ((unsigned int)(1 << RDS_FRAG_SHIFT))
55 /* Used to limit both RDMA and non-RDMA RDS message to 1MB */
56 #define RDS_MAX_MSG_SIZE ((unsigned int)(1 << 20))
58 #define RDS_CONG_MAP_BYTES (65536 / 8)
59 #define RDS_CONG_MAP_PAGES (PAGE_ALIGN(RDS_CONG_MAP_BYTES) / PAGE_SIZE)
60 #define RDS_CONG_MAP_PAGE_BITS (PAGE_SIZE * 8)
63 struct rb_node m_rb_node;
65 wait_queue_head_t m_waitq;
66 struct list_head m_conn_list;
67 unsigned long m_page_addrs[RDS_CONG_MAP_PAGES];
72 * This is how we will track the connection state:
73 * A connection is always in one of the following
74 * states. Updates to the state are atomic and imply
80 RDS_CONN_DISCONNECTING,
86 /* Bits for c_flags */
87 #define RDS_LL_SEND_FULL 0
88 #define RDS_RECONNECT_PENDING 1
90 #define RDS_RECV_REFILL 3
92 /* Max number of multipaths per RDS connection. Must be a power of 2 */
93 #define RDS_MPATH_WORKERS 8
94 #define RDS_MPATH_HASH(rs, n) (jhash_1word((rs)->rs_bound_port, \
95 (rs)->rs_hash_initval) & ((n) - 1))
97 #define IS_CANONICAL(laddr, faddr) (htonl(laddr) < htonl(faddr))
99 /* Per mpath connection state */
100 struct rds_conn_path {
101 struct rds_connection *cp_conn;
102 struct rds_message *cp_xmit_rm;
103 unsigned long cp_xmit_sg;
104 unsigned int cp_xmit_hdr_off;
105 unsigned int cp_xmit_data_off;
106 unsigned int cp_xmit_atomic_sent;
107 unsigned int cp_xmit_rdma_sent;
108 unsigned int cp_xmit_data_sent;
110 spinlock_t cp_lock; /* protect msg queues */
112 struct list_head cp_send_queue;
113 struct list_head cp_retrans;
117 void *cp_transport_data;
120 unsigned long cp_send_gen;
121 unsigned long cp_flags;
122 unsigned long cp_reconnect_jiffies;
123 struct delayed_work cp_send_w;
124 struct delayed_work cp_recv_w;
125 struct delayed_work cp_conn_w;
126 struct work_struct cp_down_w;
127 struct mutex cp_cm_lock; /* protect cp_state & cm */
128 wait_queue_head_t cp_waitq;
130 unsigned int cp_unacked_packets;
131 unsigned int cp_unacked_bytes;
132 unsigned int cp_index;
135 /* One rds_connection per RDS address pair */
136 struct rds_connection {
137 struct hlist_node c_hash_node;
140 unsigned int c_loopback:1,
145 struct rds_connection *c_passive;
146 struct rds_transport *c_trans;
148 struct rds_cong_map *c_lcong;
149 struct rds_cong_map *c_fcong;
151 /* Protocol version */
152 unsigned int c_version;
155 struct list_head c_map_item;
156 unsigned long c_map_queued;
158 struct rds_conn_path *c_path;
159 wait_queue_head_t c_hs_waitq; /* handshake waitq */
166 struct net *rds_conn_net(struct rds_connection *conn)
172 void rds_conn_net_set(struct rds_connection *conn, struct net *net)
174 conn->c_net = get_net(net);
177 #define RDS_FLAG_CONG_BITMAP 0x01
178 #define RDS_FLAG_ACK_REQUIRED 0x02
179 #define RDS_FLAG_RETRANSMITTED 0x04
180 #define RDS_MAX_ADV_CREDIT 255
182 /* RDS_FLAG_PROBE_PORT is the reserved sport used for sending a ping
183 * probe to exchange control information before establishing a connection.
184 * Currently the control information that is exchanged is the number of
185 * supported paths. If the peer is a legacy (older kernel revision) peer,
186 * it would return a pong message without additional control information
187 * that would then alert the sender that the peer was an older rev.
189 #define RDS_FLAG_PROBE_PORT 1
190 #define RDS_HS_PROBE(sport, dport) \
191 ((sport == RDS_FLAG_PROBE_PORT && dport == 0) || \
192 (sport == 0 && dport == RDS_FLAG_PROBE_PORT))
194 * Maximum space available for extension headers.
196 #define RDS_HEADER_EXT_SPACE 16
209 u8 h_exthdr[RDS_HEADER_EXT_SPACE];
213 * Reserved - indicates end of extensions
215 #define RDS_EXTHDR_NONE 0
218 * This extension header is included in the very
219 * first message that is sent on a new connection,
220 * and identifies the protocol level. This will help
221 * rolling updates if a future change requires breaking
223 * NB: This is no longer true for IB, where we do a version
224 * negotiation during the connection setup phase (protocol
225 * version information is included in the RDMA CM private data).
227 #define RDS_EXTHDR_VERSION 1
228 struct rds_ext_header_version {
233 * This extension header is included in the RDS message
234 * chasing an RDMA operation.
236 #define RDS_EXTHDR_RDMA 2
237 struct rds_ext_header_rdma {
242 * This extension header tells the peer about the
243 * destination <R_Key,offset> of the requested RDMA
246 #define RDS_EXTHDR_RDMA_DEST 3
247 struct rds_ext_header_rdma_dest {
249 __be32 h_rdma_offset;
252 /* Extension header announcing number of paths.
253 * Implicit length = 2 bytes.
255 #define RDS_EXTHDR_NPATHS 5
256 #define RDS_EXTHDR_GEN_NUM 6
258 #define __RDS_EXTHDR_MAX 16 /* for now */
259 #define RDS_RX_MAX_TRACES (RDS_MSG_RX_DGRAM_TRACE_MAX + 1)
260 #define RDS_MSG_RX_HDR 0
261 #define RDS_MSG_RX_START 1
262 #define RDS_MSG_RX_END 2
263 #define RDS_MSG_RX_CMSG 3
265 struct rds_incoming {
266 refcount_t i_refcount;
267 struct list_head i_item;
268 struct rds_connection *i_conn;
269 struct rds_conn_path *i_conn_path;
270 struct rds_header i_hdr;
271 unsigned long i_rx_jiffies;
274 rds_rdma_cookie_t i_rdma_cookie;
275 struct timeval i_rx_tstamp;
276 u64 i_rx_lat_trace[RDS_RX_MAX_TRACES];
280 struct rb_node r_rb_node;
281 refcount_t r_refcount;
284 /* A copy of the creation flags */
285 unsigned int r_use_once:1;
286 unsigned int r_invalidate:1;
287 unsigned int r_write:1;
289 /* This is for RDS_MR_DEAD.
290 * It would be nice & consistent to make this part of the above
291 * bit field here, but we need to use test_and_set_bit.
293 unsigned long r_state;
294 struct rds_sock *r_sock; /* back pointer to the socket that owns us */
295 struct rds_transport *r_trans;
296 void *r_trans_private;
299 /* Flags for mr->r_state */
300 #define RDS_MR_DEAD 0
302 static inline rds_rdma_cookie_t rds_rdma_make_cookie(u32 r_key, u32 offset)
304 return r_key | (((u64) offset) << 32);
307 static inline u32 rds_rdma_cookie_key(rds_rdma_cookie_t cookie)
312 static inline u32 rds_rdma_cookie_offset(rds_rdma_cookie_t cookie)
317 /* atomic operation types */
318 #define RDS_ATOMIC_TYPE_CSWP 0
319 #define RDS_ATOMIC_TYPE_FADD 1
322 * m_sock_item and m_conn_item are on lists that are serialized under
323 * conn->c_lock. m_sock_item has additional meaning in that once it is empty
324 * the message will not be put back on the retransmit list after being sent.
325 * messages that are canceled while being sent rely on this.
327 * m_inc is used by loopback so that it can pass an incoming message straight
328 * back up into the rx path. It embeds a wire header which is also used by
329 * the send path, which is kind of awkward.
331 * m_sock_item indicates the message's presence on a socket's send or receive
332 * queue. m_rs will point to that socket.
334 * m_daddr is used by cancellation to prune messages to a given destination.
336 * The RDS_MSG_ON_SOCK and RDS_MSG_ON_CONN flags are used to avoid lock
337 * nesting. As paths iterate over messages on a sock, or conn, they must
338 * also lock the conn, or sock, to remove the message from those lists too.
339 * Testing the flag to determine if the message is still on the lists lets
340 * us avoid testing the list_head directly. That means each path can use
341 * the message's list_head to keep it on a local list while juggling locks
342 * without confusing the other path.
344 * m_ack_seq is an optional field set by transports who need a different
345 * sequence number range to invalidate. They can use this in a callback
346 * that they pass to rds_send_drop_acked() to see if each message has been
347 * acked. The HAS_ACK_SEQ flag can be used to detect messages which haven't
348 * had ack_seq set yet.
350 #define RDS_MSG_ON_SOCK 1
351 #define RDS_MSG_ON_CONN 2
352 #define RDS_MSG_HAS_ACK_SEQ 3
353 #define RDS_MSG_ACK_REQUIRED 4
354 #define RDS_MSG_RETRANSMITTED 5
355 #define RDS_MSG_MAPPED 6
356 #define RDS_MSG_PAGEVEC 7
357 #define RDS_MSG_FLUSH 8
360 refcount_t m_refcount;
361 struct list_head m_sock_item;
362 struct list_head m_conn_item;
363 struct rds_incoming m_inc;
366 unsigned long m_flags;
368 /* Never access m_rs without holding m_rs_lock.
373 spinlock_t m_rs_lock;
374 wait_queue_head_t m_flush_wait;
376 struct rds_sock *m_rs;
378 /* cookie to send to remote, in rds header */
379 rds_rdma_cookie_t m_rdma_cookie;
381 unsigned int m_used_sgs;
382 unsigned int m_total_sgs;
387 struct rm_atomic_op {
393 uint64_t compare_mask;
398 uint64_t nocarry_mask;
404 unsigned int op_notify:1;
405 unsigned int op_recverr:1;
406 unsigned int op_mapped:1;
407 unsigned int op_silent:1;
408 unsigned int op_active:1;
409 struct scatterlist *op_sg;
410 struct rds_notifier *op_notifier;
412 struct rds_mr *op_rdma_mr;
417 unsigned int op_write:1;
418 unsigned int op_fence:1;
419 unsigned int op_notify:1;
420 unsigned int op_recverr:1;
421 unsigned int op_mapped:1;
422 unsigned int op_silent:1;
423 unsigned int op_active:1;
424 unsigned int op_bytes;
425 unsigned int op_nents;
426 unsigned int op_count;
427 struct scatterlist *op_sg;
428 struct rds_notifier *op_notifier;
430 struct rds_mr *op_rdma_mr;
433 unsigned int op_active:1;
434 unsigned int op_notify:1;
435 unsigned int op_nents;
436 unsigned int op_count;
437 unsigned int op_dmasg;
438 unsigned int op_dmaoff;
439 struct scatterlist *op_sg;
445 * The RDS notifier is used (optionally) to tell the application about
446 * completed RDMA operations. Rather than keeping the whole rds message
447 * around on the queue, we allocate a small notifier that is put on the
448 * socket's notifier_list. Notifications are delivered to the application
449 * through control messages.
451 struct rds_notifier {
452 struct list_head n_list;
453 uint64_t n_user_token;
458 * struct rds_transport - transport specific behavioural hooks
460 * @xmit: .xmit is called by rds_send_xmit() to tell the transport to send
461 * part of a message. The caller serializes on the send_sem so this
462 * doesn't need to be reentrant for a given conn. The header must be
463 * sent before the data payload. .xmit must be prepared to send a
464 * message with no data payload. .xmit should return the number of
465 * bytes that were sent down the connection, including header bytes.
466 * Returning 0 tells the caller that it doesn't need to perform any
467 * additional work now. This is usually the case when the transport has
468 * filled the sending queue for its connection and will handle
469 * triggering the rds thread to continue the send when space becomes
470 * available. Returning -EAGAIN tells the caller to retry the send
471 * immediately. Returning -ENOMEM tells the caller to retry the send at
472 * some point in the future.
474 * @conn_shutdown: conn_shutdown stops traffic on the given connection. Once
475 * it returns the connection can not call rds_recv_incoming().
476 * This will only be called once after conn_connect returns
477 * non-zero success and will The caller serializes this with
478 * the send and connecting paths (xmit_* and conn_*). The
479 * transport is responsible for other serialization, including
480 * rds_recv_incoming(). This is called in process context but
481 * should try hard not to block.
484 struct rds_transport {
485 char t_name[TRANSNAMSIZ];
486 struct list_head t_item;
487 struct module *t_owner;
488 unsigned int t_prefer_loopback:1,
492 int (*laddr_check)(struct net *net, __be32 addr);
493 int (*conn_alloc)(struct rds_connection *conn, gfp_t gfp);
494 void (*conn_free)(void *data);
495 int (*conn_path_connect)(struct rds_conn_path *cp);
496 void (*conn_path_shutdown)(struct rds_conn_path *conn);
497 void (*xmit_path_prepare)(struct rds_conn_path *cp);
498 void (*xmit_path_complete)(struct rds_conn_path *cp);
499 int (*xmit)(struct rds_connection *conn, struct rds_message *rm,
500 unsigned int hdr_off, unsigned int sg, unsigned int off);
501 int (*xmit_rdma)(struct rds_connection *conn, struct rm_rdma_op *op);
502 int (*xmit_atomic)(struct rds_connection *conn, struct rm_atomic_op *op);
503 int (*recv_path)(struct rds_conn_path *cp);
504 int (*inc_copy_to_user)(struct rds_incoming *inc, struct iov_iter *to);
505 void (*inc_free)(struct rds_incoming *inc);
507 int (*cm_handle_connect)(struct rdma_cm_id *cm_id,
508 struct rdma_cm_event *event);
509 int (*cm_initiate_connect)(struct rdma_cm_id *cm_id);
510 void (*cm_connect_complete)(struct rds_connection *conn,
511 struct rdma_cm_event *event);
513 unsigned int (*stats_info_copy)(struct rds_info_iterator *iter,
516 void *(*get_mr)(struct scatterlist *sg, unsigned long nr_sg,
517 struct rds_sock *rs, u32 *key_ret);
518 void (*sync_mr)(void *trans_private, int direction);
519 void (*free_mr)(void *trans_private, int invalidate);
520 void (*flush_mrs)(void);
530 * bound_addr used for both incoming and outgoing, no INADDR_ANY
533 struct rhash_head rs_bound_node;
535 __be32 rs_bound_addr;
537 __be16 rs_bound_port;
539 struct rds_transport *rs_transport;
542 * rds_sendmsg caches the conn it used the last time around.
543 * This helps avoid costly lookups.
545 struct rds_connection *rs_conn;
547 /* flag indicating we were congested or not */
549 /* seen congestion (ENOBUFS) when sending? */
550 int rs_seen_congestion;
552 /* rs_lock protects all these adjacent members before the newline */
554 struct list_head rs_send_queue;
557 struct list_head rs_notify_queue; /* currently used for failed RDMAs */
559 /* Congestion wake_up. If rs_cong_monitor is set, we use cong_mask
560 * to decide whether the application should be woken up.
561 * If not set, we use rs_cong_track to find out whether a cong map
564 uint64_t rs_cong_mask;
565 uint64_t rs_cong_notify;
566 struct list_head rs_cong_list;
567 unsigned long rs_cong_track;
570 * rs_recv_lock protects the receive queue, and is
571 * used to serialize with rds_release.
573 rwlock_t rs_recv_lock;
574 struct list_head rs_recv_queue;
576 /* just for stats reporting */
577 struct list_head rs_item;
579 /* these have their own lock */
580 spinlock_t rs_rdma_lock;
581 struct rb_root rs_rdma_keys;
583 /* Socket options - in case there will be more */
584 unsigned char rs_recverr,
588 /* Socket receive path trace points*/
590 u8 rs_rx_trace[RDS_MSG_RX_DGRAM_TRACE_MAX];
593 static inline struct rds_sock *rds_sk_to_rs(const struct sock *sk)
595 return container_of(sk, struct rds_sock, rs_sk);
597 static inline struct sock *rds_rs_to_sk(struct rds_sock *rs)
603 * The stack assigns sk_sndbuf and sk_rcvbuf to twice the specified value
604 * to account for overhead. We don't account for overhead, we just apply
605 * the number of payload bytes to the specified value.
607 static inline int rds_sk_sndbuf(struct rds_sock *rs)
609 return rds_rs_to_sk(rs)->sk_sndbuf / 2;
611 static inline int rds_sk_rcvbuf(struct rds_sock *rs)
613 return rds_rs_to_sk(rs)->sk_rcvbuf / 2;
616 struct rds_statistics {
617 uint64_t s_conn_reset;
618 uint64_t s_recv_drop_bad_checksum;
619 uint64_t s_recv_drop_old_seq;
620 uint64_t s_recv_drop_no_sock;
621 uint64_t s_recv_drop_dead_sock;
622 uint64_t s_recv_deliver_raced;
623 uint64_t s_recv_delivered;
624 uint64_t s_recv_queued;
625 uint64_t s_recv_immediate_retry;
626 uint64_t s_recv_delayed_retry;
627 uint64_t s_recv_ack_required;
628 uint64_t s_recv_rdma_bytes;
629 uint64_t s_recv_ping;
630 uint64_t s_send_queue_empty;
631 uint64_t s_send_queue_full;
632 uint64_t s_send_lock_contention;
633 uint64_t s_send_lock_queue_raced;
634 uint64_t s_send_immediate_retry;
635 uint64_t s_send_delayed_retry;
636 uint64_t s_send_drop_acked;
637 uint64_t s_send_ack_required;
638 uint64_t s_send_queued;
639 uint64_t s_send_rdma;
640 uint64_t s_send_rdma_bytes;
641 uint64_t s_send_pong;
642 uint64_t s_page_remainder_hit;
643 uint64_t s_page_remainder_miss;
644 uint64_t s_copy_to_user;
645 uint64_t s_copy_from_user;
646 uint64_t s_cong_update_queued;
647 uint64_t s_cong_update_received;
648 uint64_t s_cong_send_error;
649 uint64_t s_cong_send_blocked;
650 uint64_t s_recv_bytes_added_to_socket;
651 uint64_t s_recv_bytes_removed_from_socket;
656 void rds_sock_addref(struct rds_sock *rs);
657 void rds_sock_put(struct rds_sock *rs);
658 void rds_wake_sk_sleep(struct rds_sock *rs);
659 static inline void __rds_wake_sk_sleep(struct sock *sk)
661 wait_queue_head_t *waitq = sk_sleep(sk);
663 if (!sock_flag(sk, SOCK_DEAD) && waitq)
666 extern wait_queue_head_t rds_poll_waitq;
670 int rds_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len);
671 void rds_remove_bound(struct rds_sock *rs);
672 struct rds_sock *rds_find_bound(__be32 addr, __be16 port);
673 int rds_bind_lock_init(void);
674 void rds_bind_lock_destroy(void);
677 int rds_cong_get_maps(struct rds_connection *conn);
678 void rds_cong_add_conn(struct rds_connection *conn);
679 void rds_cong_remove_conn(struct rds_connection *conn);
680 void rds_cong_set_bit(struct rds_cong_map *map, __be16 port);
681 void rds_cong_clear_bit(struct rds_cong_map *map, __be16 port);
682 int rds_cong_wait(struct rds_cong_map *map, __be16 port, int nonblock, struct rds_sock *rs);
683 void rds_cong_queue_updates(struct rds_cong_map *map);
684 void rds_cong_map_updated(struct rds_cong_map *map, uint64_t);
685 int rds_cong_updated_since(unsigned long *recent);
686 void rds_cong_add_socket(struct rds_sock *);
687 void rds_cong_remove_socket(struct rds_sock *);
688 void rds_cong_exit(void);
689 struct rds_message *rds_cong_update_alloc(struct rds_connection *conn);
692 extern u32 rds_gen_num;
693 int rds_conn_init(void);
694 void rds_conn_exit(void);
695 struct rds_connection *rds_conn_create(struct net *net,
696 __be32 laddr, __be32 faddr,
697 struct rds_transport *trans, gfp_t gfp);
698 struct rds_connection *rds_conn_create_outgoing(struct net *net,
699 __be32 laddr, __be32 faddr,
700 struct rds_transport *trans, gfp_t gfp);
701 void rds_conn_shutdown(struct rds_conn_path *cpath);
702 void rds_conn_destroy(struct rds_connection *conn);
703 void rds_conn_drop(struct rds_connection *conn);
704 void rds_conn_path_drop(struct rds_conn_path *cpath, bool destroy);
705 void rds_conn_connect_if_down(struct rds_connection *conn);
706 void rds_conn_path_connect_if_down(struct rds_conn_path *cp);
707 void rds_for_each_conn_info(struct socket *sock, unsigned int len,
708 struct rds_info_iterator *iter,
709 struct rds_info_lengths *lens,
710 int (*visitor)(struct rds_connection *, void *),
714 void __rds_conn_path_error(struct rds_conn_path *cp, const char *, ...);
715 #define rds_conn_path_error(cp, fmt...) \
716 __rds_conn_path_error(cp, KERN_WARNING "RDS: " fmt)
719 rds_conn_path_transition(struct rds_conn_path *cp, int old, int new)
721 return atomic_cmpxchg(&cp->cp_state, old, new) == old;
725 rds_conn_transition(struct rds_connection *conn, int old, int new)
727 WARN_ON(conn->c_trans->t_mp_capable);
728 return rds_conn_path_transition(&conn->c_path[0], old, new);
732 rds_conn_path_state(struct rds_conn_path *cp)
734 return atomic_read(&cp->cp_state);
738 rds_conn_state(struct rds_connection *conn)
740 WARN_ON(conn->c_trans->t_mp_capable);
741 return rds_conn_path_state(&conn->c_path[0]);
745 rds_conn_path_up(struct rds_conn_path *cp)
747 return atomic_read(&cp->cp_state) == RDS_CONN_UP;
751 rds_conn_up(struct rds_connection *conn)
753 WARN_ON(conn->c_trans->t_mp_capable);
754 return rds_conn_path_up(&conn->c_path[0]);
758 rds_conn_path_connecting(struct rds_conn_path *cp)
760 return atomic_read(&cp->cp_state) == RDS_CONN_CONNECTING;
764 rds_conn_connecting(struct rds_connection *conn)
766 WARN_ON(conn->c_trans->t_mp_capable);
767 return rds_conn_path_connecting(&conn->c_path[0]);
771 struct rds_message *rds_message_alloc(unsigned int nents, gfp_t gfp);
772 struct scatterlist *rds_message_alloc_sgs(struct rds_message *rm, int nents);
773 int rds_message_copy_from_user(struct rds_message *rm, struct iov_iter *from);
774 struct rds_message *rds_message_map_pages(unsigned long *page_addrs, unsigned int total_len);
775 void rds_message_populate_header(struct rds_header *hdr, __be16 sport,
776 __be16 dport, u64 seq);
777 int rds_message_add_extension(struct rds_header *hdr,
778 unsigned int type, const void *data, unsigned int len);
779 int rds_message_next_extension(struct rds_header *hdr,
780 unsigned int *pos, void *buf, unsigned int *buflen);
781 int rds_message_add_rdma_dest_extension(struct rds_header *hdr, u32 r_key, u32 offset);
782 int rds_message_inc_copy_to_user(struct rds_incoming *inc, struct iov_iter *to);
783 void rds_message_inc_free(struct rds_incoming *inc);
784 void rds_message_addref(struct rds_message *rm);
785 void rds_message_put(struct rds_message *rm);
786 void rds_message_wait(struct rds_message *rm);
787 void rds_message_unmapped(struct rds_message *rm);
789 static inline void rds_message_make_checksum(struct rds_header *hdr)
792 hdr->h_csum = ip_fast_csum((void *) hdr, sizeof(*hdr) >> 2);
795 static inline int rds_message_verify_checksum(const struct rds_header *hdr)
797 return !hdr->h_csum || ip_fast_csum((void *) hdr, sizeof(*hdr) >> 2) == 0;
802 int rds_page_remainder_alloc(struct scatterlist *scat, unsigned long bytes,
804 void rds_page_exit(void);
807 void rds_inc_init(struct rds_incoming *inc, struct rds_connection *conn,
809 void rds_inc_path_init(struct rds_incoming *inc, struct rds_conn_path *conn,
811 void rds_inc_put(struct rds_incoming *inc);
812 void rds_recv_incoming(struct rds_connection *conn, __be32 saddr, __be32 daddr,
813 struct rds_incoming *inc, gfp_t gfp);
814 int rds_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
816 void rds_clear_recv_queue(struct rds_sock *rs);
817 int rds_notify_queue_get(struct rds_sock *rs, struct msghdr *msg);
818 void rds_inc_info_copy(struct rds_incoming *inc,
819 struct rds_info_iterator *iter,
820 __be32 saddr, __be32 daddr, int flip);
823 int rds_sendmsg(struct socket *sock, struct msghdr *msg, size_t payload_len);
824 void rds_send_path_reset(struct rds_conn_path *conn);
825 int rds_send_xmit(struct rds_conn_path *cp);
827 void rds_send_drop_to(struct rds_sock *rs, struct sockaddr_in *dest);
828 typedef int (*is_acked_func)(struct rds_message *rm, uint64_t ack);
829 void rds_send_drop_acked(struct rds_connection *conn, u64 ack,
830 is_acked_func is_acked);
831 void rds_send_path_drop_acked(struct rds_conn_path *cp, u64 ack,
832 is_acked_func is_acked);
833 void rds_send_ping(struct rds_connection *conn, int cp_index);
834 int rds_send_pong(struct rds_conn_path *cp, __be16 dport);
837 void rds_rdma_unuse(struct rds_sock *rs, u32 r_key, int force);
838 int rds_get_mr(struct rds_sock *rs, char __user *optval, int optlen);
839 int rds_get_mr_for_dest(struct rds_sock *rs, char __user *optval, int optlen);
840 int rds_free_mr(struct rds_sock *rs, char __user *optval, int optlen);
841 void rds_rdma_drop_keys(struct rds_sock *rs);
842 int rds_rdma_extra_size(struct rds_rdma_args *args);
843 int rds_cmsg_rdma_args(struct rds_sock *rs, struct rds_message *rm,
844 struct cmsghdr *cmsg);
845 int rds_cmsg_rdma_dest(struct rds_sock *rs, struct rds_message *rm,
846 struct cmsghdr *cmsg);
847 int rds_cmsg_rdma_args(struct rds_sock *rs, struct rds_message *rm,
848 struct cmsghdr *cmsg);
849 int rds_cmsg_rdma_map(struct rds_sock *rs, struct rds_message *rm,
850 struct cmsghdr *cmsg);
851 void rds_rdma_free_op(struct rm_rdma_op *ro);
852 void rds_atomic_free_op(struct rm_atomic_op *ao);
853 void rds_rdma_send_complete(struct rds_message *rm, int wc_status);
854 void rds_atomic_send_complete(struct rds_message *rm, int wc_status);
855 int rds_cmsg_atomic(struct rds_sock *rs, struct rds_message *rm,
856 struct cmsghdr *cmsg);
858 void __rds_put_mr_final(struct rds_mr *mr);
859 static inline void rds_mr_put(struct rds_mr *mr)
861 if (refcount_dec_and_test(&mr->r_refcount))
862 __rds_put_mr_final(mr);
866 DECLARE_PER_CPU_SHARED_ALIGNED(struct rds_statistics, rds_stats);
867 #define rds_stats_inc_which(which, member) do { \
868 per_cpu(which, get_cpu()).member++; \
871 #define rds_stats_inc(member) rds_stats_inc_which(rds_stats, member)
872 #define rds_stats_add_which(which, member, count) do { \
873 per_cpu(which, get_cpu()).member += count; \
876 #define rds_stats_add(member, count) rds_stats_add_which(rds_stats, member, count)
877 int rds_stats_init(void);
878 void rds_stats_exit(void);
879 void rds_stats_info_copy(struct rds_info_iterator *iter,
880 uint64_t *values, const char *const *names,
884 int rds_sysctl_init(void);
885 void rds_sysctl_exit(void);
886 extern unsigned long rds_sysctl_sndbuf_min;
887 extern unsigned long rds_sysctl_sndbuf_default;
888 extern unsigned long rds_sysctl_sndbuf_max;
889 extern unsigned long rds_sysctl_reconnect_min_jiffies;
890 extern unsigned long rds_sysctl_reconnect_max_jiffies;
891 extern unsigned int rds_sysctl_max_unacked_packets;
892 extern unsigned int rds_sysctl_max_unacked_bytes;
893 extern unsigned int rds_sysctl_ping_enable;
894 extern unsigned long rds_sysctl_trace_flags;
895 extern unsigned int rds_sysctl_trace_level;
898 int rds_threads_init(void);
899 void rds_threads_exit(void);
900 extern struct workqueue_struct *rds_wq;
901 void rds_queue_reconnect(struct rds_conn_path *cp);
902 void rds_connect_worker(struct work_struct *);
903 void rds_shutdown_worker(struct work_struct *);
904 void rds_send_worker(struct work_struct *);
905 void rds_recv_worker(struct work_struct *);
906 void rds_connect_path_complete(struct rds_conn_path *conn, int curr);
907 void rds_connect_complete(struct rds_connection *conn);
910 void rds_trans_register(struct rds_transport *trans);
911 void rds_trans_unregister(struct rds_transport *trans);
912 struct rds_transport *rds_trans_get_preferred(struct net *net, __be32 addr);
913 void rds_trans_put(struct rds_transport *trans);
914 unsigned int rds_trans_stats_info_copy(struct rds_info_iterator *iter,
916 struct rds_transport *rds_trans_get(int t_type);
917 int rds_trans_init(void);
918 void rds_trans_exit(void);