net/smc/af_smc.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *  Shared Memory Communications over RDMA (SMC-R) and RoCE
   4  *
   5  *  AF_SMC protocol family socket handler keeping the AF_INET sock address type
   6  *  applies to SOCK_STREAM sockets only
   7  *  offers an alternative communication option for TCP-protocol sockets
   8  *  applicable with RoCE-cards only
   9  *
  10  *  Initial restrictions:
  11  *    - support for alternate links postponed
  12  *
  13  *  Copyright IBM Corp. 2016, 2018
  14  *
  15  *  Author(s):  Ursula Braun <ubraun@linux.vnet.ibm.com>
  16  *              based on prototype from Frank Blaschka
  17  */
  18
  19 #define KMSG_COMPONENT "smc"
  20 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  21
  22 #include <linux/module.h>
  23 #include <linux/socket.h>
  24 #include <linux/workqueue.h>
  25 #include <linux/in.h>
  26 #include <linux/sched/signal.h>
  27 #include <linux/if_vlan.h>
  28 #include <linux/rcupdate_wait.h>
  29
  30 #include <net/sock.h>
  31 #include <net/tcp.h>
  32 #include <net/smc.h>
  33 #include <asm/ioctls.h>
  34
  35 #include <net/net_namespace.h>
  36 #include <net/netns/generic.h>
  37 #include "smc_netns.h"
  38
  39 #include "smc.h"
  40 #include "smc_clc.h"
  41 #include "smc_llc.h"
  42 #include "smc_cdc.h"
  43 #include "smc_core.h"
  44 #include "smc_ib.h"
  45 #include "smc_ism.h"
  46 #include "smc_pnet.h"
  47 #include "smc_tx.h"
  48 #include "smc_rx.h"
  49 #include "smc_close.h"
  50
  51 static DEFINE_MUTEX(smc_server_lgr_pending);    /* serialize link group
  52                                                  * creation on server
  53                                                  */
  54 static DEFINE_MUTEX(smc_client_lgr_pending);    /* serialize link group
  55                                                  * creation on client
  56                                                  */
  57
  58 static void smc_tcp_listen_work(struct work_struct *);
  59 static void smc_connect_work(struct work_struct *);
  60
  61 static void smc_set_keepalive(struct sock *sk, int val)
  62 {
  63         struct smc_sock *smc = smc_sk(sk);
  64
  65         smc->clcsock->sk->sk_prot->keepalive(smc->clcsock->sk, val);
  66 }
  67
  68 static struct smc_hashinfo smc_v4_hashinfo = {
  69         .lock = __RW_LOCK_UNLOCKED(smc_v4_hashinfo.lock),
  70 };
  71
  72 static struct smc_hashinfo smc_v6_hashinfo = {
  73         .lock = __RW_LOCK_UNLOCKED(smc_v6_hashinfo.lock),
  74 };
  75
  76 int smc_hash_sk(struct sock *sk)
  77 {
  78         struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
  79         struct hlist_head *head;
  80
  81         head = &h->ht;
  82
  83         write_lock_bh(&h->lock);
  84         sk_add_node(sk, head);
  85         sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
  86         write_unlock_bh(&h->lock);
  87
  88         return 0;
  89 }
  90 EXPORT_SYMBOL_GPL(smc_hash_sk);
  91
  92 void smc_unhash_sk(struct sock *sk)
  93 {
  94         struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
  95
  96         write_lock_bh(&h->lock);
  97         if (sk_del_node_init(sk))
  98                 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
  99         write_unlock_bh(&h->lock);
 100 }
 101 EXPORT_SYMBOL_GPL(smc_unhash_sk);
 102
 103 struct proto smc_proto = {
 104         .name           = "SMC",
 105         .owner          = THIS_MODULE,
 106         .keepalive      = smc_set_keepalive,
 107         .hash           = smc_hash_sk,
 108         .unhash         = smc_unhash_sk,
 109         .obj_size       = sizeof(struct smc_sock),
 110         .h.smc_hash     = &smc_v4_hashinfo,
 111         .slab_flags     = SLAB_TYPESAFE_BY_RCU,
 112 };
 113 EXPORT_SYMBOL_GPL(smc_proto);
 114
 115 struct proto smc_proto6 = {
 116         .name           = "SMC6",
 117         .owner          = THIS_MODULE,
 118         .keepalive      = smc_set_keepalive,
 119         .hash           = smc_hash_sk,
 120         .unhash         = smc_unhash_sk,
 121         .obj_size       = sizeof(struct smc_sock),
 122         .h.smc_hash     = &smc_v6_hashinfo,
 123         .slab_flags     = SLAB_TYPESAFE_BY_RCU,
 124 };
 125 EXPORT_SYMBOL_GPL(smc_proto6);
 126
 127 static void smc_restore_fallback_changes(struct smc_sock *smc)
 128 {
 129         smc->clcsock->file->private_data = smc->sk.sk_socket;
 130         smc->clcsock->file = NULL;
 131 }
 132
 133 static int __smc_release(struct smc_sock *smc)
 134 {
 135         struct sock *sk = &smc->sk;
 136         int rc = 0;
 137
 138         if (!smc->use_fallback) {
 139                 rc = smc_close_active(smc);
 140                 sock_set_flag(sk, SOCK_DEAD);
 141                 sk->sk_shutdown |= SHUTDOWN_MASK;
 142         } else {
 143                 if (sk->sk_state != SMC_LISTEN && sk->sk_state != SMC_INIT)
 144                         sock_put(sk); /* passive closing */
 145                 if (sk->sk_state == SMC_LISTEN) {
 146                         /* wake up clcsock accept */
 147                         rc = kernel_sock_shutdown(smc->clcsock, SHUT_RDWR);
 148                 }
 149                 sk->sk_state = SMC_CLOSED;
 150                 sk->sk_state_change(sk);
 151                 smc_restore_fallback_changes(smc);
 152         }
 153
 154         sk->sk_prot->unhash(sk);
 155
 156         if (sk->sk_state == SMC_CLOSED) {
 157                 if (smc->clcsock) {
 158                         release_sock(sk);
 159                         smc_clcsock_release(smc);
 160                         lock_sock(sk);
 161                 }
 162                 if (!smc->use_fallback)
 163                         smc_conn_free(&smc->conn);
 164         }
 165
 166         return rc;
 167 }
 168
 169 static int smc_release(struct socket *sock)
 170 {
 171         struct sock *sk = sock->sk;
 172         struct smc_sock *smc;
 173         int rc = 0;
 174
 175         if (!sk)
 176                 goto out;
 177
 178         sock_hold(sk); /* sock_put below */
 179         smc = smc_sk(sk);
 180
 181         /* cleanup for a dangling non-blocking connect */
 182         if (smc->connect_nonblock && sk->sk_state == SMC_INIT)
 183                 tcp_abort(smc->clcsock->sk, ECONNABORTED);
 184         flush_work(&smc->connect_work);
 185
 186         if (sk->sk_state == SMC_LISTEN)
 187                 /* smc_close_non_accepted() is called and acquires
 188                  * sock lock for child sockets again
 189                  */
 190                 lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
 191         else
 192                 lock_sock(sk);
 193
 194         rc = __smc_release(smc);
 195
 196         /* detach socket */
 197         sock_orphan(sk);
 198         sock->sk = NULL;
 199         release_sock(sk);
 200
 201         sock_put(sk); /* sock_hold above */
 202         sock_put(sk); /* final sock_put */
 203 out:
 204         return rc;
 205 }
 206
 207 static void smc_destruct(struct sock *sk)
 208 {
 209         if (sk->sk_state != SMC_CLOSED)
 210                 return;
 211         if (!sock_flag(sk, SOCK_DEAD))
 212                 return;
 213
 214         sk_refcnt_debug_dec(sk);
 215 }
 216
 217 static struct sock *smc_sock_alloc(struct net *net, struct socket *sock,
 218                                    int protocol)
 219 {
 220         struct smc_sock *smc;
 221         struct proto *prot;
 222         struct sock *sk;
 223
 224         prot = (protocol == SMCPROTO_SMC6) ? &smc_proto6 : &smc_proto;
 225         sk = sk_alloc(net, PF_SMC, GFP_KERNEL, prot, 0);
 226         if (!sk)
 227                 return NULL;
 228
 229         sock_init_data(sock, sk); /* sets sk_refcnt to 1 */
 230         sk->sk_state = SMC_INIT;
 231         sk->sk_destruct = smc_destruct;
 232         sk->sk_protocol = protocol;
 233         smc = smc_sk(sk);
 234         INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
 235         INIT_WORK(&smc->connect_work, smc_connect_work);
 236         INIT_DELAYED_WORK(&smc->conn.tx_work, smc_tx_work);
 237         INIT_LIST_HEAD(&smc->accept_q);
 238         spin_lock_init(&smc->accept_q_lock);
 239         spin_lock_init(&smc->conn.send_lock);
 240         sk->sk_prot->hash(sk);
 241         sk_refcnt_debug_inc(sk);
 242         mutex_init(&smc->clcsock_release_lock);
 243
 244         return sk;
 245 }
 246
 247 static int smc_bind(struct socket *sock, struct sockaddr *uaddr,
 248                     int addr_len)
 249 {
 250         struct sockaddr_in *addr = (struct sockaddr_in *)uaddr;
 251         struct sock *sk = sock->sk;
 252         struct smc_sock *smc;
 253         int rc;
 254
 255         smc = smc_sk(sk);
 256
 257         /* replicate tests from inet_bind(), to be safe wrt. future changes */
 258         rc = -EINVAL;
 259         if (addr_len < sizeof(struct sockaddr_in))
 260                 goto out;
 261
 262         rc = -EAFNOSUPPORT;
 263         if (addr->sin_family != AF_INET &&
 264             addr->sin_family != AF_INET6 &&
 265             addr->sin_family != AF_UNSPEC)
 266                 goto out;
 267         /* accept AF_UNSPEC (mapped to AF_INET) only if s_addr is INADDR_ANY */
 268         if (addr->sin_family == AF_UNSPEC &&
 269             addr->sin_addr.s_addr != htonl(INADDR_ANY))
 270                 goto out;
 271
 272         lock_sock(sk);
 273
 274         /* Check if socket is already active */
 275         rc = -EINVAL;
 276         if (sk->sk_state != SMC_INIT || smc->connect_nonblock)
 277                 goto out_rel;
 278
 279         smc->clcsock->sk->sk_reuse = sk->sk_reuse;
 280         rc = kernel_bind(smc->clcsock, uaddr, addr_len);
 281
 282 out_rel:
 283         release_sock(sk);
 284 out:
 285         return rc;
 286 }
 287
 288 static void smc_copy_sock_settings(struct sock *nsk, struct sock *osk,
 289                                    unsigned long mask)
 290 {
 291         /* options we don't get control via setsockopt for */
 292         nsk->sk_type = osk->sk_type;
 293         nsk->sk_sndbuf = osk->sk_sndbuf;
 294         nsk->sk_rcvbuf = osk->sk_rcvbuf;
 295         nsk->sk_sndtimeo = osk->sk_sndtimeo;
 296         nsk->sk_rcvtimeo = osk->sk_rcvtimeo;
 297         nsk->sk_mark = osk->sk_mark;
 298         nsk->sk_priority = osk->sk_priority;
 299         nsk->sk_rcvlowat = osk->sk_rcvlowat;
 300         nsk->sk_bound_dev_if = osk->sk_bound_dev_if;
 301         nsk->sk_err = osk->sk_err;
 302
 303         nsk->sk_flags &= ~mask;
 304         nsk->sk_flags |= osk->sk_flags & mask;
 305 }
 306
 307 #define SK_FLAGS_SMC_TO_CLC ((1UL << SOCK_URGINLINE) | \
 308                              (1UL << SOCK_KEEPOPEN) | \
 309                              (1UL << SOCK_LINGER) | \
 310                              (1UL << SOCK_BROADCAST) | \
 311                              (1UL << SOCK_TIMESTAMP) | \
 312                              (1UL << SOCK_DBG) | \
 313                              (1UL << SOCK_RCVTSTAMP) | \
 314                              (1UL << SOCK_RCVTSTAMPNS) | \
 315                              (1UL << SOCK_LOCALROUTE) | \
 316                              (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \
 317                              (1UL << SOCK_RXQ_OVFL) | \
 318                              (1UL << SOCK_WIFI_STATUS) | \
 319                              (1UL << SOCK_NOFCS) | \
 320                              (1UL << SOCK_FILTER_LOCKED) | \
 321                              (1UL << SOCK_TSTAMP_NEW))
 322 /* copy only relevant settings and flags of SOL_SOCKET level from smc to
 323  * clc socket (since smc is not called for these options from net/core)
 324  */
 325 static void smc_copy_sock_settings_to_clc(struct smc_sock *smc)
 326 {
 327         smc_copy_sock_settings(smc->clcsock->sk, &smc->sk, SK_FLAGS_SMC_TO_CLC);
 328 }
 329
 330 #define SK_FLAGS_CLC_TO_SMC ((1UL << SOCK_URGINLINE) | \
 331                              (1UL << SOCK_KEEPOPEN) | \
 332                              (1UL << SOCK_LINGER) | \
 333                              (1UL << SOCK_DBG))
 334 /* copy only settings and flags relevant for smc from clc to smc socket */
 335 static void smc_copy_sock_settings_to_smc(struct smc_sock *smc)
 336 {
 337         smc_copy_sock_settings(&smc->sk, smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC);
 338 }
 339
 340 /* register a new rmb, send confirm_rkey msg to register with peer */
 341 static int smc_reg_rmb(struct smc_link *link, struct smc_buf_desc *rmb_desc,
 342                        bool conf_rkey)
 343 {
 344         if (!rmb_desc->wr_reg) {
 345                 /* register memory region for new rmb */
 346                 if (smc_wr_reg_send(link, rmb_desc->mr_rx[SMC_SINGLE_LINK])) {
 347                         rmb_desc->regerr = 1;
 348                         return -EFAULT;
 349                 }
 350                 rmb_desc->wr_reg = 1;
 351         }
 352         if (!conf_rkey)
 353                 return 0;
 354         /* exchange confirm_rkey msg with peer */
 355         if (smc_llc_do_confirm_rkey(link, rmb_desc)) {
 356                 rmb_desc->regerr = 1;
 357                 return -EFAULT;
 358         }
 359         return 0;
 360 }
 361
 362 static int smc_clnt_conf_first_link(struct smc_sock *smc)
 363 {
 364         struct net *net = sock_net(smc->clcsock->sk);
 365         struct smc_link_group *lgr = smc->conn.lgr;
 366         struct smc_link *link;
 367         int rest;
 368         int rc;
 369
 370         link = &lgr->lnk[SMC_SINGLE_LINK];
 371         /* receive CONFIRM LINK request from server over RoCE fabric */
 372         rest = wait_for_completion_interruptible_timeout(
 373                 &link->llc_confirm,
 374                 SMC_LLC_WAIT_FIRST_TIME);
 375         if (rest <= 0) {
 376                 struct smc_clc_msg_decline dclc;
 377
 378                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
 379                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
 380                 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
 381         }
 382
 383         if (link->llc_confirm_rc)
 384                 return SMC_CLC_DECL_RMBE_EC;
 385
 386         rc = smc_ib_modify_qp_rts(link);
 387         if (rc)
 388                 return SMC_CLC_DECL_ERR_RDYLNK;
 389
 390         smc_wr_remember_qp_attr(link);
 391
 392         if (smc_reg_rmb(link, smc->conn.rmb_desc, false))
 393                 return SMC_CLC_DECL_ERR_REGRMB;
 394
 395         /* send CONFIRM LINK response over RoCE fabric */
 396         rc = smc_llc_send_confirm_link(link, SMC_LLC_RESP);
 397         if (rc < 0)
 398                 return SMC_CLC_DECL_TIMEOUT_CL;
 399
 400         /* receive ADD LINK request from server over RoCE fabric */
 401         rest = wait_for_completion_interruptible_timeout(&link->llc_add,
 402                                                          SMC_LLC_WAIT_TIME);
 403         if (rest <= 0) {
 404                 struct smc_clc_msg_decline dclc;
 405
 406                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
 407                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
 408                 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_AL : rc;
 409         }
 410
 411         /* send add link reject message, only one link supported for now */
 412         rc = smc_llc_send_add_link(link,
 413                                    link->smcibdev->mac[link->ibport - 1],
 414                                    link->gid, SMC_LLC_RESP);
 415         if (rc < 0)
 416                 return SMC_CLC_DECL_TIMEOUT_AL;
 417
 418         smc_llc_link_active(link, net->ipv4.sysctl_tcp_keepalive_time);
 419
 420         return 0;
 421 }
 422
 423 static void smcr_conn_save_peer_info(struct smc_sock *smc,
 424                                      struct smc_clc_msg_accept_confirm *clc)
 425 {
 426         int bufsize = smc_uncompress_bufsize(clc->rmbe_size);
 427
 428         smc->conn.peer_rmbe_idx = clc->rmbe_idx;
 429         smc->conn.local_tx_ctrl.token = ntohl(clc->rmbe_alert_token);
 430         smc->conn.peer_rmbe_size = bufsize;
 431         atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
 432         smc->conn.tx_off = bufsize * (smc->conn.peer_rmbe_idx - 1);
 433 }
 434
 435 static void smcd_conn_save_peer_info(struct smc_sock *smc,
 436                                      struct smc_clc_msg_accept_confirm *clc)
 437 {
 438         int bufsize = smc_uncompress_bufsize(clc->dmbe_size);
 439
 440         smc->conn.peer_rmbe_idx = clc->dmbe_idx;
 441         smc->conn.peer_token = clc->token;
 442         /* msg header takes up space in the buffer */
 443         smc->conn.peer_rmbe_size = bufsize - sizeof(struct smcd_cdc_msg);
 444         atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
 445         smc->conn.tx_off = bufsize * smc->conn.peer_rmbe_idx;
 446 }
 447
 448 static void smc_conn_save_peer_info(struct smc_sock *smc,
 449                                     struct smc_clc_msg_accept_confirm *clc)
 450 {
 451         if (smc->conn.lgr->is_smcd)
 452                 smcd_conn_save_peer_info(smc, clc);
 453         else
 454                 smcr_conn_save_peer_info(smc, clc);
 455 }
 456
 457 static void smc_link_save_peer_info(struct smc_link *link,
 458                                     struct smc_clc_msg_accept_confirm *clc)
 459 {
 460         link->peer_qpn = ntoh24(clc->qpn);
 461         memcpy(link->peer_gid, clc->lcl.gid, SMC_GID_SIZE);
 462         memcpy(link->peer_mac, clc->lcl.mac, sizeof(link->peer_mac));
 463         link->peer_psn = ntoh24(clc->psn);
 464         link->peer_mtu = clc->qp_mtu;
 465 }
 466
 467 static void smc_switch_to_fallback(struct smc_sock *smc)
 468 {
 469         smc->use_fallback = true;
 470         if (smc->sk.sk_socket && smc->sk.sk_socket->file) {
 471                 smc->clcsock->file = smc->sk.sk_socket->file;
 472                 smc->clcsock->file->private_data = smc->clcsock;
 473         }
 474 }
 475
 476 /* fall back during connect */
 477 static int smc_connect_fallback(struct smc_sock *smc, int reason_code)
 478 {
 479         smc_switch_to_fallback(smc);
 480         smc->fallback_rsn = reason_code;
 481         smc_copy_sock_settings_to_clc(smc);
 482         smc->connect_nonblock = 0;
 483         if (smc->sk.sk_state == SMC_INIT)
 484                 smc->sk.sk_state = SMC_ACTIVE;
 485         return 0;
 486 }
 487
 488 /* decline and fall back during connect */
 489 static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code)
 490 {
 491         int rc;
 492
 493         if (reason_code < 0) { /* error, fallback is not possible */
 494                 if (smc->sk.sk_state == SMC_INIT)
 495                         sock_put(&smc->sk); /* passive closing */
 496                 return reason_code;
 497         }
 498         if (reason_code != SMC_CLC_DECL_PEERDECL) {
 499                 rc = smc_clc_send_decline(smc, reason_code);
 500                 if (rc < 0) {
 501                         if (smc->sk.sk_state == SMC_INIT)
 502                                 sock_put(&smc->sk); /* passive closing */
 503                         return rc;
 504                 }
 505         }
 506         return smc_connect_fallback(smc, reason_code);
 507 }
 508
 509 /* abort connecting */
 510 static int smc_connect_abort(struct smc_sock *smc, int reason_code,
 511                              int local_contact)
 512 {
 513         if (local_contact == SMC_FIRST_CONTACT)
 514                 smc_lgr_forget(smc->conn.lgr);
 515         if (smc->conn.lgr->is_smcd)
 516                 /* there is only one lgr role for SMC-D; use server lock */
 517                 mutex_unlock(&smc_server_lgr_pending);
 518         else
 519                 mutex_unlock(&smc_client_lgr_pending);
 520
 521         smc_conn_free(&smc->conn);
 522         smc->connect_nonblock = 0;
 523         return reason_code;
 524 }
 525
 526 /* check if there is a rdma device available for this connection. */
 527 /* called for connect and listen */
 528 static int smc_find_rdma_device(struct smc_sock *smc, struct smc_init_info *ini)
 529 {
 530         /* PNET table look up: search active ib_device and port
 531          * within same PNETID that also contains the ethernet device
 532          * used for the internal TCP socket
 533          */
 534         smc_pnet_find_roce_resource(smc->clcsock->sk, ini);
 535         if (!ini->ib_dev)
 536                 return SMC_CLC_DECL_NOSMCRDEV;
 537         return 0;
 538 }
 539
 540 /* check if there is an ISM device available for this connection. */
 541 /* called for connect and listen */
 542 static int smc_find_ism_device(struct smc_sock *smc, struct smc_init_info *ini)
 543 {
 544         /* Find ISM device with same PNETID as connecting interface  */
 545         smc_pnet_find_ism_resource(smc->clcsock->sk, ini);
 546         if (!ini->ism_dev)
 547                 return SMC_CLC_DECL_NOSMCDDEV;
 548         return 0;
 549 }
 550
 551 /* Check for VLAN ID and register it on ISM device just for CLC handshake */
 552 static int smc_connect_ism_vlan_setup(struct smc_sock *smc,
 553                                       struct smc_init_info *ini)
 554 {
 555         if (ini->vlan_id && smc_ism_get_vlan(ini->ism_dev, ini->vlan_id))
 556                 return SMC_CLC_DECL_ISMVLANERR;
 557         return 0;
 558 }
 559
 560 /* cleanup temporary VLAN ID registration used for CLC handshake. If ISM is
 561  * used, the VLAN ID will be registered again during the connection setup.
 562  */
 563 static int smc_connect_ism_vlan_cleanup(struct smc_sock *smc, bool is_smcd,
 564                                         struct smc_init_info *ini)
 565 {
 566         if (!is_smcd)
 567                 return 0;
 568         if (ini->vlan_id && smc_ism_put_vlan(ini->ism_dev, ini->vlan_id))
 569                 return SMC_CLC_DECL_CNFERR;
 570         return 0;
 571 }
 572
 573 /* CLC handshake during connect */
 574 static int smc_connect_clc(struct smc_sock *smc, int smc_type,
 575                            struct smc_clc_msg_accept_confirm *aclc,
 576                            struct smc_init_info *ini)
 577 {
 578         int rc = 0;
 579
 580         /* do inband token exchange */
 581         rc = smc_clc_send_proposal(smc, smc_type, ini);
 582         if (rc)
 583                 return rc;
 584         /* receive SMC Accept CLC message */
 585         return smc_clc_wait_msg(smc, aclc, sizeof(*aclc), SMC_CLC_ACCEPT,
 586                                 CLC_WAIT_TIME);
 587 }
 588
 589 /* setup for RDMA connection of client */
 590 static int smc_connect_rdma(struct smc_sock *smc,
 591                             struct smc_clc_msg_accept_confirm *aclc,
 592                             struct smc_init_info *ini)
 593 {
 594         struct smc_link *link;
 595         int reason_code = 0;
 596
 597         ini->is_smcd = false;
 598         ini->ib_lcl = &aclc->lcl;
 599         ini->ib_clcqpn = ntoh24(aclc->qpn);
 600         ini->srv_first_contact = aclc->hdr.flag;
 601
 602         mutex_lock(&smc_client_lgr_pending);
 603         reason_code = smc_conn_create(smc, ini);
 604         if (reason_code) {
 605                 mutex_unlock(&smc_client_lgr_pending);
 606                 return reason_code;
 607         }
 608         link = &smc->conn.lgr->lnk[SMC_SINGLE_LINK];
 609
 610         smc_conn_save_peer_info(smc, aclc);
 611
 612         /* create send buffer and rmb */
 613         if (smc_buf_create(smc, false))
 614                 return smc_connect_abort(smc, SMC_CLC_DECL_MEM,
 615                                          ini->cln_first_contact);
 616
 617         if (ini->cln_first_contact == SMC_FIRST_CONTACT)
 618                 smc_link_save_peer_info(link, aclc);
 619
 620         if (smc_rmb_rtoken_handling(&smc->conn, aclc))
 621                 return smc_connect_abort(smc, SMC_CLC_DECL_ERR_RTOK,
 622                                          ini->cln_first_contact);
 623
 624         smc_close_init(smc);
 625         smc_rx_init(smc);
 626
 627         if (ini->cln_first_contact == SMC_FIRST_CONTACT) {
 628                 if (smc_ib_ready_link(link))
 629                         return smc_connect_abort(smc, SMC_CLC_DECL_ERR_RDYLNK,
 630                                                  ini->cln_first_contact);
 631         } else {
 632                 if (smc_reg_rmb(link, smc->conn.rmb_desc, true))
 633                         return smc_connect_abort(smc, SMC_CLC_DECL_ERR_REGRMB,
 634                                                  ini->cln_first_contact);
 635         }
 636         smc_rmb_sync_sg_for_device(&smc->conn);
 637
 638         reason_code = smc_clc_send_confirm(smc);
 639         if (reason_code)
 640                 return smc_connect_abort(smc, reason_code,
 641                                          ini->cln_first_contact);
 642
 643         smc_tx_init(smc);
 644
 645         if (ini->cln_first_contact == SMC_FIRST_CONTACT) {
 646                 /* QP confirmation over RoCE fabric */
 647                 reason_code = smc_clnt_conf_first_link(smc);
 648                 if (reason_code)
 649                         return smc_connect_abort(smc, reason_code,
 650                                                  ini->cln_first_contact);
 651         }
 652         mutex_unlock(&smc_client_lgr_pending);
 653
 654         smc_copy_sock_settings_to_clc(smc);
 655         smc->connect_nonblock = 0;
 656         if (smc->sk.sk_state == SMC_INIT)
 657                 smc->sk.sk_state = SMC_ACTIVE;
 658
 659         return 0;
 660 }
 661
 662 /* setup for ISM connection of client */
 663 static int smc_connect_ism(struct smc_sock *smc,
 664                            struct smc_clc_msg_accept_confirm *aclc,
 665                            struct smc_init_info *ini)
 666 {
 667         int rc = 0;
 668
 669         ini->is_smcd = true;
 670         ini->ism_gid = aclc->gid;
 671         ini->srv_first_contact = aclc->hdr.flag;
 672
 673         /* there is only one lgr role for SMC-D; use server lock */
 674         mutex_lock(&smc_server_lgr_pending);
 675         rc = smc_conn_create(smc, ini);
 676         if (rc) {
 677                 mutex_unlock(&smc_server_lgr_pending);
 678                 return rc;
 679         }
 680
 681         /* Create send and receive buffers */
 682         if (smc_buf_create(smc, true))
 683                 return smc_connect_abort(smc, SMC_CLC_DECL_MEM,
 684                                          ini->cln_first_contact);
 685
 686         smc_conn_save_peer_info(smc, aclc);
 687         smc_close_init(smc);
 688         smc_rx_init(smc);
 689         smc_tx_init(smc);
 690
 691         rc = smc_clc_send_confirm(smc);
 692         if (rc)
 693                 return smc_connect_abort(smc, rc, ini->cln_first_contact);
 694         mutex_unlock(&smc_server_lgr_pending);
 695
 696         smc_copy_sock_settings_to_clc(smc);
 697         smc->connect_nonblock = 0;
 698         if (smc->sk.sk_state == SMC_INIT)
 699                 smc->sk.sk_state = SMC_ACTIVE;
 700
 701         return 0;
 702 }
 703
 704 /* perform steps before actually connecting */
 705 static int __smc_connect(struct smc_sock *smc)
 706 {
 707         bool ism_supported = false, rdma_supported = false;
 708         struct smc_clc_msg_accept_confirm aclc;
 709         struct smc_init_info ini = {0};
 710         int smc_type;
 711         int rc = 0;
 712
 713         if (smc->use_fallback)
 714                 return smc_connect_fallback(smc, smc->fallback_rsn);
 715
 716         /* if peer has not signalled SMC-capability, fall back */
 717         if (!tcp_sk(smc->clcsock->sk)->syn_smc)
 718                 return smc_connect_fallback(smc, SMC_CLC_DECL_PEERNOSMC);
 719
 720         /* IPSec connections opt out of SMC-R optimizations */
 721         if (using_ipsec(smc))
 722                 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_IPSEC);
 723
 724         /* get vlan id from IP device */
 725         if (smc_vlan_by_tcpsk(smc->clcsock, &ini))
 726                 return smc_connect_decline_fallback(smc,
 727                                                     SMC_CLC_DECL_GETVLANERR);
 728
 729         /* check if there is an ism device available */
 730         if (!smc_find_ism_device(smc, &ini) &&
 731             !smc_connect_ism_vlan_setup(smc, &ini)) {
 732                 /* ISM is supported for this connection */
 733                 ism_supported = true;
 734                 smc_type = SMC_TYPE_D;
 735         }
 736
 737         /* check if there is a rdma device available */
 738         if (!smc_find_rdma_device(smc, &ini)) {
 739                 /* RDMA is supported for this connection */
 740                 rdma_supported = true;
 741                 if (ism_supported)
 742                         smc_type = SMC_TYPE_B; /* both */
 743                 else
 744                         smc_type = SMC_TYPE_R; /* only RDMA */
 745         }
 746
 747         /* if neither ISM nor RDMA are supported, fallback */
 748         if (!rdma_supported && !ism_supported)
 749                 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_NOSMCDEV);
 750
 751         /* perform CLC handshake */
 752         rc = smc_connect_clc(smc, smc_type, &aclc, &ini);
 753         if (rc) {
 754                 smc_connect_ism_vlan_cleanup(smc, ism_supported, &ini);
 755                 return smc_connect_decline_fallback(smc, rc);
 756         }
 757
 758         /* depending on previous steps, connect using rdma or ism */
 759         if (rdma_supported && aclc.hdr.path == SMC_TYPE_R)
 760                 rc = smc_connect_rdma(smc, &aclc, &ini);
 761         else if (ism_supported && aclc.hdr.path == SMC_TYPE_D)
 762                 rc = smc_connect_ism(smc, &aclc, &ini);
 763         else
 764                 rc = SMC_CLC_DECL_MODEUNSUPP;
 765         if (rc) {
 766                 smc_connect_ism_vlan_cleanup(smc, ism_supported, &ini);
 767                 return smc_connect_decline_fallback(smc, rc);
 768         }
 769
 770         smc_connect_ism_vlan_cleanup(smc, ism_supported, &ini);
 771         return 0;
 772 }
 773
 774 static void smc_connect_work(struct work_struct *work)
 775 {
 776         struct smc_sock *smc = container_of(work, struct smc_sock,
 777                                             connect_work);
 778         long timeo = smc->sk.sk_sndtimeo;
 779         int rc = 0;
 780
 781         if (!timeo)
 782                 timeo = MAX_SCHEDULE_TIMEOUT;
 783         lock_sock(smc->clcsock->sk);
 784         if (smc->clcsock->sk->sk_err) {
 785                 smc->sk.sk_err = smc->clcsock->sk->sk_err;
 786         } else if ((1 << smc->clcsock->sk->sk_state) &
 787                                         (TCPF_SYN_SENT | TCP_SYN_RECV)) {
 788                 rc = sk_stream_wait_connect(smc->clcsock->sk, &timeo);
 789                 if ((rc == -EPIPE) &&
 790                     ((1 << smc->clcsock->sk->sk_state) &
 791                                         (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)))
 792                         rc = 0;
 793         }
 794         release_sock(smc->clcsock->sk);
 795         lock_sock(&smc->sk);
 796         if (rc != 0 || smc->sk.sk_err) {
 797                 smc->sk.sk_state = SMC_CLOSED;
 798                 if (rc == -EPIPE || rc == -EAGAIN)
 799                         smc->sk.sk_err = EPIPE;
 800                 else if (signal_pending(current))
 801                         smc->sk.sk_err = -sock_intr_errno(timeo);
 802                 goto out;
 803         }
 804
 805         rc = __smc_connect(smc);
 806         if (rc < 0)
 807                 smc->sk.sk_err = -rc;
 808
 809 out:
 810         if (!sock_flag(&smc->sk, SOCK_DEAD)) {
 811                 if (smc->sk.sk_err) {
 812                         smc->sk.sk_state_change(&smc->sk);
 813                 } else { /* allow polling before and after fallback decision */
 814                         smc->clcsock->sk->sk_write_space(smc->clcsock->sk);
 815                         smc->sk.sk_write_space(&smc->sk);
 816                 }
 817         }
 818         release_sock(&smc->sk);
 819 }
 820
 821 static int smc_connect(struct socket *sock, struct sockaddr *addr,
 822                        int alen, int flags)
 823 {
 824         struct sock *sk = sock->sk;
 825         struct smc_sock *smc;
 826         int rc = -EINVAL;
 827
 828         smc = smc_sk(sk);
 829
 830         /* separate smc parameter checking to be safe */
 831         if (alen < sizeof(addr->sa_family))
 832                 goto out_err;
 833         if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6)
 834                 goto out_err;
 835
 836         lock_sock(sk);
 837         switch (sk->sk_state) {
 838         default:
 839                 goto out;
 840         case SMC_ACTIVE:
 841                 rc = -EISCONN;
 842                 goto out;
 843         case SMC_INIT:
 844                 rc = 0;
 845                 break;
 846         }
 847
 848         smc_copy_sock_settings_to_clc(smc);
 849         tcp_sk(smc->clcsock->sk)->syn_smc = 1;
 850         if (smc->connect_nonblock) {
 851                 rc = -EALREADY;
 852                 goto out;
 853         }
 854         rc = kernel_connect(smc->clcsock, addr, alen, flags);
 855         if (rc && rc != -EINPROGRESS)
 856                 goto out;
 857
 858         sock_hold(&smc->sk); /* sock put in passive closing */
 859         if (flags & O_NONBLOCK) {
 860                 if (schedule_work(&smc->connect_work))
 861                         smc->connect_nonblock = 1;
 862                 rc = -EINPROGRESS;
 863         } else {
 864                 rc = __smc_connect(smc);
 865                 if (rc < 0)
 866                         goto out;
 867                 else
 868                         rc = 0; /* success cases including fallback */
 869         }
 870
 871 out:
 872         release_sock(sk);
 873 out_err:
 874         return rc;
 875 }
 876
 877 static int smc_clcsock_accept(struct smc_sock *lsmc, struct smc_sock **new_smc)
 878 {
 879         struct socket *new_clcsock = NULL;
 880         struct sock *lsk = &lsmc->sk;
 881         struct sock *new_sk;
 882         int rc = -EINVAL;
 883
 884         release_sock(lsk);
 885         new_sk = smc_sock_alloc(sock_net(lsk), NULL, lsk->sk_protocol);
 886         if (!new_sk) {
 887                 rc = -ENOMEM;
 888                 lsk->sk_err = ENOMEM;
 889                 *new_smc = NULL;
 890                 lock_sock(lsk);
 891                 goto out;
 892         }
 893         *new_smc = smc_sk(new_sk);
 894
 895         mutex_lock(&lsmc->clcsock_release_lock);
 896         if (lsmc->clcsock)
 897                 rc = kernel_accept(lsmc->clcsock, &new_clcsock, 0);
 898         mutex_unlock(&lsmc->clcsock_release_lock);
 899         lock_sock(lsk);
 900         if  (rc < 0)
 901                 lsk->sk_err = -rc;
 902         if (rc < 0 || lsk->sk_state == SMC_CLOSED) {
 903                 new_sk->sk_prot->unhash(new_sk);
 904                 if (new_clcsock)
 905                         sock_release(new_clcsock);
 906                 new_sk->sk_state = SMC_CLOSED;
 907                 sock_set_flag(new_sk, SOCK_DEAD);
 908                 sock_put(new_sk); /* final */
 909                 *new_smc = NULL;
 910                 goto out;
 911         }
 912
 913         (*new_smc)->clcsock = new_clcsock;
 914 out:
 915         return rc;
 916 }
 917
 918 /* add a just created sock to the accept queue of the listen sock as
 919  * candidate for a following socket accept call from user space
 920  */
 921 static void smc_accept_enqueue(struct sock *parent, struct sock *sk)
 922 {
 923         struct smc_sock *par = smc_sk(parent);
 924
 925         sock_hold(sk); /* sock_put in smc_accept_unlink () */
 926         spin_lock(&par->accept_q_lock);
 927         list_add_tail(&smc_sk(sk)->accept_q, &par->accept_q);
 928         spin_unlock(&par->accept_q_lock);
 929         sk_acceptq_added(parent);
 930 }
 931
 932 /* remove a socket from the accept queue of its parental listening socket */
 933 static void smc_accept_unlink(struct sock *sk)
 934 {
 935         struct smc_sock *par = smc_sk(sk)->listen_smc;
 936
 937         spin_lock(&par->accept_q_lock);
 938         list_del_init(&smc_sk(sk)->accept_q);
 939         spin_unlock(&par->accept_q_lock);
 940         sk_acceptq_removed(&smc_sk(sk)->listen_smc->sk);
 941         sock_put(sk); /* sock_hold in smc_accept_enqueue */
 942 }
 943
 944 /* remove a sock from the accept queue to bind it to a new socket created
 945  * for a socket accept call from user space
 946  */
 947 struct sock *smc_accept_dequeue(struct sock *parent,
 948                                 struct socket *new_sock)
 949 {
 950         struct smc_sock *isk, *n;
 951         struct sock *new_sk;
 952
 953         list_for_each_entry_safe(isk, n, &smc_sk(parent)->accept_q, accept_q) {
 954                 new_sk = (struct sock *)isk;
 955
 956                 smc_accept_unlink(new_sk);
 957                 if (new_sk->sk_state == SMC_CLOSED) {
 958                         new_sk->sk_prot->unhash(new_sk);
 959                         if (isk->clcsock) {
 960                                 sock_release(isk->clcsock);
 961                                 isk->clcsock = NULL;
 962                         }
 963                         sock_put(new_sk); /* final */
 964                         continue;
 965                 }
 966                 if (new_sock) {
 967                         sock_graft(new_sk, new_sock);
 968                         if (isk->use_fallback) {
 969                                 smc_sk(new_sk)->clcsock->file = new_sock->file;
 970                                 isk->clcsock->file->private_data = isk->clcsock;
 971                         }
 972                 }
 973                 return new_sk;
 974         }
 975         return NULL;
 976 }
 977
 978 /* clean up for a created but never accepted sock */
 979 void smc_close_non_accepted(struct sock *sk)
 980 {
 981         struct smc_sock *smc = smc_sk(sk);
 982
 983         sock_hold(sk); /* sock_put below */
 984         lock_sock(sk);
 985         if (!sk->sk_lingertime)
 986                 /* wait for peer closing */
 987                 sk->sk_lingertime = SMC_MAX_STREAM_WAIT_TIMEOUT;
 988         __smc_release(smc);
 989         release_sock(sk);
 990         sock_put(sk); /* sock_hold above */
 991         sock_put(sk); /* final sock_put */
 992 }
 993
 994 static int smc_serv_conf_first_link(struct smc_sock *smc)
 995 {
 996         struct net *net = sock_net(smc->clcsock->sk);
 997         struct smc_link_group *lgr = smc->conn.lgr;
 998         struct smc_link *link;
 999         int rest;
1000         int rc;
1001
1002         link = &lgr->lnk[SMC_SINGLE_LINK];
1003
1004         if (smc_reg_rmb(link, smc->conn.rmb_desc, false))
1005                 return SMC_CLC_DECL_ERR_REGRMB;
1006
1007         /* send CONFIRM LINK request to client over the RoCE fabric */
1008         rc = smc_llc_send_confirm_link(link, SMC_LLC_REQ);
1009         if (rc < 0)
1010                 return SMC_CLC_DECL_TIMEOUT_CL;
1011
1012         /* receive CONFIRM LINK response from client over the RoCE fabric */
1013         rest = wait_for_completion_interruptible_timeout(
1014                 &link->llc_confirm_resp,
1015                 SMC_LLC_WAIT_FIRST_TIME);
1016         if (rest <= 0) {
1017                 struct smc_clc_msg_decline dclc;
1018
1019                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
1020                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
1021                 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
1022         }
1023
1024         if (link->llc_confirm_resp_rc)
1025                 return SMC_CLC_DECL_RMBE_EC;
1026
1027         /* send ADD LINK request to client over the RoCE fabric */
1028         rc = smc_llc_send_add_link(link,
1029                                    link->smcibdev->mac[link->ibport - 1],
1030                                    link->gid, SMC_LLC_REQ);
1031         if (rc < 0)
1032                 return SMC_CLC_DECL_TIMEOUT_AL;
1033
1034         /* receive ADD LINK response from client over the RoCE fabric */
1035         rest = wait_for_completion_interruptible_timeout(&link->llc_add_resp,
1036                                                          SMC_LLC_WAIT_TIME);
1037         if (rest <= 0) {
1038                 struct smc_clc_msg_decline dclc;
1039
1040                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
1041                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
1042                 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_AL : rc;
1043         }
1044
1045         smc_llc_link_active(link, net->ipv4.sysctl_tcp_keepalive_time);
1046
1047         return 0;
1048 }
1049
1050 /* listen worker: finish */
1051 static void smc_listen_out(struct smc_sock *new_smc)
1052 {
1053         struct smc_sock *lsmc = new_smc->listen_smc;
1054         struct sock *newsmcsk = &new_smc->sk;
1055
1056         if (lsmc->sk.sk_state == SMC_LISTEN) {
1057                 lock_sock_nested(&lsmc->sk, SINGLE_DEPTH_NESTING);
1058                 smc_accept_enqueue(&lsmc->sk, newsmcsk);
1059                 release_sock(&lsmc->sk);
1060         } else { /* no longer listening */
1061                 smc_close_non_accepted(newsmcsk);
1062         }
1063
1064         /* Wake up accept */
1065         lsmc->sk.sk_data_ready(&lsmc->sk);
1066         sock_put(&lsmc->sk); /* sock_hold in smc_tcp_listen_work */
1067 }
1068
1069 /* listen worker: finish in state connected */
1070 static void smc_listen_out_connected(struct smc_sock *new_smc)
1071 {
1072         struct sock *newsmcsk = &new_smc->sk;
1073
1074         sk_refcnt_debug_inc(newsmcsk);
1075         if (newsmcsk->sk_state == SMC_INIT)
1076                 newsmcsk->sk_state = SMC_ACTIVE;
1077
1078         smc_listen_out(new_smc);
1079 }
1080
1081 /* listen worker: finish in error state */
1082 static void smc_listen_out_err(struct smc_sock *new_smc)
1083 {
1084         struct sock *newsmcsk = &new_smc->sk;
1085
1086         if (newsmcsk->sk_state == SMC_INIT)
1087                 sock_put(&new_smc->sk); /* passive closing */
1088         newsmcsk->sk_state = SMC_CLOSED;
1089         smc_conn_free(&new_smc->conn);
1090
1091         smc_listen_out(new_smc);
1092 }
1093
1094 /* listen worker: decline and fall back if possible */
1095 static void smc_listen_decline(struct smc_sock *new_smc, int reason_code,
1096                                int local_contact)
1097 {
1098         /* RDMA setup failed, switch back to TCP */
1099         if (local_contact == SMC_FIRST_CONTACT)
1100                 smc_lgr_forget(new_smc->conn.lgr);
1101         if (reason_code < 0) { /* error, no fallback possible */
1102                 smc_listen_out_err(new_smc);
1103                 return;
1104         }
1105         smc_conn_free(&new_smc->conn);
1106         smc_switch_to_fallback(new_smc);
1107         new_smc->fallback_rsn = reason_code;
1108         if (reason_code && reason_code != SMC_CLC_DECL_PEERDECL) {
1109                 if (smc_clc_send_decline(new_smc, reason_code) < 0) {
1110                         smc_listen_out_err(new_smc);
1111                         return;
1112                 }
1113         }
1114         smc_listen_out_connected(new_smc);
1115 }
1116
1117 /* listen worker: check prefixes */
1118 static int smc_listen_prfx_check(struct smc_sock *new_smc,
1119                                  struct smc_clc_msg_proposal *pclc)
1120 {
1121         struct smc_clc_msg_proposal_prefix *pclc_prfx;
1122         struct socket *newclcsock = new_smc->clcsock;
1123
1124         pclc_prfx = smc_clc_proposal_get_prefix(pclc);
1125         if (smc_clc_prfx_match(newclcsock, pclc_prfx))
1126                 return SMC_CLC_DECL_DIFFPREFIX;
1127
1128         return 0;
1129 }
1130
1131 /* listen worker: initialize connection and buffers */
1132 static int smc_listen_rdma_init(struct smc_sock *new_smc,
1133                                 struct smc_init_info *ini)
1134 {
1135         int rc;
1136
1137         /* allocate connection / link group */
1138         rc = smc_conn_create(new_smc, ini);
1139         if (rc)
1140                 return rc;
1141
1142         /* create send buffer and rmb */
1143         if (smc_buf_create(new_smc, false))
1144                 return SMC_CLC_DECL_MEM;
1145
1146         return 0;
1147 }
1148
1149 /* listen worker: initialize connection and buffers for SMC-D */
1150 static int smc_listen_ism_init(struct smc_sock *new_smc,
1151                                struct smc_clc_msg_proposal *pclc,
1152                                struct smc_init_info *ini)
1153 {
1154         struct smc_clc_msg_smcd *pclc_smcd;
1155         int rc;
1156
1157         pclc_smcd = smc_get_clc_msg_smcd(pclc);
1158         ini->ism_gid = pclc_smcd->gid;
1159         rc = smc_conn_create(new_smc, ini);
1160         if (rc)
1161                 return rc;
1162
1163         /* Check if peer can be reached via ISM device */
1164         if (smc_ism_cantalk(new_smc->conn.lgr->peer_gid,
1165                             new_smc->conn.lgr->vlan_id,
1166                             new_smc->conn.lgr->smcd)) {
1167                 if (ini->cln_first_contact == SMC_FIRST_CONTACT)
1168                         smc_lgr_forget(new_smc->conn.lgr);
1169                 smc_conn_free(&new_smc->conn);
1170                 return SMC_CLC_DECL_SMCDNOTALK;
1171         }
1172
1173         /* Create send and receive buffers */
1174         if (smc_buf_create(new_smc, true)) {
1175                 if (ini->cln_first_contact == SMC_FIRST_CONTACT)
1176                         smc_lgr_forget(new_smc->conn.lgr);
1177                 smc_conn_free(&new_smc->conn);
1178                 return SMC_CLC_DECL_MEM;
1179         }
1180
1181         return 0;
1182 }
1183
1184 /* listen worker: register buffers */
1185 static int smc_listen_rdma_reg(struct smc_sock *new_smc, int local_contact)
1186 {
1187         struct smc_link *link = &new_smc->conn.lgr->lnk[SMC_SINGLE_LINK];
1188
1189         if (local_contact != SMC_FIRST_CONTACT) {
1190                 if (smc_reg_rmb(link, new_smc->conn.rmb_desc, true))
1191                         return SMC_CLC_DECL_ERR_REGRMB;
1192         }
1193         smc_rmb_sync_sg_for_device(&new_smc->conn);
1194
1195         return 0;
1196 }
1197
1198 /* listen worker: finish RDMA setup */
1199 static int smc_listen_rdma_finish(struct smc_sock *new_smc,
1200                                   struct smc_clc_msg_accept_confirm *cclc,
1201                                   int local_contact)
1202 {
1203         struct smc_link *link = &new_smc->conn.lgr->lnk[SMC_SINGLE_LINK];
1204         int reason_code = 0;
1205
1206         if (local_contact == SMC_FIRST_CONTACT)
1207                 smc_link_save_peer_info(link, cclc);
1208
1209         if (smc_rmb_rtoken_handling(&new_smc->conn, cclc)) {
1210                 reason_code = SMC_CLC_DECL_ERR_RTOK;
1211                 goto decline;
1212         }
1213
1214         if (local_contact == SMC_FIRST_CONTACT) {
1215                 if (smc_ib_ready_link(link)) {
1216                         reason_code = SMC_CLC_DECL_ERR_RDYLNK;
1217                         goto decline;
1218                 }
1219                 /* QP confirmation over RoCE fabric */
1220                 reason_code = smc_serv_conf_first_link(new_smc);
1221                 if (reason_code)
1222                         goto decline;
1223         }
1224         return 0;
1225
1226 decline:
1227         smc_listen_decline(new_smc, reason_code, local_contact);
1228         return reason_code;
1229 }
1230
1231 /* setup for RDMA connection of server */
1232 static void smc_listen_work(struct work_struct *work)
1233 {
1234         struct smc_sock *new_smc = container_of(work, struct smc_sock,
1235                                                 smc_listen_work);
1236         struct socket *newclcsock = new_smc->clcsock;
1237         struct smc_clc_msg_accept_confirm cclc;
1238         struct smc_clc_msg_proposal *pclc;
1239         struct smc_init_info ini = {0};
1240         bool ism_supported = false;
1241         u8 buf[SMC_CLC_MAX_LEN];
1242         int rc = 0;
1243
1244         if (new_smc->listen_smc->sk.sk_state != SMC_LISTEN)
1245                 return smc_listen_out_err(new_smc);
1246
1247         if (new_smc->use_fallback) {
1248                 smc_listen_out_connected(new_smc);
1249                 return;
1250         }
1251
1252         /* check if peer is smc capable */
1253         if (!tcp_sk(newclcsock->sk)->syn_smc) {
1254                 smc_switch_to_fallback(new_smc);
1255                 new_smc->fallback_rsn = SMC_CLC_DECL_PEERNOSMC;
1256                 smc_listen_out_connected(new_smc);
1257                 return;
1258         }
1259
1260         /* do inband token exchange -
1261          * wait for and receive SMC Proposal CLC message
1262          */
1263         pclc = (struct smc_clc_msg_proposal *)&buf;
1264         rc = smc_clc_wait_msg(new_smc, pclc, SMC_CLC_MAX_LEN,
1265                               SMC_CLC_PROPOSAL, CLC_WAIT_TIME);
1266         if (rc)
1267                 goto out_decl;
1268
1269         /* IPSec connections opt out of SMC-R optimizations */
1270         if (using_ipsec(new_smc)) {
1271                 rc = SMC_CLC_DECL_IPSEC;
1272                 goto out_decl;
1273         }
1274
1275         /* check for matching IP prefix and subnet length */
1276         rc = smc_listen_prfx_check(new_smc, pclc);
1277         if (rc)
1278                 goto out_decl;
1279
1280         /* get vlan id from IP device */
1281         if (smc_vlan_by_tcpsk(new_smc->clcsock, &ini)) {
1282                 rc = SMC_CLC_DECL_GETVLANERR;
1283                 goto out_decl;
1284         }
1285
1286         mutex_lock(&smc_server_lgr_pending);
1287         smc_close_init(new_smc);
1288         smc_rx_init(new_smc);
1289         smc_tx_init(new_smc);
1290
1291         /* check if ISM is available */
1292         if (pclc->hdr.path == SMC_TYPE_D || pclc->hdr.path == SMC_TYPE_B) {
1293                 ini.is_smcd = true; /* prepare ISM check */
1294                 rc = smc_find_ism_device(new_smc, &ini);
1295                 if (!rc)
1296                         rc = smc_listen_ism_init(new_smc, pclc, &ini);
1297                 if (!rc)
1298                         ism_supported = true;
1299                 else if (pclc->hdr.path == SMC_TYPE_D)
1300                         goto out_unlock; /* skip RDMA and decline */
1301         }
1302
1303         /* check if RDMA is available */
1304         if (!ism_supported) { /* SMC_TYPE_R or SMC_TYPE_B */
1305                 /* prepare RDMA check */
1306                 ini.is_smcd = false;
1307                 ini.ism_dev = NULL;
1308                 ini.ib_lcl = &pclc->lcl;
1309                 rc = smc_find_rdma_device(new_smc, &ini);
1310                 if (rc) {
1311                         /* no RDMA device found */
1312                         if (pclc->hdr.path == SMC_TYPE_B)
1313                                 /* neither ISM nor RDMA device found */
1314                                 rc = SMC_CLC_DECL_NOSMCDEV;
1315                         goto out_unlock;
1316                 }
1317                 rc = smc_listen_rdma_init(new_smc, &ini);
1318                 if (rc)
1319                         goto out_unlock;
1320                 rc = smc_listen_rdma_reg(new_smc, ini.cln_first_contact);
1321                 if (rc)
1322                         goto out_unlock;
1323         }
1324
1325         /* send SMC Accept CLC message */
1326         rc = smc_clc_send_accept(new_smc, ini.cln_first_contact);
1327         if (rc)
1328                 goto out_unlock;
1329
1330         /* SMC-D does not need this lock any more */
1331         if (ism_supported)
1332                 mutex_unlock(&smc_server_lgr_pending);
1333
1334         /* receive SMC Confirm CLC message */
1335         rc = smc_clc_wait_msg(new_smc, &cclc, sizeof(cclc),
1336                               SMC_CLC_CONFIRM, CLC_WAIT_TIME);
1337         if (rc) {
1338                 if (!ism_supported)
1339                         goto out_unlock;
1340                 goto out_decl;
1341         }
1342
1343         /* finish worker */
1344         if (!ism_supported) {
1345                 rc = smc_listen_rdma_finish(new_smc, &cclc,
1346                                             ini.cln_first_contact);
1347                 mutex_unlock(&smc_server_lgr_pending);
1348                 if (rc)
1349                         return;
1350         }
1351         smc_conn_save_peer_info(new_smc, &cclc);
1352         smc_listen_out_connected(new_smc);
1353         return;
1354
1355 out_unlock:
1356         mutex_unlock(&smc_server_lgr_pending);
1357 out_decl:
1358         smc_listen_decline(new_smc, rc, ini.cln_first_contact);
1359 }
1360
1361 static void smc_tcp_listen_work(struct work_struct *work)
1362 {
1363         struct smc_sock *lsmc = container_of(work, struct smc_sock,
1364                                              tcp_listen_work);
1365         struct sock *lsk = &lsmc->sk;
1366         struct smc_sock *new_smc;
1367         int rc = 0;
1368
1369         lock_sock(lsk);
1370         while (lsk->sk_state == SMC_LISTEN) {
1371                 rc = smc_clcsock_accept(lsmc, &new_smc);
1372                 if (rc)
1373                         goto out;
1374                 if (!new_smc)
1375                         continue;
1376
1377                 new_smc->listen_smc = lsmc;
1378                 new_smc->use_fallback = lsmc->use_fallback;
1379                 new_smc->fallback_rsn = lsmc->fallback_rsn;
1380                 sock_hold(lsk); /* sock_put in smc_listen_work */
1381                 INIT_WORK(&new_smc->smc_listen_work, smc_listen_work);
1382                 smc_copy_sock_settings_to_smc(new_smc);
1383                 new_smc->sk.sk_sndbuf = lsmc->sk.sk_sndbuf;
1384                 new_smc->sk.sk_rcvbuf = lsmc->sk.sk_rcvbuf;
1385                 sock_hold(&new_smc->sk); /* sock_put in passive closing */
1386                 if (!schedule_work(&new_smc->smc_listen_work))
1387                         sock_put(&new_smc->sk);
1388         }
1389
1390 out:
1391         release_sock(lsk);
1392         sock_put(&lsmc->sk); /* sock_hold in smc_listen */
1393 }
1394
1395 static int smc_listen(struct socket *sock, int backlog)
1396 {
1397         struct sock *sk = sock->sk;
1398         struct smc_sock *smc;
1399         int rc;
1400
1401         smc = smc_sk(sk);
1402         lock_sock(sk);
1403
1404         rc = -EINVAL;
1405         if ((sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) ||
1406             smc->connect_nonblock)
1407                 goto out;
1408
1409         rc = 0;
1410         if (sk->sk_state == SMC_LISTEN) {
1411                 sk->sk_max_ack_backlog = backlog;
1412                 goto out;
1413         }
1414         /* some socket options are handled in core, so we could not apply
1415          * them to the clc socket -- copy smc socket options to clc socket
1416          */
1417         smc_copy_sock_settings_to_clc(smc);
1418         if (!smc->use_fallback)
1419                 tcp_sk(smc->clcsock->sk)->syn_smc = 1;
1420
1421         rc = kernel_listen(smc->clcsock, backlog);
1422         if (rc)
1423                 goto out;
1424         sk->sk_max_ack_backlog = backlog;
1425         sk->sk_ack_backlog = 0;
1426         sk->sk_state = SMC_LISTEN;
1427         sock_hold(sk); /* sock_hold in tcp_listen_worker */
1428         if (!schedule_work(&smc->tcp_listen_work))
1429                 sock_put(sk);
1430
1431 out:
1432         release_sock(sk);
1433         return rc;
1434 }
1435
1436 static int smc_accept(struct socket *sock, struct socket *new_sock,
1437                       int flags, bool kern)
1438 {
1439         struct sock *sk = sock->sk, *nsk;
1440         DECLARE_WAITQUEUE(wait, current);
1441         struct smc_sock *lsmc;
1442         long timeo;
1443         int rc = 0;
1444
1445         lsmc = smc_sk(sk);
1446         sock_hold(sk); /* sock_put below */
1447         lock_sock(sk);
1448
1449         if (lsmc->sk.sk_state != SMC_LISTEN) {
1450                 rc = -EINVAL;
1451                 release_sock(sk);
1452                 goto out;
1453         }
1454
1455         /* Wait for an incoming connection */
1456         timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK);
1457         add_wait_queue_exclusive(sk_sleep(sk), &wait);
1458         while (!(nsk = smc_accept_dequeue(sk, new_sock))) {
1459                 set_current_state(TASK_INTERRUPTIBLE);
1460                 if (!timeo) {
1461                         rc = -EAGAIN;
1462                         break;
1463                 }
1464                 release_sock(sk);
1465                 timeo = schedule_timeout(timeo);
1466                 /* wakeup by sk_data_ready in smc_listen_work() */
1467                 sched_annotate_sleep();
1468                 lock_sock(sk);
1469                 if (signal_pending(current)) {
1470                         rc = sock_intr_errno(timeo);
1471                         break;
1472                 }
1473         }
1474         set_current_state(TASK_RUNNING);
1475         remove_wait_queue(sk_sleep(sk), &wait);
1476
1477         if (!rc)
1478                 rc = sock_error(nsk);
1479         release_sock(sk);
1480         if (rc)
1481                 goto out;
1482
1483         if (lsmc->sockopt_defer_accept && !(flags & O_NONBLOCK)) {
1484                 /* wait till data arrives on the socket */
1485                 timeo = msecs_to_jiffies(lsmc->sockopt_defer_accept *
1486                                                                 MSEC_PER_SEC);
1487                 if (smc_sk(nsk)->use_fallback) {
1488                         struct sock *clcsk = smc_sk(nsk)->clcsock->sk;
1489
1490                         lock_sock(clcsk);
1491                         if (skb_queue_empty(&clcsk->sk_receive_queue))
1492                                 sk_wait_data(clcsk, &timeo, NULL);
1493                         release_sock(clcsk);
1494                 } else if (!atomic_read(&smc_sk(nsk)->conn.bytes_to_rcv)) {
1495                         lock_sock(nsk);
1496                         smc_rx_wait(smc_sk(nsk), &timeo, smc_rx_data_available);
1497                         release_sock(nsk);
1498                 }
1499         }
1500
1501 out:
1502         sock_put(sk); /* sock_hold above */
1503         return rc;
1504 }
1505
1506 static int smc_getname(struct socket *sock, struct sockaddr *addr,
1507                        int peer)
1508 {
1509         struct smc_sock *smc;
1510
1511         if (peer && (sock->sk->sk_state != SMC_ACTIVE) &&
1512             (sock->sk->sk_state != SMC_APPCLOSEWAIT1))
1513                 return -ENOTCONN;
1514
1515         smc = smc_sk(sock->sk);
1516
1517         return smc->clcsock->ops->getname(smc->clcsock, addr, peer);
1518 }
1519
1520 static int smc_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
1521 {
1522         struct sock *sk = sock->sk;
1523         struct smc_sock *smc;
1524         int rc = -EPIPE;
1525
1526         smc = smc_sk(sk);
1527         lock_sock(sk);
1528         if ((sk->sk_state != SMC_ACTIVE) &&
1529             (sk->sk_state != SMC_APPCLOSEWAIT1) &&
1530             (sk->sk_state != SMC_INIT))
1531                 goto out;
1532
1533         if (msg->msg_flags & MSG_FASTOPEN) {
1534                 if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
1535                         smc_switch_to_fallback(smc);
1536                         smc->fallback_rsn = SMC_CLC_DECL_OPTUNSUPP;
1537                 } else {
1538                         rc = -EINVAL;
1539                         goto out;
1540                 }
1541         }
1542
1543         if (smc->use_fallback)
1544                 rc = smc->clcsock->ops->sendmsg(smc->clcsock, msg, len);
1545         else
1546                 rc = smc_tx_sendmsg(smc, msg, len);
1547 out:
1548         release_sock(sk);
1549         return rc;
1550 }
1551
1552 static int smc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
1553                        int flags)
1554 {
1555         struct sock *sk = sock->sk;
1556         struct smc_sock *smc;
1557         int rc = -ENOTCONN;
1558
1559         smc = smc_sk(sk);
1560         lock_sock(sk);
1561         if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
1562                 /* socket was connected before, no more data to read */
1563                 rc = 0;
1564                 goto out;
1565         }
1566         if ((sk->sk_state == SMC_INIT) ||
1567             (sk->sk_state == SMC_LISTEN) ||
1568             (sk->sk_state == SMC_CLOSED))
1569                 goto out;
1570
1571         if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
1572                 rc = 0;
1573                 goto out;
1574         }
1575
1576         if (smc->use_fallback) {
1577                 rc = smc->clcsock->ops->recvmsg(smc->clcsock, msg, len, flags);
1578         } else {
1579                 msg->msg_namelen = 0;
1580                 rc = smc_rx_recvmsg(smc, msg, NULL, len, flags);
1581         }
1582
1583 out:
1584         release_sock(sk);
1585         return rc;
1586 }
1587
1588 static __poll_t smc_accept_poll(struct sock *parent)
1589 {
1590         struct smc_sock *isk = smc_sk(parent);
1591         __poll_t mask = 0;
1592
1593         spin_lock(&isk->accept_q_lock);
1594         if (!list_empty(&isk->accept_q))
1595                 mask = EPOLLIN | EPOLLRDNORM;
1596         spin_unlock(&isk->accept_q_lock);
1597
1598         return mask;
1599 }
1600
1601 static __poll_t smc_poll(struct file *file, struct socket *sock,
1602                              poll_table *wait)
1603 {
1604         struct sock *sk = sock->sk;
1605         struct smc_sock *smc;
1606         __poll_t mask = 0;
1607
1608         if (!sk)
1609                 return EPOLLNVAL;
1610
1611         smc = smc_sk(sock->sk);
1612         if (smc->use_fallback) {
1613                 /* delegate to CLC child sock */
1614                 mask = smc->clcsock->ops->poll(file, smc->clcsock, wait);
1615                 sk->sk_err = smc->clcsock->sk->sk_err;
1616         } else {
1617                 if (sk->sk_state != SMC_CLOSED)
1618                         sock_poll_wait(file, sock, wait);
1619                 if (sk->sk_err)
1620                         mask |= EPOLLERR;
1621                 if ((sk->sk_shutdown == SHUTDOWN_MASK) ||
1622                     (sk->sk_state == SMC_CLOSED))
1623                         mask |= EPOLLHUP;
1624                 if (sk->sk_state == SMC_LISTEN) {
1625                         /* woken up by sk_data_ready in smc_listen_work() */
1626                         mask |= smc_accept_poll(sk);
1627                 } else if (smc->use_fallback) { /* as result of connect_work()*/
1628                         mask |= smc->clcsock->ops->poll(file, smc->clcsock,
1629                                                            wait);
1630                         sk->sk_err = smc->clcsock->sk->sk_err;
1631                 } else {
1632                         if ((sk->sk_state != SMC_INIT &&
1633                              atomic_read(&smc->conn.sndbuf_space)) ||
1634                             sk->sk_shutdown & SEND_SHUTDOWN) {
1635                                 mask |= EPOLLOUT | EPOLLWRNORM;
1636                         } else {
1637                                 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1638                                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1639                         }
1640                         if (atomic_read(&smc->conn.bytes_to_rcv))
1641                                 mask |= EPOLLIN | EPOLLRDNORM;
1642                         if (sk->sk_shutdown & RCV_SHUTDOWN)
1643                                 mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
1644                         if (sk->sk_state == SMC_APPCLOSEWAIT1)
1645                                 mask |= EPOLLIN;
1646                         if (smc->conn.urg_state == SMC_URG_VALID)
1647                                 mask |= EPOLLPRI;
1648                 }
1649         }
1650
1651         return mask;
1652 }
1653
1654 static int smc_shutdown(struct socket *sock, int how)
1655 {
1656         struct sock *sk = sock->sk;
1657         struct smc_sock *smc;
1658         int rc = -EINVAL;
1659         int rc1 = 0;
1660
1661         smc = smc_sk(sk);
1662
1663         if ((how < SHUT_RD) || (how > SHUT_RDWR))
1664                 return rc;
1665
1666         lock_sock(sk);
1667
1668         rc = -ENOTCONN;
1669         if ((sk->sk_state != SMC_ACTIVE) &&
1670             (sk->sk_state != SMC_PEERCLOSEWAIT1) &&
1671             (sk->sk_state != SMC_PEERCLOSEWAIT2) &&
1672             (sk->sk_state != SMC_APPCLOSEWAIT1) &&
1673             (sk->sk_state != SMC_APPCLOSEWAIT2) &&
1674             (sk->sk_state != SMC_APPFINCLOSEWAIT))
1675                 goto out;
1676         if (smc->use_fallback) {
1677                 rc = kernel_sock_shutdown(smc->clcsock, how);
1678                 sk->sk_shutdown = smc->clcsock->sk->sk_shutdown;
1679                 if (sk->sk_shutdown == SHUTDOWN_MASK)
1680                         sk->sk_state = SMC_CLOSED;
1681                 goto out;
1682         }
1683         switch (how) {
1684         case SHUT_RDWR:         /* shutdown in both directions */
1685                 rc = smc_close_active(smc);
1686                 break;
1687         case SHUT_WR:
1688                 rc = smc_close_shutdown_write(smc);
1689                 break;
1690         case SHUT_RD:
1691                 rc = 0;
1692                 /* nothing more to do because peer is not involved */
1693                 break;
1694         }
1695         if (smc->clcsock)
1696                 rc1 = kernel_sock_shutdown(smc->clcsock, how);
1697         /* map sock_shutdown_cmd constants to sk_shutdown value range */
1698         sk->sk_shutdown |= how + 1;
1699
1700 out:
1701         release_sock(sk);
1702         return rc ? rc : rc1;
1703 }
1704
1705 static int smc_setsockopt(struct socket *sock, int level, int optname,
1706                           char __user *optval, unsigned int optlen)
1707 {
1708         struct sock *sk = sock->sk;
1709         struct smc_sock *smc;
1710         int val, rc;
1711
1712         smc = smc_sk(sk);
1713
1714         /* generic setsockopts reaching us here always apply to the
1715          * CLC socket
1716          */
1717         rc = smc->clcsock->ops->setsockopt(smc->clcsock, level, optname,
1718                                            optval, optlen);
1719         if (smc->clcsock->sk->sk_err) {
1720                 sk->sk_err = smc->clcsock->sk->sk_err;
1721                 sk->sk_error_report(sk);
1722         }
1723         if (rc)
1724                 return rc;
1725
1726         if (optlen < sizeof(int))
1727                 return -EINVAL;
1728         if (get_user(val, (int __user *)optval))
1729                 return -EFAULT;
1730
1731         lock_sock(sk);
1732         switch (optname) {
1733         case TCP_ULP:
1734         case TCP_FASTOPEN:
1735         case TCP_FASTOPEN_CONNECT:
1736         case TCP_FASTOPEN_KEY:
1737         case TCP_FASTOPEN_NO_COOKIE:
1738                 /* option not supported by SMC */
1739                 if (sk->sk_state == SMC_INIT) {
1740                         smc_switch_to_fallback(smc);
1741                         smc->fallback_rsn = SMC_CLC_DECL_OPTUNSUPP;
1742                 } else {
1743                         if (!smc->use_fallback)
1744                                 rc = -EINVAL;
1745                 }
1746                 break;
1747         case TCP_NODELAY:
1748                 if (sk->sk_state != SMC_INIT &&
1749                     sk->sk_state != SMC_LISTEN &&
1750                     sk->sk_state != SMC_CLOSED) {
1751                         if (val && !smc->use_fallback)
1752                                 mod_delayed_work(system_wq, &smc->conn.tx_work,
1753                                                  0);
1754                 }
1755                 break;
1756         case TCP_CORK:
1757                 if (sk->sk_state != SMC_INIT &&
1758                     sk->sk_state != SMC_LISTEN &&
1759                     sk->sk_state != SMC_CLOSED) {
1760                         if (!val && !smc->use_fallback)
1761                                 mod_delayed_work(system_wq, &smc->conn.tx_work,
1762                                                  0);
1763                 }
1764                 break;
1765         case TCP_DEFER_ACCEPT:
1766                 smc->sockopt_defer_accept = val;
1767                 break;
1768         default:
1769                 break;
1770         }
1771         release_sock(sk);
1772
1773         return rc;
1774 }
1775
1776 static int smc_getsockopt(struct socket *sock, int level, int optname,
1777                           char __user *optval, int __user *optlen)
1778 {
1779         struct smc_sock *smc;
1780
1781         smc = smc_sk(sock->sk);
1782         /* socket options apply to the CLC socket */
1783         return smc->clcsock->ops->getsockopt(smc->clcsock, level, optname,
1784                                              optval, optlen);
1785 }
1786
1787 static int smc_ioctl(struct socket *sock, unsigned int cmd,
1788                      unsigned long arg)
1789 {
1790         union smc_host_cursor cons, urg;
1791         struct smc_connection *conn;
1792         struct smc_sock *smc;
1793         int answ;
1794
1795         smc = smc_sk(sock->sk);
1796         conn = &smc->conn;
1797         lock_sock(&smc->sk);
1798         if (smc->use_fallback) {
1799                 if (!smc->clcsock) {
1800                         release_sock(&smc->sk);
1801                         return -EBADF;
1802                 }
1803                 answ = smc->clcsock->ops->ioctl(smc->clcsock, cmd, arg);
1804                 release_sock(&smc->sk);
1805                 return answ;
1806         }
1807         switch (cmd) {
1808         case SIOCINQ: /* same as FIONREAD */
1809                 if (smc->sk.sk_state == SMC_LISTEN) {
1810                         release_sock(&smc->sk);
1811                         return -EINVAL;
1812                 }
1813                 if (smc->sk.sk_state == SMC_INIT ||
1814                     smc->sk.sk_state == SMC_CLOSED)
1815                         answ = 0;
1816                 else
1817                         answ = atomic_read(&smc->conn.bytes_to_rcv);
1818                 break;
1819         case SIOCOUTQ:
1820                 /* output queue size (not send + not acked) */
1821                 if (smc->sk.sk_state == SMC_LISTEN) {
1822                         release_sock(&smc->sk);
1823                         return -EINVAL;
1824                 }
1825                 if (smc->sk.sk_state == SMC_INIT ||
1826                     smc->sk.sk_state == SMC_CLOSED)
1827                         answ = 0;
1828                 else
1829                         answ = smc->conn.sndbuf_desc->len -
1830                                         atomic_read(&smc->conn.sndbuf_space);
1831                 break;
1832         case SIOCOUTQNSD:
1833                 /* output queue size (not send only) */
1834                 if (smc->sk.sk_state == SMC_LISTEN) {
1835                         release_sock(&smc->sk);
1836                         return -EINVAL;
1837                 }
1838                 if (smc->sk.sk_state == SMC_INIT ||
1839                     smc->sk.sk_state == SMC_CLOSED)
1840                         answ = 0;
1841                 else
1842                         answ = smc_tx_prepared_sends(&smc->conn);
1843                 break;
1844         case SIOCATMARK:
1845                 if (smc->sk.sk_state == SMC_LISTEN) {
1846                         release_sock(&smc->sk);
1847                         return -EINVAL;
1848                 }
1849                 if (smc->sk.sk_state == SMC_INIT ||
1850                     smc->sk.sk_state == SMC_CLOSED) {
1851                         answ = 0;
1852                 } else {
1853                         smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn);
1854                         smc_curs_copy(&urg, &conn->urg_curs, conn);
1855                         answ = smc_curs_diff(conn->rmb_desc->len,
1856                                              &cons, &urg) == 1;
1857                 }
1858                 break;
1859         default:
1860                 release_sock(&smc->sk);
1861                 return -ENOIOCTLCMD;
1862         }
1863         release_sock(&smc->sk);
1864
1865         return put_user(answ, (int __user *)arg);
1866 }
1867
1868 static ssize_t smc_sendpage(struct socket *sock, struct page *page,
1869                             int offset, size_t size, int flags)
1870 {
1871         struct sock *sk = sock->sk;
1872         struct smc_sock *smc;
1873         int rc = -EPIPE;
1874
1875         smc = smc_sk(sk);
1876         lock_sock(sk);
1877         if (sk->sk_state != SMC_ACTIVE) {
1878                 release_sock(sk);
1879                 goto out;
1880         }
1881         release_sock(sk);
1882         if (smc->use_fallback)
1883                 rc = kernel_sendpage(smc->clcsock, page, offset,
1884                                      size, flags);
1885         else
1886                 rc = sock_no_sendpage(sock, page, offset, size, flags);
1887
1888 out:
1889         return rc;
1890 }
1891
1892 /* Map the affected portions of the rmbe into an spd, note the number of bytes
1893  * to splice in conn->splice_pending, and press 'go'. Delays consumer cursor
1894  * updates till whenever a respective page has been fully processed.
1895  * Note that subsequent recv() calls have to wait till all splice() processing
1896  * completed.
1897  */
1898 static ssize_t smc_splice_read(struct socket *sock, loff_t *ppos,
1899                                struct pipe_inode_info *pipe, size_t len,
1900                                unsigned int flags)
1901 {
1902         struct sock *sk = sock->sk;
1903         struct smc_sock *smc;
1904         int rc = -ENOTCONN;
1905
1906         smc = smc_sk(sk);
1907         lock_sock(sk);
1908         if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
1909                 /* socket was connected before, no more data to read */
1910                 rc = 0;
1911                 goto out;
1912         }
1913         if (sk->sk_state == SMC_INIT ||
1914             sk->sk_state == SMC_LISTEN ||
1915             sk->sk_state == SMC_CLOSED)
1916                 goto out;
1917
1918         if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
1919                 rc = 0;
1920                 goto out;
1921         }
1922
1923         if (smc->use_fallback) {
1924                 rc = smc->clcsock->ops->splice_read(smc->clcsock, ppos,
1925                                                     pipe, len, flags);
1926         } else {
1927                 if (*ppos) {
1928                         rc = -ESPIPE;
1929                         goto out;
1930                 }
1931                 if (flags & SPLICE_F_NONBLOCK)
1932                         flags = MSG_DONTWAIT;
1933                 else
1934                         flags = 0;
1935                 rc = smc_rx_recvmsg(smc, NULL, pipe, len, flags);
1936         }
1937 out:
1938         release_sock(sk);
1939
1940         return rc;
1941 }
1942
1943 /* must look like tcp */
1944 static const struct proto_ops smc_sock_ops = {
1945         .family         = PF_SMC,
1946         .owner          = THIS_MODULE,
1947         .release        = smc_release,
1948         .bind           = smc_bind,
1949         .connect        = smc_connect,
1950         .socketpair     = sock_no_socketpair,
1951         .accept         = smc_accept,
1952         .getname        = smc_getname,
1953         .poll           = smc_poll,
1954         .ioctl          = smc_ioctl,
1955         .listen         = smc_listen,
1956         .shutdown       = smc_shutdown,
1957         .setsockopt     = smc_setsockopt,
1958         .getsockopt     = smc_getsockopt,
1959         .sendmsg        = smc_sendmsg,
1960         .recvmsg        = smc_recvmsg,
1961         .mmap           = sock_no_mmap,
1962         .sendpage       = smc_sendpage,
1963         .splice_read    = smc_splice_read,
1964 };
1965
1966 static int smc_create(struct net *net, struct socket *sock, int protocol,
1967                       int kern)
1968 {
1969         int family = (protocol == SMCPROTO_SMC6) ? PF_INET6 : PF_INET;
1970         struct smc_sock *smc;
1971         struct sock *sk;
1972         int rc;
1973
1974         rc = -ESOCKTNOSUPPORT;
1975         if (sock->type != SOCK_STREAM)
1976                 goto out;
1977
1978         rc = -EPROTONOSUPPORT;
1979         if (protocol != SMCPROTO_SMC && protocol != SMCPROTO_SMC6)
1980                 goto out;
1981
1982         rc = -ENOBUFS;
1983         sock->ops = &smc_sock_ops;
1984         sk = smc_sock_alloc(net, sock, protocol);
1985         if (!sk)
1986                 goto out;
1987
1988         /* create internal TCP socket for CLC handshake and fallback */
1989         smc = smc_sk(sk);
1990         smc->use_fallback = false; /* assume rdma capability first */
1991         smc->fallback_rsn = 0;
1992         rc = sock_create_kern(net, family, SOCK_STREAM, IPPROTO_TCP,
1993                               &smc->clcsock);
1994         if (rc) {
1995                 sk_common_release(sk);
1996                 goto out;
1997         }
1998         smc->sk.sk_sndbuf = max(smc->clcsock->sk->sk_sndbuf, SMC_BUF_MIN_SIZE);
1999         smc->sk.sk_rcvbuf = max(smc->clcsock->sk->sk_rcvbuf, SMC_BUF_MIN_SIZE);
2000
2001 out:
2002         return rc;
2003 }
2004
2005 static const struct net_proto_family smc_sock_family_ops = {
2006         .family = PF_SMC,
2007         .owner  = THIS_MODULE,
2008         .create = smc_create,
2009 };
2010
2011 unsigned int smc_net_id;
2012
2013 static __net_init int smc_net_init(struct net *net)
2014 {
2015         return smc_pnet_net_init(net);
2016 }
2017
2018 static void __net_exit smc_net_exit(struct net *net)
2019 {
2020         smc_pnet_net_exit(net);
2021 }
2022
2023 static struct pernet_operations smc_net_ops = {
2024         .init = smc_net_init,
2025         .exit = smc_net_exit,
2026         .id   = &smc_net_id,
2027         .size = sizeof(struct smc_net),
2028 };
2029
2030 static int __init smc_init(void)
2031 {
2032         int rc;
2033
2034         rc = register_pernet_subsys(&smc_net_ops);
2035         if (rc)
2036                 return rc;
2037
2038         rc = smc_pnet_init();
2039         if (rc)
2040                 goto out_pernet_subsys;
2041
2042         rc = smc_core_init();
2043         if (rc) {
2044                 pr_err("%s: smc_core_init fails with %d\n", __func__, rc);
2045                 goto out_pnet;
2046         }
2047
2048         rc = smc_llc_init();
2049         if (rc) {
2050                 pr_err("%s: smc_llc_init fails with %d\n", __func__, rc);
2051                 goto out_core;
2052         }
2053
2054         rc = smc_cdc_init();
2055         if (rc) {
2056                 pr_err("%s: smc_cdc_init fails with %d\n", __func__, rc);
2057                 goto out_core;
2058         }
2059
2060         rc = proto_register(&smc_proto, 1);
2061         if (rc) {
2062                 pr_err("%s: proto_register(v4) fails with %d\n", __func__, rc);
2063                 goto out_core;
2064         }
2065
2066         rc = proto_register(&smc_proto6, 1);
2067         if (rc) {
2068                 pr_err("%s: proto_register(v6) fails with %d\n", __func__, rc);
2069                 goto out_proto;
2070         }
2071
2072         rc = sock_register(&smc_sock_family_ops);
2073         if (rc) {
2074                 pr_err("%s: sock_register fails with %d\n", __func__, rc);
2075                 goto out_proto6;
2076         }
2077         INIT_HLIST_HEAD(&smc_v4_hashinfo.ht);
2078         INIT_HLIST_HEAD(&smc_v6_hashinfo.ht);
2079
2080         rc = smc_ib_register_client();
2081         if (rc) {
2082                 pr_err("%s: ib_register fails with %d\n", __func__, rc);
2083                 goto out_sock;
2084         }
2085
2086         static_branch_enable(&tcp_have_smc);
2087         return 0;
2088
2089 out_sock:
2090         sock_unregister(PF_SMC);
2091 out_proto6:
2092         proto_unregister(&smc_proto6);
2093 out_proto:
2094         proto_unregister(&smc_proto);
2095 out_core:
2096         smc_core_exit();
2097 out_pnet:
2098         smc_pnet_exit();
2099 out_pernet_subsys:
2100         unregister_pernet_subsys(&smc_net_ops);
2101
2102         return rc;
2103 }
2104
2105 static void __exit smc_exit(void)
2106 {
2107         static_branch_disable(&tcp_have_smc);
2108         sock_unregister(PF_SMC);
2109         smc_core_exit();
2110         smc_ib_unregister_client();
2111         proto_unregister(&smc_proto6);
2112         proto_unregister(&smc_proto);
2113         smc_pnet_exit();
2114         unregister_pernet_subsys(&smc_net_ops);
2115         rcu_barrier();
2116 }
2117
2118 module_init(smc_init);
2119 module_exit(smc_exit);
2120
2121 MODULE_AUTHOR("Ursula Braun <ubraun@linux.vnet.ibm.com>");
2122 MODULE_DESCRIPTION("smc socket address family");
2123 MODULE_LICENSE("GPL");
2124 MODULE_ALIAS_NETPROTO(PF_SMC);