drivers/net/ethernet/sfc/efx.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /****************************************************************************
   3  * Driver for Solarflare network controllers and boards
   4  * Copyright 2005-2006 Fen Systems Ltd.
   5  * Copyright 2005-2013 Solarflare Communications Inc.
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/pci.h>
  10 #include <linux/netdevice.h>
  11 #include <linux/etherdevice.h>
  12 #include <linux/delay.h>
  13 #include <linux/notifier.h>
  14 #include <linux/ip.h>
  15 #include <linux/tcp.h>
  16 #include <linux/in.h>
  17 #include <linux/ethtool.h>
  18 #include <linux/topology.h>
  19 #include <linux/gfp.h>
  20 #include <linux/aer.h>
  21 #include <linux/interrupt.h>
  22 #include "net_driver.h"
  23 #include <net/gre.h>
  24 #include <net/udp_tunnel.h>
  25 #include "efx.h"
  26 #include "efx_common.h"
  27 #include "efx_channels.h"
  28 #include "rx_common.h"
  29 #include "tx_common.h"
  30 #include "nic.h"
  31 #include "io.h"
  32 #include "selftest.h"
  33 #include "sriov.h"
  34
  35 #include "mcdi.h"
  36 #include "mcdi_pcol.h"
  37 #include "workarounds.h"
  38
  39 /**************************************************************************
  40  *
  41  * Type name strings
  42  *
  43  **************************************************************************
  44  */
  45
  46 /* UDP tunnel type names */
  47 static const char *const efx_udp_tunnel_type_names[] = {
  48         [TUNNEL_ENCAP_UDP_PORT_ENTRY_VXLAN] = "vxlan",
  49         [TUNNEL_ENCAP_UDP_PORT_ENTRY_GENEVE] = "geneve",
  50 };
  51
  52 void efx_get_udp_tunnel_type_name(u16 type, char *buf, size_t buflen)
  53 {
  54         if (type < ARRAY_SIZE(efx_udp_tunnel_type_names) &&
  55             efx_udp_tunnel_type_names[type] != NULL)
  56                 snprintf(buf, buflen, "%s", efx_udp_tunnel_type_names[type]);
  57         else
  58                 snprintf(buf, buflen, "type %d", type);
  59 }
  60
  61 /**************************************************************************
  62  *
  63  * Configurable values
  64  *
  65  *************************************************************************/
  66
  67 /*
  68  * Use separate channels for TX and RX events
  69  *
  70  * Set this to 1 to use separate channels for TX and RX. It allows us
  71  * to control interrupt affinity separately for TX and RX.
  72  *
  73  * This is only used in MSI-X interrupt mode
  74  */
  75 bool efx_separate_tx_channels;
  76 module_param(efx_separate_tx_channels, bool, 0444);
  77 MODULE_PARM_DESC(efx_separate_tx_channels,
  78                  "Use separate channels for TX and RX");
  79
  80 /* This is the weight assigned to each of the (per-channel) virtual
  81  * NAPI devices.
  82  */
  83 static int napi_weight = 64;
  84
  85 /* Initial interrupt moderation settings.  They can be modified after
  86  * module load with ethtool.
  87  *
  88  * The default for RX should strike a balance between increasing the
  89  * round-trip latency and reducing overhead.
  90  */
  91 static unsigned int rx_irq_mod_usec = 60;
  92
  93 /* Initial interrupt moderation settings.  They can be modified after
  94  * module load with ethtool.
  95  *
  96  * This default is chosen to ensure that a 10G link does not go idle
  97  * while a TX queue is stopped after it has become full.  A queue is
  98  * restarted when it drops below half full.  The time this takes (assuming
  99  * worst case 3 descriptors per packet and 1024 descriptors) is
 100  *   512 / 3 * 1.2 = 205 usec.
 101  */
 102 static unsigned int tx_irq_mod_usec = 150;
 103
 104 /* This is the first interrupt mode to try out of:
 105  * 0 => MSI-X
 106  * 1 => MSI
 107  * 2 => legacy
 108  */
 109 static unsigned int interrupt_mode;
 110
 111 /* This is the requested number of CPUs to use for Receive-Side Scaling (RSS),
 112  * i.e. the number of CPUs among which we may distribute simultaneous
 113  * interrupt handling.
 114  *
 115  * Cards without MSI-X will only target one CPU via legacy or MSI interrupt.
 116  * The default (0) means to assign an interrupt to each core.
 117  */
 118 static unsigned int rss_cpus;
 119 module_param(rss_cpus, uint, 0444);
 120 MODULE_PARM_DESC(rss_cpus, "Number of CPUs to use for Receive-Side Scaling");
 121
 122 static bool phy_flash_cfg;
 123 module_param(phy_flash_cfg, bool, 0644);
 124 MODULE_PARM_DESC(phy_flash_cfg, "Set PHYs into reflash mode initially");
 125
 126 static unsigned irq_adapt_low_thresh = 8000;
 127 module_param(irq_adapt_low_thresh, uint, 0644);
 128 MODULE_PARM_DESC(irq_adapt_low_thresh,
 129                  "Threshold score for reducing IRQ moderation");
 130
 131 static unsigned irq_adapt_high_thresh = 16000;
 132 module_param(irq_adapt_high_thresh, uint, 0644);
 133 MODULE_PARM_DESC(irq_adapt_high_thresh,
 134                  "Threshold score for increasing IRQ moderation");
 135
 136 static unsigned debug = (NETIF_MSG_DRV | NETIF_MSG_PROBE |
 137                          NETIF_MSG_LINK | NETIF_MSG_IFDOWN |
 138                          NETIF_MSG_IFUP | NETIF_MSG_RX_ERR |
 139                          NETIF_MSG_TX_ERR | NETIF_MSG_HW);
 140 module_param(debug, uint, 0);
 141 MODULE_PARM_DESC(debug, "Bitmapped debugging message enable value");
 142
 143 /**************************************************************************
 144  *
 145  * Utility functions and prototypes
 146  *
 147  *************************************************************************/
 148
 149 static const struct efx_channel_type efx_default_channel_type;
 150 static void efx_remove_port(struct efx_nic *efx);
 151 static int efx_xdp_setup_prog(struct efx_nic *efx, struct bpf_prog *prog);
 152 static int efx_xdp(struct net_device *dev, struct netdev_bpf *xdp);
 153 static int efx_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **xdpfs,
 154                         u32 flags);
 155
 156 #define EFX_ASSERT_RESET_SERIALISED(efx)                \
 157         do {                                            \
 158                 if ((efx->state == STATE_READY) ||      \
 159                     (efx->state == STATE_RECOVERY) ||   \
 160                     (efx->state == STATE_DISABLED))     \
 161                         ASSERT_RTNL();                  \
 162         } while (0)
 163
 164 /**************************************************************************
 165  *
 166  * Event queue processing
 167  *
 168  *************************************************************************/
 169
 170 /* Process channel's event queue
 171  *
 172  * This function is responsible for processing the event queue of a
 173  * single channel.  The caller must guarantee that this function will
 174  * never be concurrently called more than once on the same channel,
 175  * though different channels may be being processed concurrently.
 176  */
 177 static int efx_process_channel(struct efx_channel *channel, int budget)
 178 {
 179         struct efx_tx_queue *tx_queue;
 180         struct list_head rx_list;
 181         int spent;
 182
 183         if (unlikely(!channel->enabled))
 184                 return 0;
 185
 186         /* Prepare the batch receive list */
 187         EFX_WARN_ON_PARANOID(channel->rx_list != NULL);
 188         INIT_LIST_HEAD(&rx_list);
 189         channel->rx_list = &rx_list;
 190
 191         efx_for_each_channel_tx_queue(tx_queue, channel) {
 192                 tx_queue->pkts_compl = 0;
 193                 tx_queue->bytes_compl = 0;
 194         }
 195
 196         spent = efx_nic_process_eventq(channel, budget);
 197         if (spent && efx_channel_has_rx_queue(channel)) {
 198                 struct efx_rx_queue *rx_queue =
 199                         efx_channel_get_rx_queue(channel);
 200
 201                 efx_rx_flush_packet(channel);
 202                 efx_fast_push_rx_descriptors(rx_queue, true);
 203         }
 204
 205         /* Update BQL */
 206         efx_for_each_channel_tx_queue(tx_queue, channel) {
 207                 if (tx_queue->bytes_compl) {
 208                         netdev_tx_completed_queue(tx_queue->core_txq,
 209                                 tx_queue->pkts_compl, tx_queue->bytes_compl);
 210                 }
 211         }
 212
 213         /* Receive any packets we queued up */
 214         netif_receive_skb_list(channel->rx_list);
 215         channel->rx_list = NULL;
 216
 217         return spent;
 218 }
 219
 220 /* NAPI poll handler
 221  *
 222  * NAPI guarantees serialisation of polls of the same device, which
 223  * provides the guarantee required by efx_process_channel().
 224  */
 225 static void efx_update_irq_mod(struct efx_nic *efx, struct efx_channel *channel)
 226 {
 227         int step = efx->irq_mod_step_us;
 228
 229         if (channel->irq_mod_score < irq_adapt_low_thresh) {
 230                 if (channel->irq_moderation_us > step) {
 231                         channel->irq_moderation_us -= step;
 232                         efx->type->push_irq_moderation(channel);
 233                 }
 234         } else if (channel->irq_mod_score > irq_adapt_high_thresh) {
 235                 if (channel->irq_moderation_us <
 236                     efx->irq_rx_moderation_us) {
 237                         channel->irq_moderation_us += step;
 238                         efx->type->push_irq_moderation(channel);
 239                 }
 240         }
 241
 242         channel->irq_count = 0;
 243         channel->irq_mod_score = 0;
 244 }
 245
 246 static int efx_poll(struct napi_struct *napi, int budget)
 247 {
 248         struct efx_channel *channel =
 249                 container_of(napi, struct efx_channel, napi_str);
 250         struct efx_nic *efx = channel->efx;
 251         int spent;
 252
 253         netif_vdbg(efx, intr, efx->net_dev,
 254                    "channel %d NAPI poll executing on CPU %d\n",
 255                    channel->channel, raw_smp_processor_id());
 256
 257         spent = efx_process_channel(channel, budget);
 258
 259         xdp_do_flush_map();
 260
 261         if (spent < budget) {
 262                 if (efx_channel_has_rx_queue(channel) &&
 263                     efx->irq_rx_adaptive &&
 264                     unlikely(++channel->irq_count == 1000)) {
 265                         efx_update_irq_mod(efx, channel);
 266                 }
 267
 268 #ifdef CONFIG_RFS_ACCEL
 269                 /* Perhaps expire some ARFS filters */
 270                 mod_delayed_work(system_wq, &channel->filter_work, 0);
 271 #endif
 272
 273                 /* There is no race here; although napi_disable() will
 274                  * only wait for napi_complete(), this isn't a problem
 275                  * since efx_nic_eventq_read_ack() will have no effect if
 276                  * interrupts have already been disabled.
 277                  */
 278                 if (napi_complete_done(napi, spent))
 279                         efx_nic_eventq_read_ack(channel);
 280         }
 281
 282         return spent;
 283 }
 284
 285 /* Create event queue
 286  * Event queue memory allocations are done only once.  If the channel
 287  * is reset, the memory buffer will be reused; this guards against
 288  * errors during channel reset and also simplifies interrupt handling.
 289  */
 290 int efx_probe_eventq(struct efx_channel *channel)
 291 {
 292         struct efx_nic *efx = channel->efx;
 293         unsigned long entries;
 294
 295         netif_dbg(efx, probe, efx->net_dev,
 296                   "chan %d create event queue\n", channel->channel);
 297
 298         /* Build an event queue with room for one event per tx and rx buffer,
 299          * plus some extra for link state events and MCDI completions. */
 300         entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128);
 301         EFX_WARN_ON_PARANOID(entries > EFX_MAX_EVQ_SIZE);
 302         channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1;
 303
 304         return efx_nic_probe_eventq(channel);
 305 }
 306
 307 /* Prepare channel's event queue */
 308 int efx_init_eventq(struct efx_channel *channel)
 309 {
 310         struct efx_nic *efx = channel->efx;
 311         int rc;
 312
 313         EFX_WARN_ON_PARANOID(channel->eventq_init);
 314
 315         netif_dbg(efx, drv, efx->net_dev,
 316                   "chan %d init event queue\n", channel->channel);
 317
 318         rc = efx_nic_init_eventq(channel);
 319         if (rc == 0) {
 320                 efx->type->push_irq_moderation(channel);
 321                 channel->eventq_read_ptr = 0;
 322                 channel->eventq_init = true;
 323         }
 324         return rc;
 325 }
 326
 327 /* Enable event queue processing and NAPI */
 328 void efx_start_eventq(struct efx_channel *channel)
 329 {
 330         netif_dbg(channel->efx, ifup, channel->efx->net_dev,
 331                   "chan %d start event queue\n", channel->channel);
 332
 333         /* Make sure the NAPI handler sees the enabled flag set */
 334         channel->enabled = true;
 335         smp_wmb();
 336
 337         napi_enable(&channel->napi_str);
 338         efx_nic_eventq_read_ack(channel);
 339 }
 340
 341 /* Disable event queue processing and NAPI */
 342 void efx_stop_eventq(struct efx_channel *channel)
 343 {
 344         if (!channel->enabled)
 345                 return;
 346
 347         napi_disable(&channel->napi_str);
 348         channel->enabled = false;
 349 }
 350
 351 void efx_fini_eventq(struct efx_channel *channel)
 352 {
 353         if (!channel->eventq_init)
 354                 return;
 355
 356         netif_dbg(channel->efx, drv, channel->efx->net_dev,
 357                   "chan %d fini event queue\n", channel->channel);
 358
 359         efx_nic_fini_eventq(channel);
 360         channel->eventq_init = false;
 361 }
 362
 363 void efx_remove_eventq(struct efx_channel *channel)
 364 {
 365         netif_dbg(channel->efx, drv, channel->efx->net_dev,
 366                   "chan %d remove event queue\n", channel->channel);
 367
 368         efx_nic_remove_eventq(channel);
 369 }
 370
 371 /**************************************************************************
 372  *
 373  * Channel handling
 374  *
 375  *************************************************************************/
 376
 377 /* Allocate and initialise a channel structure. */
 378 struct efx_channel *
 379 efx_alloc_channel(struct efx_nic *efx, int i, struct efx_channel *old_channel)
 380 {
 381         struct efx_channel *channel;
 382         struct efx_rx_queue *rx_queue;
 383         struct efx_tx_queue *tx_queue;
 384         int j;
 385
 386         channel = kzalloc(sizeof(*channel), GFP_KERNEL);
 387         if (!channel)
 388                 return NULL;
 389
 390         channel->efx = efx;
 391         channel->channel = i;
 392         channel->type = &efx_default_channel_type;
 393
 394         for (j = 0; j < EFX_TXQ_TYPES; j++) {
 395                 tx_queue = &channel->tx_queue[j];
 396                 tx_queue->efx = efx;
 397                 tx_queue->queue = i * EFX_TXQ_TYPES + j;
 398                 tx_queue->channel = channel;
 399         }
 400
 401 #ifdef CONFIG_RFS_ACCEL
 402         INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire);
 403 #endif
 404
 405         rx_queue = &channel->rx_queue;
 406         rx_queue->efx = efx;
 407         timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0);
 408
 409         return channel;
 410 }
 411
 412 /* Allocate and initialise a channel structure, copying parameters
 413  * (but not resources) from an old channel structure.
 414  */
 415 struct efx_channel *efx_copy_channel(const struct efx_channel *old_channel)
 416 {
 417         struct efx_channel *channel;
 418         struct efx_rx_queue *rx_queue;
 419         struct efx_tx_queue *tx_queue;
 420         int j;
 421
 422         channel = kmalloc(sizeof(*channel), GFP_KERNEL);
 423         if (!channel)
 424                 return NULL;
 425
 426         *channel = *old_channel;
 427
 428         channel->napi_dev = NULL;
 429         INIT_HLIST_NODE(&channel->napi_str.napi_hash_node);
 430         channel->napi_str.napi_id = 0;
 431         channel->napi_str.state = 0;
 432         memset(&channel->eventq, 0, sizeof(channel->eventq));
 433
 434         for (j = 0; j < EFX_TXQ_TYPES; j++) {
 435                 tx_queue = &channel->tx_queue[j];
 436                 if (tx_queue->channel)
 437                         tx_queue->channel = channel;
 438                 tx_queue->buffer = NULL;
 439                 memset(&tx_queue->txd, 0, sizeof(tx_queue->txd));
 440         }
 441
 442         rx_queue = &channel->rx_queue;
 443         rx_queue->buffer = NULL;
 444         memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd));
 445         timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0);
 446 #ifdef CONFIG_RFS_ACCEL
 447         INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire);
 448 #endif
 449
 450         return channel;
 451 }
 452
 453 static int efx_probe_channel(struct efx_channel *channel)
 454 {
 455         struct efx_tx_queue *tx_queue;
 456         struct efx_rx_queue *rx_queue;
 457         int rc;
 458
 459         netif_dbg(channel->efx, probe, channel->efx->net_dev,
 460                   "creating channel %d\n", channel->channel);
 461
 462         rc = channel->type->pre_probe(channel);
 463         if (rc)
 464                 goto fail;
 465
 466         rc = efx_probe_eventq(channel);
 467         if (rc)
 468                 goto fail;
 469
 470         efx_for_each_channel_tx_queue(tx_queue, channel) {
 471                 rc = efx_probe_tx_queue(tx_queue);
 472                 if (rc)
 473                         goto fail;
 474         }
 475
 476         efx_for_each_channel_rx_queue(rx_queue, channel) {
 477                 rc = efx_probe_rx_queue(rx_queue);
 478                 if (rc)
 479                         goto fail;
 480         }
 481
 482         channel->rx_list = NULL;
 483
 484         return 0;
 485
 486 fail:
 487         efx_remove_channel(channel);
 488         return rc;
 489 }
 490
 491 void efx_get_channel_name(struct efx_channel *channel, char *buf, size_t len)
 492 {
 493         struct efx_nic *efx = channel->efx;
 494         const char *type;
 495         int number;
 496
 497         number = channel->channel;
 498
 499         if (number >= efx->xdp_channel_offset &&
 500             !WARN_ON_ONCE(!efx->n_xdp_channels)) {
 501                 type = "-xdp";
 502                 number -= efx->xdp_channel_offset;
 503         } else if (efx->tx_channel_offset == 0) {
 504                 type = "";
 505         } else if (number < efx->tx_channel_offset) {
 506                 type = "-rx";
 507         } else {
 508                 type = "-tx";
 509                 number -= efx->tx_channel_offset;
 510         }
 511         snprintf(buf, len, "%s%s-%d", efx->name, type, number);
 512 }
 513
 514 void efx_set_channel_names(struct efx_nic *efx)
 515 {
 516         struct efx_channel *channel;
 517
 518         efx_for_each_channel(channel, efx)
 519                 channel->type->get_name(channel,
 520                                         efx->msi_context[channel->channel].name,
 521                                         sizeof(efx->msi_context[0].name));
 522 }
 523
 524 int efx_probe_channels(struct efx_nic *efx)
 525 {
 526         struct efx_channel *channel;
 527         int rc;
 528
 529         /* Restart special buffer allocation */
 530         efx->next_buffer_table = 0;
 531
 532         /* Probe channels in reverse, so that any 'extra' channels
 533          * use the start of the buffer table. This allows the traffic
 534          * channels to be resized without moving them or wasting the
 535          * entries before them.
 536          */
 537         efx_for_each_channel_rev(channel, efx) {
 538                 rc = efx_probe_channel(channel);
 539                 if (rc) {
 540                         netif_err(efx, probe, efx->net_dev,
 541                                   "failed to create channel %d\n",
 542                                   channel->channel);
 543                         goto fail;
 544                 }
 545         }
 546         efx_set_channel_names(efx);
 547
 548         return 0;
 549
 550 fail:
 551         efx_remove_channels(efx);
 552         return rc;
 553 }
 554
 555 void efx_remove_channel(struct efx_channel *channel)
 556 {
 557         struct efx_tx_queue *tx_queue;
 558         struct efx_rx_queue *rx_queue;
 559
 560         netif_dbg(channel->efx, drv, channel->efx->net_dev,
 561                   "destroy chan %d\n", channel->channel);
 562
 563         efx_for_each_channel_rx_queue(rx_queue, channel)
 564                 efx_remove_rx_queue(rx_queue);
 565         efx_for_each_possible_channel_tx_queue(tx_queue, channel)
 566                 efx_remove_tx_queue(tx_queue);
 567         efx_remove_eventq(channel);
 568         channel->type->post_remove(channel);
 569 }
 570
 571 void efx_remove_channels(struct efx_nic *efx)
 572 {
 573         struct efx_channel *channel;
 574
 575         efx_for_each_channel(channel, efx)
 576                 efx_remove_channel(channel);
 577
 578         kfree(efx->xdp_tx_queues);
 579 }
 580
 581 int efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries)
 582 {
 583         struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel;
 584         u32 old_rxq_entries, old_txq_entries;
 585         unsigned i, next_buffer_table = 0;
 586         int rc, rc2;
 587
 588         rc = efx_check_disabled(efx);
 589         if (rc)
 590                 return rc;
 591
 592         /* Not all channels should be reallocated. We must avoid
 593          * reallocating their buffer table entries.
 594          */
 595         efx_for_each_channel(channel, efx) {
 596                 struct efx_rx_queue *rx_queue;
 597                 struct efx_tx_queue *tx_queue;
 598
 599                 if (channel->type->copy)
 600                         continue;
 601                 next_buffer_table = max(next_buffer_table,
 602                                         channel->eventq.index +
 603                                         channel->eventq.entries);
 604                 efx_for_each_channel_rx_queue(rx_queue, channel)
 605                         next_buffer_table = max(next_buffer_table,
 606                                                 rx_queue->rxd.index +
 607                                                 rx_queue->rxd.entries);
 608                 efx_for_each_channel_tx_queue(tx_queue, channel)
 609                         next_buffer_table = max(next_buffer_table,
 610                                                 tx_queue->txd.index +
 611                                                 tx_queue->txd.entries);
 612         }
 613
 614         efx_device_detach_sync(efx);
 615         efx_stop_all(efx);
 616         efx_soft_disable_interrupts(efx);
 617
 618         /* Clone channels (where possible) */
 619         memset(other_channel, 0, sizeof(other_channel));
 620         for (i = 0; i < efx->n_channels; i++) {
 621                 channel = efx->channel[i];
 622                 if (channel->type->copy)
 623                         channel = channel->type->copy(channel);
 624                 if (!channel) {
 625                         rc = -ENOMEM;
 626                         goto out;
 627                 }
 628                 other_channel[i] = channel;
 629         }
 630
 631         /* Swap entry counts and channel pointers */
 632         old_rxq_entries = efx->rxq_entries;
 633         old_txq_entries = efx->txq_entries;
 634         efx->rxq_entries = rxq_entries;
 635         efx->txq_entries = txq_entries;
 636         for (i = 0; i < efx->n_channels; i++) {
 637                 channel = efx->channel[i];
 638                 efx->channel[i] = other_channel[i];
 639                 other_channel[i] = channel;
 640         }
 641
 642         /* Restart buffer table allocation */
 643         efx->next_buffer_table = next_buffer_table;
 644
 645         for (i = 0; i < efx->n_channels; i++) {
 646                 channel = efx->channel[i];
 647                 if (!channel->type->copy)
 648                         continue;
 649                 rc = efx_probe_channel(channel);
 650                 if (rc)
 651                         goto rollback;
 652                 efx_init_napi_channel(efx->channel[i]);
 653         }
 654
 655 out:
 656         /* Destroy unused channel structures */
 657         for (i = 0; i < efx->n_channels; i++) {
 658                 channel = other_channel[i];
 659                 if (channel && channel->type->copy) {
 660                         efx_fini_napi_channel(channel);
 661                         efx_remove_channel(channel);
 662                         kfree(channel);
 663                 }
 664         }
 665
 666         rc2 = efx_soft_enable_interrupts(efx);
 667         if (rc2) {
 668                 rc = rc ? rc : rc2;
 669                 netif_err(efx, drv, efx->net_dev,
 670                           "unable to restart interrupts on channel reallocation\n");
 671                 efx_schedule_reset(efx, RESET_TYPE_DISABLE);
 672         } else {
 673                 efx_start_all(efx);
 674                 efx_device_attach_if_not_resetting(efx);
 675         }
 676         return rc;
 677
 678 rollback:
 679         /* Swap back */
 680         efx->rxq_entries = old_rxq_entries;
 681         efx->txq_entries = old_txq_entries;
 682         for (i = 0; i < efx->n_channels; i++) {
 683                 channel = efx->channel[i];
 684                 efx->channel[i] = other_channel[i];
 685                 other_channel[i] = channel;
 686         }
 687         goto out;
 688 }
 689
 690 void efx_schedule_slow_fill(struct efx_rx_queue *rx_queue)
 691 {
 692         mod_timer(&rx_queue->slow_fill, jiffies + msecs_to_jiffies(10));
 693 }
 694
 695 bool efx_default_channel_want_txqs(struct efx_channel *channel)
 696 {
 697         return channel->channel - channel->efx->tx_channel_offset <
 698                 channel->efx->n_tx_channels;
 699 }
 700
 701 static const struct efx_channel_type efx_default_channel_type = {
 702         .pre_probe              = efx_channel_dummy_op_int,
 703         .post_remove            = efx_channel_dummy_op_void,
 704         .get_name               = efx_get_channel_name,
 705         .copy                   = efx_copy_channel,
 706         .want_txqs              = efx_default_channel_want_txqs,
 707         .keep_eventq            = false,
 708         .want_pio               = true,
 709 };
 710
 711 int efx_channel_dummy_op_int(struct efx_channel *channel)
 712 {
 713         return 0;
 714 }
 715
 716 void efx_channel_dummy_op_void(struct efx_channel *channel)
 717 {
 718 }
 719
 720 /**************************************************************************
 721  *
 722  * Port handling
 723  *
 724  **************************************************************************/
 725
 726 void efx_link_set_advertising(struct efx_nic *efx,
 727                               const unsigned long *advertising)
 728 {
 729         memcpy(efx->link_advertising, advertising,
 730                sizeof(__ETHTOOL_DECLARE_LINK_MODE_MASK()));
 731
 732         efx->link_advertising[0] |= ADVERTISED_Autoneg;
 733         if (advertising[0] & ADVERTISED_Pause)
 734                 efx->wanted_fc |= (EFX_FC_TX | EFX_FC_RX);
 735         else
 736                 efx->wanted_fc &= ~(EFX_FC_TX | EFX_FC_RX);
 737         if (advertising[0] & ADVERTISED_Asym_Pause)
 738                 efx->wanted_fc ^= EFX_FC_TX;
 739 }
 740
 741 /* Equivalent to efx_link_set_advertising with all-zeroes, except does not
 742  * force the Autoneg bit on.
 743  */
 744 void efx_link_clear_advertising(struct efx_nic *efx)
 745 {
 746         bitmap_zero(efx->link_advertising, __ETHTOOL_LINK_MODE_MASK_NBITS);
 747         efx->wanted_fc &= ~(EFX_FC_TX | EFX_FC_RX);
 748 }
 749
 750 void efx_link_set_wanted_fc(struct efx_nic *efx, u8 wanted_fc)
 751 {
 752         efx->wanted_fc = wanted_fc;
 753         if (efx->link_advertising[0]) {
 754                 if (wanted_fc & EFX_FC_RX)
 755                         efx->link_advertising[0] |= (ADVERTISED_Pause |
 756                                                      ADVERTISED_Asym_Pause);
 757                 else
 758                         efx->link_advertising[0] &= ~(ADVERTISED_Pause |
 759                                                       ADVERTISED_Asym_Pause);
 760                 if (wanted_fc & EFX_FC_TX)
 761                         efx->link_advertising[0] ^= ADVERTISED_Asym_Pause;
 762         }
 763 }
 764
 765 static void efx_fini_port(struct efx_nic *efx);
 766
 767 static int efx_probe_port(struct efx_nic *efx)
 768 {
 769         int rc;
 770
 771         netif_dbg(efx, probe, efx->net_dev, "create port\n");
 772
 773         if (phy_flash_cfg)
 774                 efx->phy_mode = PHY_MODE_SPECIAL;
 775
 776         /* Connect up MAC/PHY operations table */
 777         rc = efx->type->probe_port(efx);
 778         if (rc)
 779                 return rc;
 780
 781         /* Initialise MAC address to permanent address */
 782         ether_addr_copy(efx->net_dev->dev_addr, efx->net_dev->perm_addr);
 783
 784         return 0;
 785 }
 786
 787 static int efx_init_port(struct efx_nic *efx)
 788 {
 789         int rc;
 790
 791         netif_dbg(efx, drv, efx->net_dev, "init port\n");
 792
 793         mutex_lock(&efx->mac_lock);
 794
 795         rc = efx->phy_op->init(efx);
 796         if (rc)
 797                 goto fail1;
 798
 799         efx->port_initialized = true;
 800
 801         /* Reconfigure the MAC before creating dma queues (required for
 802          * Falcon/A1 where RX_INGR_EN/TX_DRAIN_EN isn't supported) */
 803         efx_mac_reconfigure(efx);
 804
 805         /* Ensure the PHY advertises the correct flow control settings */
 806         rc = efx->phy_op->reconfigure(efx);
 807         if (rc && rc != -EPERM)
 808                 goto fail2;
 809
 810         mutex_unlock(&efx->mac_lock);
 811         return 0;
 812
 813 fail2:
 814         efx->phy_op->fini(efx);
 815 fail1:
 816         mutex_unlock(&efx->mac_lock);
 817         return rc;
 818 }
 819
 820 static void efx_fini_port(struct efx_nic *efx)
 821 {
 822         netif_dbg(efx, drv, efx->net_dev, "shut down port\n");
 823
 824         if (!efx->port_initialized)
 825                 return;
 826
 827         efx->phy_op->fini(efx);
 828         efx->port_initialized = false;
 829
 830         efx->link_state.up = false;
 831         efx_link_status_changed(efx);
 832 }
 833
 834 static void efx_remove_port(struct efx_nic *efx)
 835 {
 836         netif_dbg(efx, drv, efx->net_dev, "destroying port\n");
 837
 838         efx->type->remove_port(efx);
 839 }
 840
 841 /**************************************************************************
 842  *
 843  * NIC handling
 844  *
 845  **************************************************************************/
 846
 847 static LIST_HEAD(efx_primary_list);
 848 static LIST_HEAD(efx_unassociated_list);
 849
 850 static bool efx_same_controller(struct efx_nic *left, struct efx_nic *right)
 851 {
 852         return left->type == right->type &&
 853                 left->vpd_sn && right->vpd_sn &&
 854                 !strcmp(left->vpd_sn, right->vpd_sn);
 855 }
 856
 857 static void efx_associate(struct efx_nic *efx)
 858 {
 859         struct efx_nic *other, *next;
 860
 861         if (efx->primary == efx) {
 862                 /* Adding primary function; look for secondaries */
 863
 864                 netif_dbg(efx, probe, efx->net_dev, "adding to primary list\n");
 865                 list_add_tail(&efx->node, &efx_primary_list);
 866
 867                 list_for_each_entry_safe(other, next, &efx_unassociated_list,
 868                                          node) {
 869                         if (efx_same_controller(efx, other)) {
 870                                 list_del(&other->node);
 871                                 netif_dbg(other, probe, other->net_dev,
 872                                           "moving to secondary list of %s %s\n",
 873                                           pci_name(efx->pci_dev),
 874                                           efx->net_dev->name);
 875                                 list_add_tail(&other->node,
 876                                               &efx->secondary_list);
 877                                 other->primary = efx;
 878                         }
 879                 }
 880         } else {
 881                 /* Adding secondary function; look for primary */
 882
 883                 list_for_each_entry(other, &efx_primary_list, node) {
 884                         if (efx_same_controller(efx, other)) {
 885                                 netif_dbg(efx, probe, efx->net_dev,
 886                                           "adding to secondary list of %s %s\n",
 887                                           pci_name(other->pci_dev),
 888                                           other->net_dev->name);
 889                                 list_add_tail(&efx->node,
 890                                               &other->secondary_list);
 891                                 efx->primary = other;
 892                                 return;
 893                         }
 894                 }
 895
 896                 netif_dbg(efx, probe, efx->net_dev,
 897                           "adding to unassociated list\n");
 898                 list_add_tail(&efx->node, &efx_unassociated_list);
 899         }
 900 }
 901
 902 static void efx_dissociate(struct efx_nic *efx)
 903 {
 904         struct efx_nic *other, *next;
 905
 906         list_del(&efx->node);
 907         efx->primary = NULL;
 908
 909         list_for_each_entry_safe(other, next, &efx->secondary_list, node) {
 910                 list_del(&other->node);
 911                 netif_dbg(other, probe, other->net_dev,
 912                           "moving to unassociated list\n");
 913                 list_add_tail(&other->node, &efx_unassociated_list);
 914                 other->primary = NULL;
 915         }
 916 }
 917
 918 void efx_set_default_rx_indir_table(struct efx_nic *efx,
 919                                     struct efx_rss_context *ctx)
 920 {
 921         size_t i;
 922
 923         for (i = 0; i < ARRAY_SIZE(ctx->rx_indir_table); i++)
 924                 ctx->rx_indir_table[i] =
 925                         ethtool_rxfh_indir_default(i, efx->rss_spread);
 926 }
 927
 928 static unsigned int efx_wanted_parallelism(struct efx_nic *efx)
 929 {
 930         cpumask_var_t thread_mask;
 931         unsigned int count;
 932         int cpu;
 933
 934         if (rss_cpus) {
 935                 count = rss_cpus;
 936         } else {
 937                 if (unlikely(!zalloc_cpumask_var(&thread_mask, GFP_KERNEL))) {
 938                         netif_warn(efx, probe, efx->net_dev,
 939                                    "RSS disabled due to allocation failure\n");
 940                         return 1;
 941                 }
 942
 943                 count = 0;
 944                 for_each_online_cpu(cpu) {
 945                         if (!cpumask_test_cpu(cpu, thread_mask)) {
 946                                 ++count;
 947                                 cpumask_or(thread_mask, thread_mask,
 948                                            topology_sibling_cpumask(cpu));
 949                         }
 950                 }
 951
 952                 free_cpumask_var(thread_mask);
 953         }
 954
 955         if (count > EFX_MAX_RX_QUEUES) {
 956                 netif_cond_dbg(efx, probe, efx->net_dev, !rss_cpus, warn,
 957                                "Reducing number of rx queues from %u to %u.\n",
 958                                count, EFX_MAX_RX_QUEUES);
 959                 count = EFX_MAX_RX_QUEUES;
 960         }
 961
 962         /* If RSS is requested for the PF *and* VFs then we can't write RSS
 963          * table entries that are inaccessible to VFs
 964          */
 965 #ifdef CONFIG_SFC_SRIOV
 966         if (efx->type->sriov_wanted) {
 967                 if (efx->type->sriov_wanted(efx) && efx_vf_size(efx) > 1 &&
 968                     count > efx_vf_size(efx)) {
 969                         netif_warn(efx, probe, efx->net_dev,
 970                                    "Reducing number of RSS channels from %u to %u for "
 971                                    "VF support. Increase vf-msix-limit to use more "
 972                                    "channels on the PF.\n",
 973                                    count, efx_vf_size(efx));
 974                         count = efx_vf_size(efx);
 975                 }
 976         }
 977 #endif
 978
 979         return count;
 980 }
 981
 982 static int efx_allocate_msix_channels(struct efx_nic *efx,
 983                                       unsigned int max_channels,
 984                                       unsigned int extra_channels,
 985                                       unsigned int parallelism)
 986 {
 987         unsigned int n_channels = parallelism;
 988         int vec_count;
 989         int n_xdp_tx;
 990         int n_xdp_ev;
 991
 992         if (efx_separate_tx_channels)
 993                 n_channels *= 2;
 994         n_channels += extra_channels;
 995
 996         /* To allow XDP transmit to happen from arbitrary NAPI contexts
 997          * we allocate a TX queue per CPU. We share event queues across
 998          * multiple tx queues, assuming tx and ev queues are both
 999          * maximum size.
1000          */
1001
1002         n_xdp_tx = num_possible_cpus();
1003         n_xdp_ev = DIV_ROUND_UP(n_xdp_tx, EFX_TXQ_TYPES);
1004
1005         vec_count = pci_msix_vec_count(efx->pci_dev);
1006         if (vec_count < 0)
1007                 return vec_count;
1008
1009         max_channels = min_t(unsigned int, vec_count, max_channels);
1010
1011         /* Check resources.
1012          * We need a channel per event queue, plus a VI per tx queue.
1013          * This may be more pessimistic than it needs to be.
1014          */
1015         if (n_channels + n_xdp_ev > max_channels) {
1016                 netif_err(efx, drv, efx->net_dev,
1017                           "Insufficient resources for %d XDP event queues (%d other channels, max %d)\n",
1018                           n_xdp_ev, n_channels, max_channels);
1019                 efx->n_xdp_channels = 0;
1020                 efx->xdp_tx_per_channel = 0;
1021                 efx->xdp_tx_queue_count = 0;
1022         } else {
1023                 efx->n_xdp_channels = n_xdp_ev;
1024                 efx->xdp_tx_per_channel = EFX_TXQ_TYPES;
1025                 efx->xdp_tx_queue_count = n_xdp_tx;
1026                 n_channels += n_xdp_ev;
1027                 netif_dbg(efx, drv, efx->net_dev,
1028                           "Allocating %d TX and %d event queues for XDP\n",
1029                           n_xdp_tx, n_xdp_ev);
1030         }
1031
1032         if (vec_count < n_channels) {
1033                 netif_err(efx, drv, efx->net_dev,
1034                           "WARNING: Insufficient MSI-X vectors available (%d < %u).\n",
1035                           vec_count, n_channels);
1036                 netif_err(efx, drv, efx->net_dev,
1037                           "WARNING: Performance may be reduced.\n");
1038                 n_channels = vec_count;
1039         }
1040
1041         n_channels = min(n_channels, max_channels);
1042
1043         efx->n_channels = n_channels;
1044
1045         /* Ignore XDP tx channels when creating rx channels. */
1046         n_channels -= efx->n_xdp_channels;
1047
1048         if (efx_separate_tx_channels) {
1049                 efx->n_tx_channels =
1050                         min(max(n_channels / 2, 1U),
1051                             efx->max_tx_channels);
1052                 efx->tx_channel_offset =
1053                         n_channels - efx->n_tx_channels;
1054                 efx->n_rx_channels =
1055                         max(n_channels -
1056                             efx->n_tx_channels, 1U);
1057         } else {
1058                 efx->n_tx_channels = min(n_channels, efx->max_tx_channels);
1059                 efx->tx_channel_offset = 0;
1060                 efx->n_rx_channels = n_channels;
1061         }
1062
1063         efx->n_rx_channels = min(efx->n_rx_channels, parallelism);
1064         efx->n_tx_channels = min(efx->n_tx_channels, parallelism);
1065
1066         efx->xdp_channel_offset = n_channels;
1067
1068         netif_dbg(efx, drv, efx->net_dev,
1069                   "Allocating %u RX channels\n",
1070                   efx->n_rx_channels);
1071
1072         return efx->n_channels;
1073 }
1074
1075 /* Probe the number and type of interrupts we are able to obtain, and
1076  * the resulting numbers of channels and RX queues.
1077  */
1078 int efx_probe_interrupts(struct efx_nic *efx)
1079 {
1080         unsigned int extra_channels = 0;
1081         unsigned int rss_spread;
1082         unsigned int i, j;
1083         int rc;
1084
1085         for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++)
1086                 if (efx->extra_channel_type[i])
1087                         ++extra_channels;
1088
1089         if (efx->interrupt_mode == EFX_INT_MODE_MSIX) {
1090                 unsigned int parallelism = efx_wanted_parallelism(efx);
1091                 struct msix_entry xentries[EFX_MAX_CHANNELS];
1092                 unsigned int n_channels;
1093
1094                 rc = efx_allocate_msix_channels(efx, efx->max_channels,
1095                                                 extra_channels, parallelism);
1096                 if (rc >= 0) {
1097                         n_channels = rc;
1098                         for (i = 0; i < n_channels; i++)
1099                                 xentries[i].entry = i;
1100                         rc = pci_enable_msix_range(efx->pci_dev, xentries, 1,
1101                                                    n_channels);
1102                 }
1103                 if (rc < 0) {
1104                         /* Fall back to single channel MSI */
1105                         netif_err(efx, drv, efx->net_dev,
1106                                   "could not enable MSI-X\n");
1107                         if (efx->type->min_interrupt_mode >= EFX_INT_MODE_MSI)
1108                                 efx->interrupt_mode = EFX_INT_MODE_MSI;
1109                         else
1110                                 return rc;
1111                 } else if (rc < n_channels) {
1112                         netif_err(efx, drv, efx->net_dev,
1113                                   "WARNING: Insufficient MSI-X vectors"
1114                                   " available (%d < %u).\n", rc, n_channels);
1115                         netif_err(efx, drv, efx->net_dev,
1116                                   "WARNING: Performance may be reduced.\n");
1117                         n_channels = rc;
1118                 }
1119
1120                 if (rc > 0) {
1121                         for (i = 0; i < efx->n_channels; i++)
1122                                 efx_get_channel(efx, i)->irq =
1123                                         xentries[i].vector;
1124                 }
1125         }
1126
1127         /* Try single interrupt MSI */
1128         if (efx->interrupt_mode == EFX_INT_MODE_MSI) {
1129                 efx->n_channels = 1;
1130                 efx->n_rx_channels = 1;
1131                 efx->n_tx_channels = 1;
1132                 efx->n_xdp_channels = 0;
1133                 efx->xdp_channel_offset = efx->n_channels;
1134                 rc = pci_enable_msi(efx->pci_dev);
1135                 if (rc == 0) {
1136                         efx_get_channel(efx, 0)->irq = efx->pci_dev->irq;
1137                 } else {
1138                         netif_err(efx, drv, efx->net_dev,
1139                                   "could not enable MSI\n");
1140                         if (efx->type->min_interrupt_mode >= EFX_INT_MODE_LEGACY)
1141                                 efx->interrupt_mode = EFX_INT_MODE_LEGACY;
1142                         else
1143                                 return rc;
1144                 }
1145         }
1146
1147         /* Assume legacy interrupts */
1148         if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) {
1149                 efx->n_channels = 1 + (efx_separate_tx_channels ? 1 : 0);
1150                 efx->n_rx_channels = 1;
1151                 efx->n_tx_channels = 1;
1152                 efx->n_xdp_channels = 0;
1153                 efx->xdp_channel_offset = efx->n_channels;
1154                 efx->legacy_irq = efx->pci_dev->irq;
1155         }
1156
1157         /* Assign extra channels if possible, before XDP channels */
1158         efx->n_extra_tx_channels = 0;
1159         j = efx->xdp_channel_offset;
1160         for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) {
1161                 if (!efx->extra_channel_type[i])
1162                         continue;
1163                 if (j <= efx->tx_channel_offset + efx->n_tx_channels) {
1164                         efx->extra_channel_type[i]->handle_no_channel(efx);
1165                 } else {
1166                         --j;
1167                         efx_get_channel(efx, j)->type =
1168                                 efx->extra_channel_type[i];
1169                         if (efx_channel_has_tx_queues(efx_get_channel(efx, j)))
1170                                 efx->n_extra_tx_channels++;
1171                 }
1172         }
1173
1174         rss_spread = efx->n_rx_channels;
1175         /* RSS might be usable on VFs even if it is disabled on the PF */
1176 #ifdef CONFIG_SFC_SRIOV
1177         if (efx->type->sriov_wanted) {
1178                 efx->rss_spread = ((rss_spread > 1 ||
1179                                     !efx->type->sriov_wanted(efx)) ?
1180                                    rss_spread : efx_vf_size(efx));
1181                 return 0;
1182         }
1183 #endif
1184         efx->rss_spread = rss_spread;
1185
1186         return 0;
1187 }
1188
1189 #if defined(CONFIG_SMP)
1190 void efx_set_interrupt_affinity(struct efx_nic *efx)
1191 {
1192         struct efx_channel *channel;
1193         unsigned int cpu;
1194
1195         efx_for_each_channel(channel, efx) {
1196                 cpu = cpumask_local_spread(channel->channel,
1197                                            pcibus_to_node(efx->pci_dev->bus));
1198                 irq_set_affinity_hint(channel->irq, cpumask_of(cpu));
1199         }
1200 }
1201
1202 void efx_clear_interrupt_affinity(struct efx_nic *efx)
1203 {
1204         struct efx_channel *channel;
1205
1206         efx_for_each_channel(channel, efx)
1207                 irq_set_affinity_hint(channel->irq, NULL);
1208 }
1209 #else
1210 void efx_set_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused)))
1211 {
1212 }
1213
1214 void efx_clear_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused)))
1215 {
1216 }
1217 #endif /* CONFIG_SMP */
1218
1219 int efx_soft_enable_interrupts(struct efx_nic *efx)
1220 {
1221         struct efx_channel *channel, *end_channel;
1222         int rc;
1223
1224         BUG_ON(efx->state == STATE_DISABLED);
1225
1226         efx->irq_soft_enabled = true;
1227         smp_wmb();
1228
1229         efx_for_each_channel(channel, efx) {
1230                 if (!channel->type->keep_eventq) {
1231                         rc = efx_init_eventq(channel);
1232                         if (rc)
1233                                 goto fail;
1234                 }
1235                 efx_start_eventq(channel);
1236         }
1237
1238         efx_mcdi_mode_event(efx);
1239
1240         return 0;
1241 fail:
1242         end_channel = channel;
1243         efx_for_each_channel(channel, efx) {
1244                 if (channel == end_channel)
1245                         break;
1246                 efx_stop_eventq(channel);
1247                 if (!channel->type->keep_eventq)
1248                         efx_fini_eventq(channel);
1249         }
1250
1251         return rc;
1252 }
1253
1254 void efx_soft_disable_interrupts(struct efx_nic *efx)
1255 {
1256         struct efx_channel *channel;
1257
1258         if (efx->state == STATE_DISABLED)
1259                 return;
1260
1261         efx_mcdi_mode_poll(efx);
1262
1263         efx->irq_soft_enabled = false;
1264         smp_wmb();
1265
1266         if (efx->legacy_irq)
1267                 synchronize_irq(efx->legacy_irq);
1268
1269         efx_for_each_channel(channel, efx) {
1270                 if (channel->irq)
1271                         synchronize_irq(channel->irq);
1272
1273                 efx_stop_eventq(channel);
1274                 if (!channel->type->keep_eventq)
1275                         efx_fini_eventq(channel);
1276         }
1277
1278         /* Flush the asynchronous MCDI request queue */
1279         efx_mcdi_flush_async(efx);
1280 }
1281
1282 int efx_enable_interrupts(struct efx_nic *efx)
1283 {
1284         struct efx_channel *channel, *end_channel;
1285         int rc;
1286
1287         BUG_ON(efx->state == STATE_DISABLED);
1288
1289         if (efx->eeh_disabled_legacy_irq) {
1290                 enable_irq(efx->legacy_irq);
1291                 efx->eeh_disabled_legacy_irq = false;
1292         }
1293
1294         efx->type->irq_enable_master(efx);
1295
1296         efx_for_each_channel(channel, efx) {
1297                 if (channel->type->keep_eventq) {
1298                         rc = efx_init_eventq(channel);
1299                         if (rc)
1300                                 goto fail;
1301                 }
1302         }
1303
1304         rc = efx_soft_enable_interrupts(efx);
1305         if (rc)
1306                 goto fail;
1307
1308         return 0;
1309
1310 fail:
1311         end_channel = channel;
1312         efx_for_each_channel(channel, efx) {
1313                 if (channel == end_channel)
1314                         break;
1315                 if (channel->type->keep_eventq)
1316                         efx_fini_eventq(channel);
1317         }
1318
1319         efx->type->irq_disable_non_ev(efx);
1320
1321         return rc;
1322 }
1323
1324 void efx_disable_interrupts(struct efx_nic *efx)
1325 {
1326         struct efx_channel *channel;
1327
1328         efx_soft_disable_interrupts(efx);
1329
1330         efx_for_each_channel(channel, efx) {
1331                 if (channel->type->keep_eventq)
1332                         efx_fini_eventq(channel);
1333         }
1334
1335         efx->type->irq_disable_non_ev(efx);
1336 }
1337
1338 void efx_remove_interrupts(struct efx_nic *efx)
1339 {
1340         struct efx_channel *channel;
1341
1342         /* Remove MSI/MSI-X interrupts */
1343         efx_for_each_channel(channel, efx)
1344                 channel->irq = 0;
1345         pci_disable_msi(efx->pci_dev);
1346         pci_disable_msix(efx->pci_dev);
1347
1348         /* Remove legacy interrupt */
1349         efx->legacy_irq = 0;
1350 }
1351
1352 int efx_set_channels(struct efx_nic *efx)
1353 {
1354         struct efx_channel *channel;
1355         struct efx_tx_queue *tx_queue;
1356         int xdp_queue_number;
1357
1358         efx->tx_channel_offset =
1359                 efx_separate_tx_channels ?
1360                 efx->n_channels - efx->n_tx_channels : 0;
1361
1362         if (efx->xdp_tx_queue_count) {
1363                 EFX_WARN_ON_PARANOID(efx->xdp_tx_queues);
1364
1365                 /* Allocate array for XDP TX queue lookup. */
1366                 efx->xdp_tx_queues = kcalloc(efx->xdp_tx_queue_count,
1367                                              sizeof(*efx->xdp_tx_queues),
1368                                              GFP_KERNEL);
1369                 if (!efx->xdp_tx_queues)
1370                         return -ENOMEM;
1371         }
1372
1373         /* We need to mark which channels really have RX and TX
1374          * queues, and adjust the TX queue numbers if we have separate
1375          * RX-only and TX-only channels.
1376          */
1377         xdp_queue_number = 0;
1378         efx_for_each_channel(channel, efx) {
1379                 if (channel->channel < efx->n_rx_channels)
1380                         channel->rx_queue.core_index = channel->channel;
1381                 else
1382                         channel->rx_queue.core_index = -1;
1383
1384                 efx_for_each_channel_tx_queue(tx_queue, channel) {
1385                         tx_queue->queue -= (efx->tx_channel_offset *
1386                                             EFX_TXQ_TYPES);
1387
1388                         if (efx_channel_is_xdp_tx(channel) &&
1389                             xdp_queue_number < efx->xdp_tx_queue_count) {
1390                                 efx->xdp_tx_queues[xdp_queue_number] = tx_queue;
1391                                 xdp_queue_number++;
1392                         }
1393                 }
1394         }
1395         return 0;
1396 }
1397
1398 static int efx_probe_nic(struct efx_nic *efx)
1399 {
1400         int rc;
1401
1402         netif_dbg(efx, probe, efx->net_dev, "creating NIC\n");
1403
1404         /* Carry out hardware-type specific initialisation */
1405         rc = efx->type->probe(efx);
1406         if (rc)
1407                 return rc;
1408
1409         do {
1410                 if (!efx->max_channels || !efx->max_tx_channels) {
1411                         netif_err(efx, drv, efx->net_dev,
1412                                   "Insufficient resources to allocate"
1413                                   " any channels\n");
1414                         rc = -ENOSPC;
1415                         goto fail1;
1416                 }
1417
1418                 /* Determine the number of channels and queues by trying
1419                  * to hook in MSI-X interrupts.
1420                  */
1421                 rc = efx_probe_interrupts(efx);
1422                 if (rc)
1423                         goto fail1;
1424
1425                 rc = efx_set_channels(efx);
1426                 if (rc)
1427                         goto fail1;
1428
1429                 /* dimension_resources can fail with EAGAIN */
1430                 rc = efx->type->dimension_resources(efx);
1431                 if (rc != 0 && rc != -EAGAIN)
1432                         goto fail2;
1433
1434                 if (rc == -EAGAIN)
1435                         /* try again with new max_channels */
1436                         efx_remove_interrupts(efx);
1437
1438         } while (rc == -EAGAIN);
1439
1440         if (efx->n_channels > 1)
1441                 netdev_rss_key_fill(efx->rss_context.rx_hash_key,
1442                                     sizeof(efx->rss_context.rx_hash_key));
1443         efx_set_default_rx_indir_table(efx, &efx->rss_context);
1444
1445         netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels);
1446         netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels);
1447
1448         /* Initialise the interrupt moderation settings */
1449         efx->irq_mod_step_us = DIV_ROUND_UP(efx->timer_quantum_ns, 1000);
1450         efx_init_irq_moderation(efx, tx_irq_mod_usec, rx_irq_mod_usec, true,
1451                                 true);
1452
1453         return 0;
1454
1455 fail2:
1456         efx_remove_interrupts(efx);
1457 fail1:
1458         efx->type->remove(efx);
1459         return rc;
1460 }
1461
1462 static void efx_remove_nic(struct efx_nic *efx)
1463 {
1464         netif_dbg(efx, drv, efx->net_dev, "destroying NIC\n");
1465
1466         efx_remove_interrupts(efx);
1467         efx->type->remove(efx);
1468 }
1469
1470 static int efx_probe_filters(struct efx_nic *efx)
1471 {
1472         int rc;
1473
1474         init_rwsem(&efx->filter_sem);
1475         mutex_lock(&efx->mac_lock);
1476         down_write(&efx->filter_sem);
1477         rc = efx->type->filter_table_probe(efx);
1478         if (rc)
1479                 goto out_unlock;
1480
1481 #ifdef CONFIG_RFS_ACCEL
1482         if (efx->type->offload_features & NETIF_F_NTUPLE) {
1483                 struct efx_channel *channel;
1484                 int i, success = 1;
1485
1486                 efx_for_each_channel(channel, efx) {
1487                         channel->rps_flow_id =
1488                                 kcalloc(efx->type->max_rx_ip_filters,
1489                                         sizeof(*channel->rps_flow_id),
1490                                         GFP_KERNEL);
1491                         if (!channel->rps_flow_id)
1492                                 success = 0;
1493                         else
1494                                 for (i = 0;
1495                                      i < efx->type->max_rx_ip_filters;
1496                                      ++i)
1497                                         channel->rps_flow_id[i] =
1498                                                 RPS_FLOW_ID_INVALID;
1499                         channel->rfs_expire_index = 0;
1500                         channel->rfs_filter_count = 0;
1501                 }
1502
1503                 if (!success) {
1504                         efx_for_each_channel(channel, efx)
1505                                 kfree(channel->rps_flow_id);
1506                         efx->type->filter_table_remove(efx);
1507                         rc = -ENOMEM;
1508                         goto out_unlock;
1509                 }
1510         }
1511 #endif
1512 out_unlock:
1513         up_write(&efx->filter_sem);
1514         mutex_unlock(&efx->mac_lock);
1515         return rc;
1516 }
1517
1518 static void efx_remove_filters(struct efx_nic *efx)
1519 {
1520 #ifdef CONFIG_RFS_ACCEL
1521         struct efx_channel *channel;
1522
1523         efx_for_each_channel(channel, efx) {
1524                 cancel_delayed_work_sync(&channel->filter_work);
1525                 kfree(channel->rps_flow_id);
1526         }
1527 #endif
1528         down_write(&efx->filter_sem);
1529         efx->type->filter_table_remove(efx);
1530         up_write(&efx->filter_sem);
1531 }
1532
1533
1534 /**************************************************************************
1535  *
1536  * NIC startup/shutdown
1537  *
1538  *************************************************************************/
1539
1540 static int efx_probe_all(struct efx_nic *efx)
1541 {
1542         int rc;
1543
1544         rc = efx_probe_nic(efx);
1545         if (rc) {
1546                 netif_err(efx, probe, efx->net_dev, "failed to create NIC\n");
1547                 goto fail1;
1548         }
1549
1550         rc = efx_probe_port(efx);
1551         if (rc) {
1552                 netif_err(efx, probe, efx->net_dev, "failed to create port\n");
1553                 goto fail2;
1554         }
1555
1556         BUILD_BUG_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_RXQ_MIN_ENT);
1557         if (WARN_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_TXQ_MIN_ENT(efx))) {
1558                 rc = -EINVAL;
1559                 goto fail3;
1560         }
1561         efx->rxq_entries = efx->txq_entries = EFX_DEFAULT_DMAQ_SIZE;
1562
1563 #ifdef CONFIG_SFC_SRIOV
1564         rc = efx->type->vswitching_probe(efx);
1565         if (rc) /* not fatal; the PF will still work fine */
1566                 netif_warn(efx, probe, efx->net_dev,
1567                            "failed to setup vswitching rc=%d;"
1568                            " VFs may not function\n", rc);
1569 #endif
1570
1571         rc = efx_probe_filters(efx);
1572         if (rc) {
1573                 netif_err(efx, probe, efx->net_dev,
1574                           "failed to create filter tables\n");
1575                 goto fail4;
1576         }
1577
1578         rc = efx_probe_channels(efx);
1579         if (rc)
1580                 goto fail5;
1581
1582         return 0;
1583
1584  fail5:
1585         efx_remove_filters(efx);
1586  fail4:
1587 #ifdef CONFIG_SFC_SRIOV
1588         efx->type->vswitching_remove(efx);
1589 #endif
1590  fail3:
1591         efx_remove_port(efx);
1592  fail2:
1593         efx_remove_nic(efx);
1594  fail1:
1595         return rc;
1596 }
1597
1598 static void efx_remove_all(struct efx_nic *efx)
1599 {
1600         rtnl_lock();
1601         efx_xdp_setup_prog(efx, NULL);
1602         rtnl_unlock();
1603
1604         efx_remove_channels(efx);
1605         efx_remove_filters(efx);
1606 #ifdef CONFIG_SFC_SRIOV
1607         efx->type->vswitching_remove(efx);
1608 #endif
1609         efx_remove_port(efx);
1610         efx_remove_nic(efx);
1611 }
1612
1613 /**************************************************************************
1614  *
1615  * Interrupt moderation
1616  *
1617  **************************************************************************/
1618 unsigned int efx_usecs_to_ticks(struct efx_nic *efx, unsigned int usecs)
1619 {
1620         if (usecs == 0)
1621                 return 0;
1622         if (usecs * 1000 < efx->timer_quantum_ns)
1623                 return 1; /* never round down to 0 */
1624         return usecs * 1000 / efx->timer_quantum_ns;
1625 }
1626
1627 unsigned int efx_ticks_to_usecs(struct efx_nic *efx, unsigned int ticks)
1628 {
1629         /* We must round up when converting ticks to microseconds
1630          * because we round down when converting the other way.
1631          */
1632         return DIV_ROUND_UP(ticks * efx->timer_quantum_ns, 1000);
1633 }
1634
1635 /* Set interrupt moderation parameters */
1636 int efx_init_irq_moderation(struct efx_nic *efx, unsigned int tx_usecs,
1637                             unsigned int rx_usecs, bool rx_adaptive,
1638                             bool rx_may_override_tx)
1639 {
1640         struct efx_channel *channel;
1641         unsigned int timer_max_us;
1642
1643         EFX_ASSERT_RESET_SERIALISED(efx);
1644
1645         timer_max_us = efx->timer_max_ns / 1000;
1646
1647         if (tx_usecs > timer_max_us || rx_usecs > timer_max_us)
1648                 return -EINVAL;
1649
1650         if (tx_usecs != rx_usecs && efx->tx_channel_offset == 0 &&
1651             !rx_may_override_tx) {
1652                 netif_err(efx, drv, efx->net_dev, "Channels are shared. "
1653                           "RX and TX IRQ moderation must be equal\n");
1654                 return -EINVAL;
1655         }
1656
1657         efx->irq_rx_adaptive = rx_adaptive;
1658         efx->irq_rx_moderation_us = rx_usecs;
1659         efx_for_each_channel(channel, efx) {
1660                 if (efx_channel_has_rx_queue(channel))
1661                         channel->irq_moderation_us = rx_usecs;
1662                 else if (efx_channel_has_tx_queues(channel))
1663                         channel->irq_moderation_us = tx_usecs;
1664                 else if (efx_channel_is_xdp_tx(channel))
1665                         channel->irq_moderation_us = tx_usecs;
1666         }
1667
1668         return 0;
1669 }
1670
1671 void efx_get_irq_moderation(struct efx_nic *efx, unsigned int *tx_usecs,
1672                             unsigned int *rx_usecs, bool *rx_adaptive)
1673 {
1674         *rx_adaptive = efx->irq_rx_adaptive;
1675         *rx_usecs = efx->irq_rx_moderation_us;
1676
1677         /* If channels are shared between RX and TX, so is IRQ
1678          * moderation.  Otherwise, IRQ moderation is the same for all
1679          * TX channels and is not adaptive.
1680          */
1681         if (efx->tx_channel_offset == 0) {
1682                 *tx_usecs = *rx_usecs;
1683         } else {
1684                 struct efx_channel *tx_channel;
1685
1686                 tx_channel = efx->channel[efx->tx_channel_offset];
1687                 *tx_usecs = tx_channel->irq_moderation_us;
1688         }
1689 }
1690
1691 /**************************************************************************
1692  *
1693  * ioctls
1694  *
1695  *************************************************************************/
1696
1697 /* Net device ioctl
1698  * Context: process, rtnl_lock() held.
1699  */
1700 static int efx_ioctl(struct net_device *net_dev, struct ifreq *ifr, int cmd)
1701 {
1702         struct efx_nic *efx = netdev_priv(net_dev);
1703         struct mii_ioctl_data *data = if_mii(ifr);
1704
1705         if (cmd == SIOCSHWTSTAMP)
1706                 return efx_ptp_set_ts_config(efx, ifr);
1707         if (cmd == SIOCGHWTSTAMP)
1708                 return efx_ptp_get_ts_config(efx, ifr);
1709
1710         /* Convert phy_id from older PRTAD/DEVAD format */
1711         if ((cmd == SIOCGMIIREG || cmd == SIOCSMIIREG) &&
1712             (data->phy_id & 0xfc00) == 0x0400)
1713                 data->phy_id ^= MDIO_PHY_ID_C45 | 0x0400;
1714
1715         return mdio_mii_ioctl(&efx->mdio, data, cmd);
1716 }
1717
1718 /**************************************************************************
1719  *
1720  * NAPI interface
1721  *
1722  **************************************************************************/
1723
1724 void efx_init_napi_channel(struct efx_channel *channel)
1725 {
1726         struct efx_nic *efx = channel->efx;
1727
1728         channel->napi_dev = efx->net_dev;
1729         netif_napi_add(channel->napi_dev, &channel->napi_str,
1730                        efx_poll, napi_weight);
1731 }
1732
1733 void efx_init_napi(struct efx_nic *efx)
1734 {
1735         struct efx_channel *channel;
1736
1737         efx_for_each_channel(channel, efx)
1738                 efx_init_napi_channel(channel);
1739 }
1740
1741 void efx_fini_napi_channel(struct efx_channel *channel)
1742 {
1743         if (channel->napi_dev)
1744                 netif_napi_del(&channel->napi_str);
1745
1746         channel->napi_dev = NULL;
1747 }
1748
1749 void efx_fini_napi(struct efx_nic *efx)
1750 {
1751         struct efx_channel *channel;
1752
1753         efx_for_each_channel(channel, efx)
1754                 efx_fini_napi_channel(channel);
1755 }
1756
1757 /**************************************************************************
1758  *
1759  * Kernel net device interface
1760  *
1761  *************************************************************************/
1762
1763 /* Context: process, rtnl_lock() held. */
1764 int efx_net_open(struct net_device *net_dev)
1765 {
1766         struct efx_nic *efx = netdev_priv(net_dev);
1767         int rc;
1768
1769         netif_dbg(efx, ifup, efx->net_dev, "opening device on CPU %d\n",
1770                   raw_smp_processor_id());
1771
1772         rc = efx_check_disabled(efx);
1773         if (rc)
1774                 return rc;
1775         if (efx->phy_mode & PHY_MODE_SPECIAL)
1776                 return -EBUSY;
1777         if (efx_mcdi_poll_reboot(efx) && efx_reset(efx, RESET_TYPE_ALL))
1778                 return -EIO;
1779
1780         /* Notify the kernel of the link state polled during driver load,
1781          * before the monitor starts running */
1782         efx_link_status_changed(efx);
1783
1784         efx_start_all(efx);
1785         if (efx->state == STATE_DISABLED || efx->reset_pending)
1786                 netif_device_detach(efx->net_dev);
1787         efx_selftest_async_start(efx);
1788         return 0;
1789 }
1790
1791 /* Context: process, rtnl_lock() held.
1792  * Note that the kernel will ignore our return code; this method
1793  * should really be a void.
1794  */
1795 int efx_net_stop(struct net_device *net_dev)
1796 {
1797         struct efx_nic *efx = netdev_priv(net_dev);
1798
1799         netif_dbg(efx, ifdown, efx->net_dev, "closing on CPU %d\n",
1800                   raw_smp_processor_id());
1801
1802         /* Stop the device and flush all the channels */
1803         efx_stop_all(efx);
1804
1805         return 0;
1806 }
1807
1808 /* Context: process, dev_base_lock or RTNL held, non-blocking. */
1809 static void efx_net_stats(struct net_device *net_dev,
1810                           struct rtnl_link_stats64 *stats)
1811 {
1812         struct efx_nic *efx = netdev_priv(net_dev);
1813
1814         spin_lock_bh(&efx->stats_lock);
1815         efx->type->update_stats(efx, NULL, stats);
1816         spin_unlock_bh(&efx->stats_lock);
1817 }
1818
1819 /* Context: netif_tx_lock held, BHs disabled. */
1820 static void efx_watchdog(struct net_device *net_dev, unsigned int txqueue)
1821 {
1822         struct efx_nic *efx = netdev_priv(net_dev);
1823
1824         netif_err(efx, tx_err, efx->net_dev,
1825                   "TX stuck with port_enabled=%d: resetting channels\n",
1826                   efx->port_enabled);
1827
1828         efx_schedule_reset(efx, RESET_TYPE_TX_WATCHDOG);
1829 }
1830
1831 static unsigned int efx_xdp_max_mtu(struct efx_nic *efx)
1832 {
1833         /* The maximum MTU that we can fit in a single page, allowing for
1834          * framing, overhead and XDP headroom.
1835          */
1836         int overhead = EFX_MAX_FRAME_LEN(0) + sizeof(struct efx_rx_page_state) +
1837                        efx->rx_prefix_size + efx->type->rx_buffer_padding +
1838                        efx->rx_ip_align + XDP_PACKET_HEADROOM;
1839
1840         return PAGE_SIZE - overhead;
1841 }
1842
1843 /* Context: process, rtnl_lock() held. */
1844 static int efx_change_mtu(struct net_device *net_dev, int new_mtu)
1845 {
1846         struct efx_nic *efx = netdev_priv(net_dev);
1847         int rc;
1848
1849         rc = efx_check_disabled(efx);
1850         if (rc)
1851                 return rc;
1852
1853         if (rtnl_dereference(efx->xdp_prog) &&
1854             new_mtu > efx_xdp_max_mtu(efx)) {
1855                 netif_err(efx, drv, efx->net_dev,
1856                           "Requested MTU of %d too big for XDP (max: %d)\n",
1857                           new_mtu, efx_xdp_max_mtu(efx));
1858                 return -EINVAL;
1859         }
1860
1861         netif_dbg(efx, drv, efx->net_dev, "changing MTU to %d\n", new_mtu);
1862
1863         efx_device_detach_sync(efx);
1864         efx_stop_all(efx);
1865
1866         mutex_lock(&efx->mac_lock);
1867         net_dev->mtu = new_mtu;
1868         efx_mac_reconfigure(efx);
1869         mutex_unlock(&efx->mac_lock);
1870
1871         efx_start_all(efx);
1872         efx_device_attach_if_not_resetting(efx);
1873         return 0;
1874 }
1875
1876 static int efx_set_mac_address(struct net_device *net_dev, void *data)
1877 {
1878         struct efx_nic *efx = netdev_priv(net_dev);
1879         struct sockaddr *addr = data;
1880         u8 *new_addr = addr->sa_data;
1881         u8 old_addr[6];
1882         int rc;
1883
1884         if (!is_valid_ether_addr(new_addr)) {
1885                 netif_err(efx, drv, efx->net_dev,
1886                           "invalid ethernet MAC address requested: %pM\n",
1887                           new_addr);
1888                 return -EADDRNOTAVAIL;
1889         }
1890
1891         /* save old address */
1892         ether_addr_copy(old_addr, net_dev->dev_addr);
1893         ether_addr_copy(net_dev->dev_addr, new_addr);
1894         if (efx->type->set_mac_address) {
1895                 rc = efx->type->set_mac_address(efx);
1896                 if (rc) {
1897                         ether_addr_copy(net_dev->dev_addr, old_addr);
1898                         return rc;
1899                 }
1900         }
1901
1902         /* Reconfigure the MAC */
1903         mutex_lock(&efx->mac_lock);
1904         efx_mac_reconfigure(efx);
1905         mutex_unlock(&efx->mac_lock);
1906
1907         return 0;
1908 }
1909
1910 /* Context: netif_addr_lock held, BHs disabled. */
1911 static void efx_set_rx_mode(struct net_device *net_dev)
1912 {
1913         struct efx_nic *efx = netdev_priv(net_dev);
1914
1915         if (efx->port_enabled)
1916                 queue_work(efx->workqueue, &efx->mac_work);
1917         /* Otherwise efx_start_port() will do this */
1918 }
1919
1920 static int efx_set_features(struct net_device *net_dev, netdev_features_t data)
1921 {
1922         struct efx_nic *efx = netdev_priv(net_dev);
1923         int rc;
1924
1925         /* If disabling RX n-tuple filtering, clear existing filters */
1926         if (net_dev->features & ~data & NETIF_F_NTUPLE) {
1927                 rc = efx->type->filter_clear_rx(efx, EFX_FILTER_PRI_MANUAL);
1928                 if (rc)
1929                         return rc;
1930         }
1931
1932         /* If Rx VLAN filter is changed, update filters via mac_reconfigure.
1933          * If rx-fcs is changed, mac_reconfigure updates that too.
1934          */
1935         if ((net_dev->features ^ data) & (NETIF_F_HW_VLAN_CTAG_FILTER |
1936                                           NETIF_F_RXFCS)) {
1937                 /* efx_set_rx_mode() will schedule MAC work to update filters
1938                  * when a new features are finally set in net_dev.
1939                  */
1940                 efx_set_rx_mode(net_dev);
1941         }
1942
1943         return 0;
1944 }
1945
1946 static int efx_get_phys_port_id(struct net_device *net_dev,
1947                                 struct netdev_phys_item_id *ppid)
1948 {
1949         struct efx_nic *efx = netdev_priv(net_dev);
1950
1951         if (efx->type->get_phys_port_id)
1952                 return efx->type->get_phys_port_id(efx, ppid);
1953         else
1954                 return -EOPNOTSUPP;
1955 }
1956
1957 static int efx_get_phys_port_name(struct net_device *net_dev,
1958                                   char *name, size_t len)
1959 {
1960         struct efx_nic *efx = netdev_priv(net_dev);
1961
1962         if (snprintf(name, len, "p%u", efx->port_num) >= len)
1963                 return -EINVAL;
1964         return 0;
1965 }
1966
1967 static int efx_vlan_rx_add_vid(struct net_device *net_dev, __be16 proto, u16 vid)
1968 {
1969         struct efx_nic *efx = netdev_priv(net_dev);
1970
1971         if (efx->type->vlan_rx_add_vid)
1972                 return efx->type->vlan_rx_add_vid(efx, proto, vid);
1973         else
1974                 return -EOPNOTSUPP;
1975 }
1976
1977 static int efx_vlan_rx_kill_vid(struct net_device *net_dev, __be16 proto, u16 vid)
1978 {
1979         struct efx_nic *efx = netdev_priv(net_dev);
1980
1981         if (efx->type->vlan_rx_kill_vid)
1982                 return efx->type->vlan_rx_kill_vid(efx, proto, vid);
1983         else
1984                 return -EOPNOTSUPP;
1985 }
1986
1987 static int efx_udp_tunnel_type_map(enum udp_parsable_tunnel_type in)
1988 {
1989         switch (in) {
1990         case UDP_TUNNEL_TYPE_VXLAN:
1991                 return TUNNEL_ENCAP_UDP_PORT_ENTRY_VXLAN;
1992         case UDP_TUNNEL_TYPE_GENEVE:
1993                 return TUNNEL_ENCAP_UDP_PORT_ENTRY_GENEVE;
1994         default:
1995                 return -1;
1996         }
1997 }
1998
1999 static void efx_udp_tunnel_add(struct net_device *dev, struct udp_tunnel_info *ti)
2000 {
2001         struct efx_nic *efx = netdev_priv(dev);
2002         struct efx_udp_tunnel tnl;
2003         int efx_tunnel_type;
2004
2005         efx_tunnel_type = efx_udp_tunnel_type_map(ti->type);
2006         if (efx_tunnel_type < 0)
2007                 return;
2008
2009         tnl.type = (u16)efx_tunnel_type;
2010         tnl.port = ti->port;
2011
2012         if (efx->type->udp_tnl_add_port)
2013                 (void)efx->type->udp_tnl_add_port(efx, tnl);
2014 }
2015
2016 static void efx_udp_tunnel_del(struct net_device *dev, struct udp_tunnel_info *ti)
2017 {
2018         struct efx_nic *efx = netdev_priv(dev);
2019         struct efx_udp_tunnel tnl;
2020         int efx_tunnel_type;
2021
2022         efx_tunnel_type = efx_udp_tunnel_type_map(ti->type);
2023         if (efx_tunnel_type < 0)
2024                 return;
2025
2026         tnl.type = (u16)efx_tunnel_type;
2027         tnl.port = ti->port;
2028
2029         if (efx->type->udp_tnl_del_port)
2030                 (void)efx->type->udp_tnl_del_port(efx, tnl);
2031 }
2032
2033 static const struct net_device_ops efx_netdev_ops = {
2034         .ndo_open               = efx_net_open,
2035         .ndo_stop               = efx_net_stop,
2036         .ndo_get_stats64        = efx_net_stats,
2037         .ndo_tx_timeout         = efx_watchdog,
2038         .ndo_start_xmit         = efx_hard_start_xmit,
2039         .ndo_validate_addr      = eth_validate_addr,
2040         .ndo_do_ioctl           = efx_ioctl,
2041         .ndo_change_mtu         = efx_change_mtu,
2042         .ndo_set_mac_address    = efx_set_mac_address,
2043         .ndo_set_rx_mode        = efx_set_rx_mode,
2044         .ndo_set_features       = efx_set_features,
2045         .ndo_vlan_rx_add_vid    = efx_vlan_rx_add_vid,
2046         .ndo_vlan_rx_kill_vid   = efx_vlan_rx_kill_vid,
2047 #ifdef CONFIG_SFC_SRIOV
2048         .ndo_set_vf_mac         = efx_sriov_set_vf_mac,
2049         .ndo_set_vf_vlan        = efx_sriov_set_vf_vlan,
2050         .ndo_set_vf_spoofchk    = efx_sriov_set_vf_spoofchk,
2051         .ndo_get_vf_config      = efx_sriov_get_vf_config,
2052         .ndo_set_vf_link_state  = efx_sriov_set_vf_link_state,
2053 #endif
2054         .ndo_get_phys_port_id   = efx_get_phys_port_id,
2055         .ndo_get_phys_port_name = efx_get_phys_port_name,
2056         .ndo_setup_tc           = efx_setup_tc,
2057 #ifdef CONFIG_RFS_ACCEL
2058         .ndo_rx_flow_steer      = efx_filter_rfs,
2059 #endif
2060         .ndo_udp_tunnel_add     = efx_udp_tunnel_add,
2061         .ndo_udp_tunnel_del     = efx_udp_tunnel_del,
2062         .ndo_xdp_xmit           = efx_xdp_xmit,
2063         .ndo_bpf                = efx_xdp
2064 };
2065
2066 static int efx_xdp_setup_prog(struct efx_nic *efx, struct bpf_prog *prog)
2067 {
2068         struct bpf_prog *old_prog;
2069
2070         if (efx->xdp_rxq_info_failed) {
2071                 netif_err(efx, drv, efx->net_dev,
2072                           "Unable to bind XDP program due to previous failure of rxq_info\n");
2073                 return -EINVAL;
2074         }
2075
2076         if (prog && efx->net_dev->mtu > efx_xdp_max_mtu(efx)) {
2077                 netif_err(efx, drv, efx->net_dev,
2078                           "Unable to configure XDP with MTU of %d (max: %d)\n",
2079                           efx->net_dev->mtu, efx_xdp_max_mtu(efx));
2080                 return -EINVAL;
2081         }
2082
2083         old_prog = rtnl_dereference(efx->xdp_prog);
2084         rcu_assign_pointer(efx->xdp_prog, prog);
2085         /* Release the reference that was originally passed by the caller. */
2086         if (old_prog)
2087                 bpf_prog_put(old_prog);
2088
2089         return 0;
2090 }
2091
2092 /* Context: process, rtnl_lock() held. */
2093 static int efx_xdp(struct net_device *dev, struct netdev_bpf *xdp)
2094 {
2095         struct efx_nic *efx = netdev_priv(dev);
2096         struct bpf_prog *xdp_prog;
2097
2098         switch (xdp->command) {
2099         case XDP_SETUP_PROG:
2100                 return efx_xdp_setup_prog(efx, xdp->prog);
2101         case XDP_QUERY_PROG:
2102                 xdp_prog = rtnl_dereference(efx->xdp_prog);
2103                 xdp->prog_id = xdp_prog ? xdp_prog->aux->id : 0;
2104                 return 0;
2105         default:
2106                 return -EINVAL;
2107         }
2108 }
2109
2110 static int efx_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **xdpfs,
2111                         u32 flags)
2112 {
2113         struct efx_nic *efx = netdev_priv(dev);
2114
2115         if (!netif_running(dev))
2116                 return -EINVAL;
2117
2118         return efx_xdp_tx_buffers(efx, n, xdpfs, flags & XDP_XMIT_FLUSH);
2119 }
2120
2121 static void efx_update_name(struct efx_nic *efx)
2122 {
2123         strcpy(efx->name, efx->net_dev->name);
2124         efx_mtd_rename(efx);
2125         efx_set_channel_names(efx);
2126 }
2127
2128 static int efx_netdev_event(struct notifier_block *this,
2129                             unsigned long event, void *ptr)
2130 {
2131         struct net_device *net_dev = netdev_notifier_info_to_dev(ptr);
2132
2133         if ((net_dev->netdev_ops == &efx_netdev_ops) &&
2134             event == NETDEV_CHANGENAME)
2135                 efx_update_name(netdev_priv(net_dev));
2136
2137         return NOTIFY_DONE;
2138 }
2139
2140 static struct notifier_block efx_netdev_notifier = {
2141         .notifier_call = efx_netdev_event,
2142 };
2143
2144 static ssize_t
2145 show_phy_type(struct device *dev, struct device_attribute *attr, char *buf)
2146 {
2147         struct efx_nic *efx = dev_get_drvdata(dev);
2148         return sprintf(buf, "%d\n", efx->phy_type);
2149 }
2150 static DEVICE_ATTR(phy_type, 0444, show_phy_type, NULL);
2151
2152 #ifdef CONFIG_SFC_MCDI_LOGGING
2153 static ssize_t show_mcdi_log(struct device *dev, struct device_attribute *attr,
2154                              char *buf)
2155 {
2156         struct efx_nic *efx = dev_get_drvdata(dev);
2157         struct efx_mcdi_iface *mcdi = efx_mcdi(efx);
2158
2159         return scnprintf(buf, PAGE_SIZE, "%d\n", mcdi->logging_enabled);
2160 }
2161 static ssize_t set_mcdi_log(struct device *dev, struct device_attribute *attr,
2162                             const char *buf, size_t count)
2163 {
2164         struct efx_nic *efx = dev_get_drvdata(dev);
2165         struct efx_mcdi_iface *mcdi = efx_mcdi(efx);
2166         bool enable = count > 0 && *buf != '0';
2167
2168         mcdi->logging_enabled = enable;
2169         return count;
2170 }
2171 static DEVICE_ATTR(mcdi_logging, 0644, show_mcdi_log, set_mcdi_log);
2172 #endif
2173
2174 static int efx_register_netdev(struct efx_nic *efx)
2175 {
2176         struct net_device *net_dev = efx->net_dev;
2177         struct efx_channel *channel;
2178         int rc;
2179
2180         net_dev->watchdog_timeo = 5 * HZ;
2181         net_dev->irq = efx->pci_dev->irq;
2182         net_dev->netdev_ops = &efx_netdev_ops;
2183         if (efx_nic_rev(efx) >= EFX_REV_HUNT_A0)
2184                 net_dev->priv_flags |= IFF_UNICAST_FLT;
2185         net_dev->ethtool_ops = &efx_ethtool_ops;
2186         net_dev->gso_max_segs = EFX_TSO_MAX_SEGS;
2187         net_dev->min_mtu = EFX_MIN_MTU;
2188         net_dev->max_mtu = EFX_MAX_MTU;
2189
2190         rtnl_lock();
2191
2192         /* Enable resets to be scheduled and check whether any were
2193          * already requested.  If so, the NIC is probably hosed so we
2194          * abort.
2195          */
2196         efx->state = STATE_READY;
2197         smp_mb(); /* ensure we change state before checking reset_pending */
2198         if (efx->reset_pending) {
2199                 netif_err(efx, probe, efx->net_dev,
2200                           "aborting probe due to scheduled reset\n");
2201                 rc = -EIO;
2202                 goto fail_locked;
2203         }
2204
2205         rc = dev_alloc_name(net_dev, net_dev->name);
2206         if (rc < 0)
2207                 goto fail_locked;
2208         efx_update_name(efx);
2209
2210         /* Always start with carrier off; PHY events will detect the link */
2211         netif_carrier_off(net_dev);
2212
2213         rc = register_netdevice(net_dev);
2214         if (rc)
2215                 goto fail_locked;
2216
2217         efx_for_each_channel(channel, efx) {
2218                 struct efx_tx_queue *tx_queue;
2219                 efx_for_each_channel_tx_queue(tx_queue, channel)
2220                         efx_init_tx_queue_core_txq(tx_queue);
2221         }
2222
2223         efx_associate(efx);
2224
2225         rtnl_unlock();
2226
2227         rc = device_create_file(&efx->pci_dev->dev, &dev_attr_phy_type);
2228         if (rc) {
2229                 netif_err(efx, drv, efx->net_dev,
2230                           "failed to init net dev attributes\n");
2231                 goto fail_registered;
2232         }
2233 #ifdef CONFIG_SFC_MCDI_LOGGING
2234         rc = device_create_file(&efx->pci_dev->dev, &dev_attr_mcdi_logging);
2235         if (rc) {
2236                 netif_err(efx, drv, efx->net_dev,
2237                           "failed to init net dev attributes\n");
2238                 goto fail_attr_mcdi_logging;
2239         }
2240 #endif
2241
2242         return 0;
2243
2244 #ifdef CONFIG_SFC_MCDI_LOGGING
2245 fail_attr_mcdi_logging:
2246         device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type);
2247 #endif
2248 fail_registered:
2249         rtnl_lock();
2250         efx_dissociate(efx);
2251         unregister_netdevice(net_dev);
2252 fail_locked:
2253         efx->state = STATE_UNINIT;
2254         rtnl_unlock();
2255         netif_err(efx, drv, efx->net_dev, "could not register net dev\n");
2256         return rc;
2257 }
2258
2259 static void efx_unregister_netdev(struct efx_nic *efx)
2260 {
2261         if (!efx->net_dev)
2262                 return;
2263
2264         BUG_ON(netdev_priv(efx->net_dev) != efx);
2265
2266         if (efx_dev_registered(efx)) {
2267                 strlcpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name));
2268 #ifdef CONFIG_SFC_MCDI_LOGGING
2269                 device_remove_file(&efx->pci_dev->dev, &dev_attr_mcdi_logging);
2270 #endif
2271                 device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type);
2272                 unregister_netdev(efx->net_dev);
2273         }
2274 }
2275
2276 /**************************************************************************
2277  *
2278  * List of NICs we support
2279  *
2280  **************************************************************************/
2281
2282 /* PCI device ID table */
2283 static const struct pci_device_id efx_pci_table[] = {
2284         {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0803),  /* SFC9020 */
2285          .driver_data = (unsigned long) &siena_a0_nic_type},
2286         {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0813),  /* SFL9021 */
2287          .driver_data = (unsigned long) &siena_a0_nic_type},
2288         {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0903),  /* SFC9120 PF */
2289          .driver_data = (unsigned long) &efx_hunt_a0_nic_type},
2290         {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1903),  /* SFC9120 VF */
2291          .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type},
2292         {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0923),  /* SFC9140 PF */
2293          .driver_data = (unsigned long) &efx_hunt_a0_nic_type},
2294         {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1923),  /* SFC9140 VF */
2295          .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type},
2296         {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0a03),  /* SFC9220 PF */
2297          .driver_data = (unsigned long) &efx_hunt_a0_nic_type},
2298         {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1a03),  /* SFC9220 VF */
2299          .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type},
2300         {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0b03),  /* SFC9250 PF */
2301          .driver_data = (unsigned long) &efx_hunt_a0_nic_type},
2302         {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1b03),  /* SFC9250 VF */
2303          .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type},
2304         {0}                     /* end of list */
2305 };
2306
2307 /**************************************************************************
2308  *
2309  * Data housekeeping
2310  *
2311  **************************************************************************/
2312
2313 int efx_init_channels(struct efx_nic *efx)
2314 {
2315         unsigned int i;
2316
2317         for (i = 0; i < EFX_MAX_CHANNELS; i++) {
2318                 efx->channel[i] = efx_alloc_channel(efx, i, NULL);
2319                 if (!efx->channel[i])
2320                         return -ENOMEM;
2321                 efx->msi_context[i].efx = efx;
2322                 efx->msi_context[i].index = i;
2323         }
2324
2325         /* Higher numbered interrupt modes are less capable! */
2326         if (WARN_ON_ONCE(efx->type->max_interrupt_mode >
2327                          efx->type->min_interrupt_mode)) {
2328                 return -EIO;
2329         }
2330         efx->interrupt_mode = max(efx->type->max_interrupt_mode,
2331                                   interrupt_mode);
2332         efx->interrupt_mode = min(efx->type->min_interrupt_mode,
2333                                   interrupt_mode);
2334
2335         return 0;
2336 }
2337
2338 void efx_update_sw_stats(struct efx_nic *efx, u64 *stats)
2339 {
2340         u64 n_rx_nodesc_trunc = 0;
2341         struct efx_channel *channel;
2342
2343         efx_for_each_channel(channel, efx)
2344                 n_rx_nodesc_trunc += channel->n_rx_nodesc_trunc;
2345         stats[GENERIC_STAT_rx_nodesc_trunc] = n_rx_nodesc_trunc;
2346         stats[GENERIC_STAT_rx_noskb_drops] = atomic_read(&efx->n_rx_noskb_drops);
2347 }
2348
2349 bool efx_filter_spec_equal(const struct efx_filter_spec *left,
2350                            const struct efx_filter_spec *right)
2351 {
2352         if ((left->match_flags ^ right->match_flags) |
2353             ((left->flags ^ right->flags) &
2354              (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)))
2355                 return false;
2356
2357         return memcmp(&left->outer_vid, &right->outer_vid,
2358                       sizeof(struct efx_filter_spec) -
2359                       offsetof(struct efx_filter_spec, outer_vid)) == 0;
2360 }
2361
2362 u32 efx_filter_spec_hash(const struct efx_filter_spec *spec)
2363 {
2364         BUILD_BUG_ON(offsetof(struct efx_filter_spec, outer_vid) & 3);
2365         return jhash2((const u32 *)&spec->outer_vid,
2366                       (sizeof(struct efx_filter_spec) -
2367                        offsetof(struct efx_filter_spec, outer_vid)) / 4,
2368                       0);
2369 }
2370
2371 #ifdef CONFIG_RFS_ACCEL
2372 bool efx_rps_check_rule(struct efx_arfs_rule *rule, unsigned int filter_idx,
2373                         bool *force)
2374 {
2375         if (rule->filter_id == EFX_ARFS_FILTER_ID_PENDING) {
2376                 /* ARFS is currently updating this entry, leave it */
2377                 return false;
2378         }
2379         if (rule->filter_id == EFX_ARFS_FILTER_ID_ERROR) {
2380                 /* ARFS tried and failed to update this, so it's probably out
2381                  * of date.  Remove the filter and the ARFS rule entry.
2382                  */
2383                 rule->filter_id = EFX_ARFS_FILTER_ID_REMOVING;
2384                 *force = true;
2385                 return true;
2386         } else if (WARN_ON(rule->filter_id != filter_idx)) { /* can't happen */
2387                 /* ARFS has moved on, so old filter is not needed.  Since we did
2388                  * not mark the rule with EFX_ARFS_FILTER_ID_REMOVING, it will
2389                  * not be removed by efx_rps_hash_del() subsequently.
2390                  */
2391                 *force = true;
2392                 return true;
2393         }
2394         /* Remove it iff ARFS wants to. */
2395         return true;
2396 }
2397
2398 static
2399 struct hlist_head *efx_rps_hash_bucket(struct efx_nic *efx,
2400                                        const struct efx_filter_spec *spec)
2401 {
2402         u32 hash = efx_filter_spec_hash(spec);
2403
2404         lockdep_assert_held(&efx->rps_hash_lock);
2405         if (!efx->rps_hash_table)
2406                 return NULL;
2407         return &efx->rps_hash_table[hash % EFX_ARFS_HASH_TABLE_SIZE];
2408 }
2409
2410 struct efx_arfs_rule *efx_rps_hash_find(struct efx_nic *efx,
2411                                         const struct efx_filter_spec *spec)
2412 {
2413         struct efx_arfs_rule *rule;
2414         struct hlist_head *head;
2415         struct hlist_node *node;
2416
2417         head = efx_rps_hash_bucket(efx, spec);
2418         if (!head)
2419                 return NULL;
2420         hlist_for_each(node, head) {
2421                 rule = container_of(node, struct efx_arfs_rule, node);
2422                 if (efx_filter_spec_equal(spec, &rule->spec))
2423                         return rule;
2424         }
2425         return NULL;
2426 }
2427
2428 struct efx_arfs_rule *efx_rps_hash_add(struct efx_nic *efx,
2429                                        const struct efx_filter_spec *spec,
2430                                        bool *new)
2431 {
2432         struct efx_arfs_rule *rule;
2433         struct hlist_head *head;
2434         struct hlist_node *node;
2435
2436         head = efx_rps_hash_bucket(efx, spec);
2437         if (!head)
2438                 return NULL;
2439         hlist_for_each(node, head) {
2440                 rule = container_of(node, struct efx_arfs_rule, node);
2441                 if (efx_filter_spec_equal(spec, &rule->spec)) {
2442                         *new = false;
2443                         return rule;
2444                 }
2445         }
2446         rule = kmalloc(sizeof(*rule), GFP_ATOMIC);
2447         *new = true;
2448         if (rule) {
2449                 memcpy(&rule->spec, spec, sizeof(rule->spec));
2450                 hlist_add_head(&rule->node, head);
2451         }
2452         return rule;
2453 }
2454
2455 void efx_rps_hash_del(struct efx_nic *efx, const struct efx_filter_spec *spec)
2456 {
2457         struct efx_arfs_rule *rule;
2458         struct hlist_head *head;
2459         struct hlist_node *node;
2460
2461         head = efx_rps_hash_bucket(efx, spec);
2462         if (WARN_ON(!head))
2463                 return;
2464         hlist_for_each(node, head) {
2465                 rule = container_of(node, struct efx_arfs_rule, node);
2466                 if (efx_filter_spec_equal(spec, &rule->spec)) {
2467                         /* Someone already reused the entry.  We know that if
2468                          * this check doesn't fire (i.e. filter_id == REMOVING)
2469                          * then the REMOVING mark was put there by our caller,
2470                          * because caller is holding a lock on filter table and
2471                          * only holders of that lock set REMOVING.
2472                          */
2473                         if (rule->filter_id != EFX_ARFS_FILTER_ID_REMOVING)
2474                                 return;
2475                         hlist_del(node);
2476                         kfree(rule);
2477                         return;
2478                 }
2479         }
2480         /* We didn't find it. */
2481         WARN_ON(1);
2482 }
2483 #endif
2484
2485 /* RSS contexts.  We're using linked lists and crappy O(n) algorithms, because
2486  * (a) this is an infrequent control-plane operation and (b) n is small (max 64)
2487  */
2488 struct efx_rss_context *efx_alloc_rss_context_entry(struct efx_nic *efx)
2489 {
2490         struct list_head *head = &efx->rss_context.list;
2491         struct efx_rss_context *ctx, *new;
2492         u32 id = 1; /* Don't use zero, that refers to the master RSS context */
2493
2494         WARN_ON(!mutex_is_locked(&efx->rss_lock));
2495
2496         /* Search for first gap in the numbering */
2497         list_for_each_entry(ctx, head, list) {
2498                 if (ctx->user_id != id)
2499                         break;
2500                 id++;
2501                 /* Check for wrap.  If this happens, we have nearly 2^32
2502                  * allocated RSS contexts, which seems unlikely.
2503                  */
2504                 if (WARN_ON_ONCE(!id))
2505                         return NULL;
2506         }
2507
2508         /* Create the new entry */
2509         new = kmalloc(sizeof(struct efx_rss_context), GFP_KERNEL);
2510         if (!new)
2511                 return NULL;
2512         new->context_id = EFX_EF10_RSS_CONTEXT_INVALID;
2513         new->rx_hash_udp_4tuple = false;
2514
2515         /* Insert the new entry into the gap */
2516         new->user_id = id;
2517         list_add_tail(&new->list, &ctx->list);
2518         return new;
2519 }
2520
2521 struct efx_rss_context *efx_find_rss_context_entry(struct efx_nic *efx, u32 id)
2522 {
2523         struct list_head *head = &efx->rss_context.list;
2524         struct efx_rss_context *ctx;
2525
2526         WARN_ON(!mutex_is_locked(&efx->rss_lock));
2527
2528         list_for_each_entry(ctx, head, list)
2529                 if (ctx->user_id == id)
2530                         return ctx;
2531         return NULL;
2532 }
2533
2534 void efx_free_rss_context_entry(struct efx_rss_context *ctx)
2535 {
2536         list_del(&ctx->list);
2537         kfree(ctx);
2538 }
2539
2540 /**************************************************************************
2541  *
2542  * PCI interface
2543  *
2544  **************************************************************************/
2545
2546 /* Main body of final NIC shutdown code
2547  * This is called only at module unload (or hotplug removal).
2548  */
2549 static void efx_pci_remove_main(struct efx_nic *efx)
2550 {
2551         /* Flush reset_work. It can no longer be scheduled since we
2552          * are not READY.
2553          */
2554         BUG_ON(efx->state == STATE_READY);
2555         efx_flush_reset_workqueue(efx);
2556
2557         efx_disable_interrupts(efx);
2558         efx_clear_interrupt_affinity(efx);
2559         efx_nic_fini_interrupt(efx);
2560         efx_fini_port(efx);
2561         efx->type->fini(efx);
2562         efx_fini_napi(efx);
2563         efx_remove_all(efx);
2564 }
2565
2566 /* Final NIC shutdown
2567  * This is called only at module unload (or hotplug removal).  A PF can call
2568  * this on its VFs to ensure they are unbound first.
2569  */
2570 static void efx_pci_remove(struct pci_dev *pci_dev)
2571 {
2572         struct efx_nic *efx;
2573
2574         efx = pci_get_drvdata(pci_dev);
2575         if (!efx)
2576                 return;
2577
2578         /* Mark the NIC as fini, then stop the interface */
2579         rtnl_lock();
2580         efx_dissociate(efx);
2581         dev_close(efx->net_dev);
2582         efx_disable_interrupts(efx);
2583         efx->state = STATE_UNINIT;
2584         rtnl_unlock();
2585
2586         if (efx->type->sriov_fini)
2587                 efx->type->sriov_fini(efx);
2588
2589         efx_unregister_netdev(efx);
2590
2591         efx_mtd_remove(efx);
2592
2593         efx_pci_remove_main(efx);
2594
2595         efx_fini_io(efx, efx->type->mem_bar(efx));
2596         netif_dbg(efx, drv, efx->net_dev, "shutdown successful\n");
2597
2598         efx_fini_struct(efx);
2599         free_netdev(efx->net_dev);
2600
2601         pci_disable_pcie_error_reporting(pci_dev);
2602 };
2603
2604 /* NIC VPD information
2605  * Called during probe to display the part number of the
2606  * installed NIC.  VPD is potentially very large but this should
2607  * always appear within the first 512 bytes.
2608  */
2609 #define SFC_VPD_LEN 512
2610 static void efx_probe_vpd_strings(struct efx_nic *efx)
2611 {
2612         struct pci_dev *dev = efx->pci_dev;
2613         char vpd_data[SFC_VPD_LEN];
2614         ssize_t vpd_size;
2615         int ro_start, ro_size, i, j;
2616
2617         /* Get the vpd data from the device */
2618         vpd_size = pci_read_vpd(dev, 0, sizeof(vpd_data), vpd_data);
2619         if (vpd_size <= 0) {
2620                 netif_err(efx, drv, efx->net_dev, "Unable to read VPD\n");
2621                 return;
2622         }
2623
2624         /* Get the Read only section */
2625         ro_start = pci_vpd_find_tag(vpd_data, 0, vpd_size, PCI_VPD_LRDT_RO_DATA);
2626         if (ro_start < 0) {
2627                 netif_err(efx, drv, efx->net_dev, "VPD Read-only not found\n");
2628                 return;
2629         }
2630
2631         ro_size = pci_vpd_lrdt_size(&vpd_data[ro_start]);
2632         j = ro_size;
2633         i = ro_start + PCI_VPD_LRDT_TAG_SIZE;
2634         if (i + j > vpd_size)
2635                 j = vpd_size - i;
2636
2637         /* Get the Part number */
2638         i = pci_vpd_find_info_keyword(vpd_data, i, j, "PN");
2639         if (i < 0) {
2640                 netif_err(efx, drv, efx->net_dev, "Part number not found\n");
2641                 return;
2642         }
2643
2644         j = pci_vpd_info_field_size(&vpd_data[i]);
2645         i += PCI_VPD_INFO_FLD_HDR_SIZE;
2646         if (i + j > vpd_size) {
2647                 netif_err(efx, drv, efx->net_dev, "Incomplete part number\n");
2648                 return;
2649         }
2650
2651         netif_info(efx, drv, efx->net_dev,
2652                    "Part Number : %.*s\n", j, &vpd_data[i]);
2653
2654         i = ro_start + PCI_VPD_LRDT_TAG_SIZE;
2655         j = ro_size;
2656         i = pci_vpd_find_info_keyword(vpd_data, i, j, "SN");
2657         if (i < 0) {
2658                 netif_err(efx, drv, efx->net_dev, "Serial number not found\n");
2659                 return;
2660         }
2661
2662         j = pci_vpd_info_field_size(&vpd_data[i]);
2663         i += PCI_VPD_INFO_FLD_HDR_SIZE;
2664         if (i + j > vpd_size) {
2665                 netif_err(efx, drv, efx->net_dev, "Incomplete serial number\n");
2666                 return;
2667         }
2668
2669         efx->vpd_sn = kmalloc(j + 1, GFP_KERNEL);
2670         if (!efx->vpd_sn)
2671                 return;
2672
2673         snprintf(efx->vpd_sn, j + 1, "%s", &vpd_data[i]);
2674 }
2675
2676
2677 /* Main body of NIC initialisation
2678  * This is called at module load (or hotplug insertion, theoretically).
2679  */
2680 static int efx_pci_probe_main(struct efx_nic *efx)
2681 {
2682         int rc;
2683
2684         /* Do start-of-day initialisation */
2685         rc = efx_probe_all(efx);
2686         if (rc)
2687                 goto fail1;
2688
2689         efx_init_napi(efx);
2690
2691         down_write(&efx->filter_sem);
2692         rc = efx->type->init(efx);
2693         up_write(&efx->filter_sem);
2694         if (rc) {
2695                 netif_err(efx, probe, efx->net_dev,
2696                           "failed to initialise NIC\n");
2697                 goto fail3;
2698         }
2699
2700         rc = efx_init_port(efx);
2701         if (rc) {
2702                 netif_err(efx, probe, efx->net_dev,
2703                           "failed to initialise port\n");
2704                 goto fail4;
2705         }
2706
2707         rc = efx_nic_init_interrupt(efx);
2708         if (rc)
2709                 goto fail5;
2710
2711         efx_set_interrupt_affinity(efx);
2712         rc = efx_enable_interrupts(efx);
2713         if (rc)
2714                 goto fail6;
2715
2716         return 0;
2717
2718  fail6:
2719         efx_clear_interrupt_affinity(efx);
2720         efx_nic_fini_interrupt(efx);
2721  fail5:
2722         efx_fini_port(efx);
2723  fail4:
2724         efx->type->fini(efx);
2725  fail3:
2726         efx_fini_napi(efx);
2727         efx_remove_all(efx);
2728  fail1:
2729         return rc;
2730 }
2731
2732 static int efx_pci_probe_post_io(struct efx_nic *efx)
2733 {
2734         struct net_device *net_dev = efx->net_dev;
2735         int rc = efx_pci_probe_main(efx);
2736
2737         if (rc)
2738                 return rc;
2739
2740         if (efx->type->sriov_init) {
2741                 rc = efx->type->sriov_init(efx);
2742                 if (rc)
2743                         netif_err(efx, probe, efx->net_dev,
2744                                   "SR-IOV can't be enabled rc %d\n", rc);
2745         }
2746
2747         /* Determine netdevice features */
2748         net_dev->features |= (efx->type->offload_features | NETIF_F_SG |
2749                               NETIF_F_TSO | NETIF_F_RXCSUM | NETIF_F_RXALL);
2750         if (efx->type->offload_features & (NETIF_F_IPV6_CSUM | NETIF_F_HW_CSUM))
2751                 net_dev->features |= NETIF_F_TSO6;
2752         /* Check whether device supports TSO */
2753         if (!efx->type->tso_versions || !efx->type->tso_versions(efx))
2754                 net_dev->features &= ~NETIF_F_ALL_TSO;
2755         /* Mask for features that also apply to VLAN devices */
2756         net_dev->vlan_features |= (NETIF_F_HW_CSUM | NETIF_F_SG |
2757                                    NETIF_F_HIGHDMA | NETIF_F_ALL_TSO |
2758                                    NETIF_F_RXCSUM);
2759
2760         net_dev->hw_features |= net_dev->features & ~efx->fixed_features;
2761
2762         /* Disable receiving frames with bad FCS, by default. */
2763         net_dev->features &= ~NETIF_F_RXALL;
2764
2765         /* Disable VLAN filtering by default.  It may be enforced if
2766          * the feature is fixed (i.e. VLAN filters are required to
2767          * receive VLAN tagged packets due to vPort restrictions).
2768          */
2769         net_dev->features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
2770         net_dev->features |= efx->fixed_features;
2771
2772         rc = efx_register_netdev(efx);
2773         if (!rc)
2774                 return 0;
2775
2776         efx_pci_remove_main(efx);
2777         return rc;
2778 }
2779
2780 /* NIC initialisation
2781  *
2782  * This is called at module load (or hotplug insertion,
2783  * theoretically).  It sets up PCI mappings, resets the NIC,
2784  * sets up and registers the network devices with the kernel and hooks
2785  * the interrupt service routine.  It does not prepare the device for
2786  * transmission; this is left to the first time one of the network
2787  * interfaces is brought up (i.e. efx_net_open).
2788  */
2789 static int efx_pci_probe(struct pci_dev *pci_dev,
2790                          const struct pci_device_id *entry)
2791 {
2792         struct net_device *net_dev;
2793         struct efx_nic *efx;
2794         int rc;
2795
2796         /* Allocate and initialise a struct net_device and struct efx_nic */
2797         net_dev = alloc_etherdev_mqs(sizeof(*efx), EFX_MAX_CORE_TX_QUEUES,
2798                                      EFX_MAX_RX_QUEUES);
2799         if (!net_dev)
2800                 return -ENOMEM;
2801         efx = netdev_priv(net_dev);
2802         efx->type = (const struct efx_nic_type *) entry->driver_data;
2803         efx->fixed_features |= NETIF_F_HIGHDMA;
2804
2805         pci_set_drvdata(pci_dev, efx);
2806         SET_NETDEV_DEV(net_dev, &pci_dev->dev);
2807         rc = efx_init_struct(efx, pci_dev, net_dev);
2808         if (rc)
2809                 goto fail1;
2810
2811         netif_info(efx, probe, efx->net_dev,
2812                    "Solarflare NIC detected\n");
2813
2814         if (!efx->type->is_vf)
2815                 efx_probe_vpd_strings(efx);
2816
2817         /* Set up basic I/O (BAR mappings etc) */
2818         rc = efx_init_io(efx, efx->type->mem_bar(efx), efx->type->max_dma_mask,
2819                          efx->type->mem_map_size(efx));
2820         if (rc)
2821                 goto fail2;
2822
2823         rc = efx_pci_probe_post_io(efx);
2824         if (rc) {
2825                 /* On failure, retry once immediately.
2826                  * If we aborted probe due to a scheduled reset, dismiss it.
2827                  */
2828                 efx->reset_pending = 0;
2829                 rc = efx_pci_probe_post_io(efx);
2830                 if (rc) {
2831                         /* On another failure, retry once more
2832                          * after a 50-305ms delay.
2833                          */
2834                         unsigned char r;
2835
2836                         get_random_bytes(&r, 1);
2837                         msleep((unsigned int)r + 50);
2838                         efx->reset_pending = 0;
2839                         rc = efx_pci_probe_post_io(efx);
2840                 }
2841         }
2842         if (rc)
2843                 goto fail3;
2844
2845         netif_dbg(efx, probe, efx->net_dev, "initialisation successful\n");
2846
2847         /* Try to create MTDs, but allow this to fail */
2848         rtnl_lock();
2849         rc = efx_mtd_probe(efx);
2850         rtnl_unlock();
2851         if (rc && rc != -EPERM)
2852                 netif_warn(efx, probe, efx->net_dev,
2853                            "failed to create MTDs (%d)\n", rc);
2854
2855         (void)pci_enable_pcie_error_reporting(pci_dev);
2856
2857         if (efx->type->udp_tnl_push_ports)
2858                 efx->type->udp_tnl_push_ports(efx);
2859
2860         return 0;
2861
2862  fail3:
2863         efx_fini_io(efx, efx->type->mem_bar(efx));
2864  fail2:
2865         efx_fini_struct(efx);
2866  fail1:
2867         WARN_ON(rc > 0);
2868         netif_dbg(efx, drv, efx->net_dev, "initialisation failed. rc=%d\n", rc);
2869         free_netdev(net_dev);
2870         return rc;
2871 }
2872
2873 /* efx_pci_sriov_configure returns the actual number of Virtual Functions
2874  * enabled on success
2875  */
2876 #ifdef CONFIG_SFC_SRIOV
2877 static int efx_pci_sriov_configure(struct pci_dev *dev, int num_vfs)
2878 {
2879         int rc;
2880         struct efx_nic *efx = pci_get_drvdata(dev);
2881
2882         if (efx->type->sriov_configure) {
2883                 rc = efx->type->sriov_configure(efx, num_vfs);
2884                 if (rc)
2885                         return rc;
2886                 else
2887                         return num_vfs;
2888         } else
2889                 return -EOPNOTSUPP;
2890 }
2891 #endif
2892
2893 static int efx_pm_freeze(struct device *dev)
2894 {
2895         struct efx_nic *efx = dev_get_drvdata(dev);
2896
2897         rtnl_lock();
2898
2899         if (efx->state != STATE_DISABLED) {
2900                 efx->state = STATE_UNINIT;
2901
2902                 efx_device_detach_sync(efx);
2903
2904                 efx_stop_all(efx);
2905                 efx_disable_interrupts(efx);
2906         }
2907
2908         rtnl_unlock();
2909
2910         return 0;
2911 }
2912
2913 static int efx_pm_thaw(struct device *dev)
2914 {
2915         int rc;
2916         struct efx_nic *efx = dev_get_drvdata(dev);
2917
2918         rtnl_lock();
2919
2920         if (efx->state != STATE_DISABLED) {
2921                 rc = efx_enable_interrupts(efx);
2922                 if (rc)
2923                         goto fail;
2924
2925                 mutex_lock(&efx->mac_lock);
2926                 efx->phy_op->reconfigure(efx);
2927                 mutex_unlock(&efx->mac_lock);
2928
2929                 efx_start_all(efx);
2930
2931                 efx_device_attach_if_not_resetting(efx);
2932
2933                 efx->state = STATE_READY;
2934
2935                 efx->type->resume_wol(efx);
2936         }
2937
2938         rtnl_unlock();
2939
2940         /* Reschedule any quenched resets scheduled during efx_pm_freeze() */
2941         efx_queue_reset_work(efx);
2942
2943         return 0;
2944
2945 fail:
2946         rtnl_unlock();
2947
2948         return rc;
2949 }
2950
2951 static int efx_pm_poweroff(struct device *dev)
2952 {
2953         struct pci_dev *pci_dev = to_pci_dev(dev);
2954         struct efx_nic *efx = pci_get_drvdata(pci_dev);
2955
2956         efx->type->fini(efx);
2957
2958         efx->reset_pending = 0;
2959
2960         pci_save_state(pci_dev);
2961         return pci_set_power_state(pci_dev, PCI_D3hot);
2962 }
2963
2964 /* Used for both resume and restore */
2965 static int efx_pm_resume(struct device *dev)
2966 {
2967         struct pci_dev *pci_dev = to_pci_dev(dev);
2968         struct efx_nic *efx = pci_get_drvdata(pci_dev);
2969         int rc;
2970
2971         rc = pci_set_power_state(pci_dev, PCI_D0);
2972         if (rc)
2973                 return rc;
2974         pci_restore_state(pci_dev);
2975         rc = pci_enable_device(pci_dev);
2976         if (rc)
2977                 return rc;
2978         pci_set_master(efx->pci_dev);
2979         rc = efx->type->reset(efx, RESET_TYPE_ALL);
2980         if (rc)
2981                 return rc;
2982         down_write(&efx->filter_sem);
2983         rc = efx->type->init(efx);
2984         up_write(&efx->filter_sem);
2985         if (rc)
2986                 return rc;
2987         rc = efx_pm_thaw(dev);
2988         return rc;
2989 }
2990
2991 static int efx_pm_suspend(struct device *dev)
2992 {
2993         int rc;
2994
2995         efx_pm_freeze(dev);
2996         rc = efx_pm_poweroff(dev);
2997         if (rc)
2998                 efx_pm_resume(dev);
2999         return rc;
3000 }
3001
3002 static const struct dev_pm_ops efx_pm_ops = {
3003         .suspend        = efx_pm_suspend,
3004         .resume         = efx_pm_resume,
3005         .freeze         = efx_pm_freeze,
3006         .thaw           = efx_pm_thaw,
3007         .poweroff       = efx_pm_poweroff,
3008         .restore        = efx_pm_resume,
3009 };
3010
3011 /* A PCI error affecting this device was detected.
3012  * At this point MMIO and DMA may be disabled.
3013  * Stop the software path and request a slot reset.
3014  */
3015 static pci_ers_result_t efx_io_error_detected(struct pci_dev *pdev,
3016                                               enum pci_channel_state state)
3017 {
3018         pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED;
3019         struct efx_nic *efx = pci_get_drvdata(pdev);
3020
3021         if (state == pci_channel_io_perm_failure)
3022                 return PCI_ERS_RESULT_DISCONNECT;
3023
3024         rtnl_lock();
3025
3026         if (efx->state != STATE_DISABLED) {
3027                 efx->state = STATE_RECOVERY;
3028                 efx->reset_pending = 0;
3029
3030                 efx_device_detach_sync(efx);
3031
3032                 efx_stop_all(efx);
3033                 efx_disable_interrupts(efx);
3034
3035                 status = PCI_ERS_RESULT_NEED_RESET;
3036         } else {
3037                 /* If the interface is disabled we don't want to do anything
3038                  * with it.
3039                  */
3040                 status = PCI_ERS_RESULT_RECOVERED;
3041         }
3042
3043         rtnl_unlock();
3044
3045         pci_disable_device(pdev);
3046
3047         return status;
3048 }
3049
3050 /* Fake a successful reset, which will be performed later in efx_io_resume. */
3051 static pci_ers_result_t efx_io_slot_reset(struct pci_dev *pdev)
3052 {
3053         struct efx_nic *efx = pci_get_drvdata(pdev);
3054         pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED;
3055
3056         if (pci_enable_device(pdev)) {
3057                 netif_err(efx, hw, efx->net_dev,
3058                           "Cannot re-enable PCI device after reset.\n");
3059                 status =  PCI_ERS_RESULT_DISCONNECT;
3060         }
3061
3062         return status;
3063 }
3064
3065 /* Perform the actual reset and resume I/O operations. */
3066 static void efx_io_resume(struct pci_dev *pdev)
3067 {
3068         struct efx_nic *efx = pci_get_drvdata(pdev);
3069         int rc;
3070
3071         rtnl_lock();
3072
3073         if (efx->state == STATE_DISABLED)
3074                 goto out;
3075
3076         rc = efx_reset(efx, RESET_TYPE_ALL);
3077         if (rc) {
3078                 netif_err(efx, hw, efx->net_dev,
3079                           "efx_reset failed after PCI error (%d)\n", rc);
3080         } else {
3081                 efx->state = STATE_READY;
3082                 netif_dbg(efx, hw, efx->net_dev,
3083                           "Done resetting and resuming IO after PCI error.\n");
3084         }
3085
3086 out:
3087         rtnl_unlock();
3088 }
3089
3090 /* For simplicity and reliability, we always require a slot reset and try to
3091  * reset the hardware when a pci error affecting the device is detected.
3092  * We leave both the link_reset and mmio_enabled callback unimplemented:
3093  * with our request for slot reset the mmio_enabled callback will never be
3094  * called, and the link_reset callback is not used by AER or EEH mechanisms.
3095  */
3096 static const struct pci_error_handlers efx_err_handlers = {
3097         .error_detected = efx_io_error_detected,
3098         .slot_reset     = efx_io_slot_reset,
3099         .resume         = efx_io_resume,
3100 };
3101
3102 static struct pci_driver efx_pci_driver = {
3103         .name           = KBUILD_MODNAME,
3104         .id_table       = efx_pci_table,
3105         .probe          = efx_pci_probe,
3106         .remove         = efx_pci_remove,
3107         .driver.pm      = &efx_pm_ops,
3108         .err_handler    = &efx_err_handlers,
3109 #ifdef CONFIG_SFC_SRIOV
3110         .sriov_configure = efx_pci_sriov_configure,
3111 #endif
3112 };
3113
3114 /**************************************************************************
3115  *
3116  * Kernel module interface
3117  *
3118  *************************************************************************/
3119
3120 module_param(interrupt_mode, uint, 0444);
3121 MODULE_PARM_DESC(interrupt_mode,
3122                  "Interrupt mode (0=>MSIX 1=>MSI 2=>legacy)");
3123
3124 static int __init efx_init_module(void)
3125 {
3126         int rc;
3127
3128         printk(KERN_INFO "Solarflare NET driver v" EFX_DRIVER_VERSION "\n");
3129
3130         rc = register_netdevice_notifier(&efx_netdev_notifier);
3131         if (rc)
3132                 goto err_notifier;
3133
3134 #ifdef CONFIG_SFC_SRIOV
3135         rc = efx_init_sriov();
3136         if (rc)
3137                 goto err_sriov;
3138 #endif
3139
3140         rc = efx_create_reset_workqueue();
3141         if (rc)
3142                 goto err_reset;
3143
3144         rc = pci_register_driver(&efx_pci_driver);
3145         if (rc < 0)
3146                 goto err_pci;
3147
3148         return 0;
3149
3150  err_pci:
3151         efx_destroy_reset_workqueue();
3152  err_reset:
3153 #ifdef CONFIG_SFC_SRIOV
3154         efx_fini_sriov();
3155  err_sriov:
3156 #endif
3157         unregister_netdevice_notifier(&efx_netdev_notifier);
3158  err_notifier:
3159         return rc;
3160 }
3161
3162 static void __exit efx_exit_module(void)
3163 {
3164         printk(KERN_INFO "Solarflare NET driver unloading\n");
3165
3166         pci_unregister_driver(&efx_pci_driver);
3167         efx_destroy_reset_workqueue();
3168 #ifdef CONFIG_SFC_SRIOV
3169         efx_fini_sriov();
3170 #endif
3171         unregister_netdevice_notifier(&efx_netdev_notifier);
3172
3173 }
3174
3175 module_init(efx_init_module);
3176 module_exit(efx_exit_module);
3177
3178 MODULE_AUTHOR("Solarflare Communications and "
3179               "Michael Brown <mbrown@fensystems.co.uk>");
3180 MODULE_DESCRIPTION("Solarflare network driver");
3181 MODULE_LICENSE("GPL");
3182 MODULE_DEVICE_TABLE(pci, efx_pci_table);
3183 MODULE_VERSION(EFX_DRIVER_VERSION);