drivers/net/ethernet/intel/igc/igc_main.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c)  2018 Intel Corporation */
   3
   4 #include <linux/module.h>
   5 #include <linux/types.h>
   6 #include <linux/if_vlan.h>
   7 #include <linux/aer.h>
   8
   9 #include "igc.h"
  10 #include "igc_hw.h"
  11
  12 #define DRV_VERSION     "0.0.1-k"
  13 #define DRV_SUMMARY     "Intel(R) 2.5G Ethernet Linux Driver"
  14
  15 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
  16
  17 static int debug = -1;
  18
  19 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
  20 MODULE_DESCRIPTION(DRV_SUMMARY);
  21 MODULE_LICENSE("GPL v2");
  22 MODULE_VERSION(DRV_VERSION);
  23 module_param(debug, int, 0);
  24 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
  25
  26 char igc_driver_name[] = "igc";
  27 char igc_driver_version[] = DRV_VERSION;
  28 static const char igc_driver_string[] = DRV_SUMMARY;
  29 static const char igc_copyright[] =
  30         "Copyright(c) 2018 Intel Corporation.";
  31
  32 static const struct igc_info *igc_info_tbl[] = {
  33         [board_base] = &igc_base_info,
  34 };
  35
  36 static const struct pci_device_id igc_pci_tbl[] = {
  37         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LM), board_base },
  38         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_V), board_base },
  39         /* required last entry */
  40         {0, }
  41 };
  42
  43 MODULE_DEVICE_TABLE(pci, igc_pci_tbl);
  44
  45 /* forward declaration */
  46 static void igc_clean_tx_ring(struct igc_ring *tx_ring);
  47 static int igc_sw_init(struct igc_adapter *);
  48 static void igc_configure(struct igc_adapter *adapter);
  49 static void igc_power_down_link(struct igc_adapter *adapter);
  50 static void igc_set_default_mac_filter(struct igc_adapter *adapter);
  51 static void igc_set_rx_mode(struct net_device *netdev);
  52 static void igc_write_itr(struct igc_q_vector *q_vector);
  53 static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector);
  54 static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx);
  55 static void igc_set_interrupt_capability(struct igc_adapter *adapter,
  56                                          bool msix);
  57 static void igc_free_q_vectors(struct igc_adapter *adapter);
  58 static void igc_irq_disable(struct igc_adapter *adapter);
  59 static void igc_irq_enable(struct igc_adapter *adapter);
  60 static void igc_configure_msix(struct igc_adapter *adapter);
  61 static bool igc_alloc_mapped_page(struct igc_ring *rx_ring,
  62                                   struct igc_rx_buffer *bi);
  63
  64 enum latency_range {
  65         lowest_latency = 0,
  66         low_latency = 1,
  67         bulk_latency = 2,
  68         latency_invalid = 255
  69 };
  70
  71 void igc_reset(struct igc_adapter *adapter)
  72 {
  73         struct pci_dev *pdev = adapter->pdev;
  74         struct igc_hw *hw = &adapter->hw;
  75
  76         hw->mac.ops.reset_hw(hw);
  77
  78         if (hw->mac.ops.init_hw(hw))
  79                 dev_err(&pdev->dev, "Hardware Error\n");
  80
  81         if (!netif_running(adapter->netdev))
  82                 igc_power_down_link(adapter);
  83
  84         igc_get_phy_info(hw);
  85 }
  86
  87 /**
  88  * igc_power_up_link - Power up the phy/serdes link
  89  * @adapter: address of board private structure
  90  */
  91 static void igc_power_up_link(struct igc_adapter *adapter)
  92 {
  93         igc_reset_phy(&adapter->hw);
  94
  95         if (adapter->hw.phy.media_type == igc_media_type_copper)
  96                 igc_power_up_phy_copper(&adapter->hw);
  97
  98         igc_setup_link(&adapter->hw);
  99 }
 100
 101 /**
 102  * igc_power_down_link - Power down the phy/serdes link
 103  * @adapter: address of board private structure
 104  */
 105 static void igc_power_down_link(struct igc_adapter *adapter)
 106 {
 107         if (adapter->hw.phy.media_type == igc_media_type_copper)
 108                 igc_power_down_phy_copper_base(&adapter->hw);
 109 }
 110
 111 /**
 112  * igc_release_hw_control - release control of the h/w to f/w
 113  * @adapter: address of board private structure
 114  *
 115  * igc_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
 116  * For ASF and Pass Through versions of f/w this means that the
 117  * driver is no longer loaded.
 118  */
 119 static void igc_release_hw_control(struct igc_adapter *adapter)
 120 {
 121         struct igc_hw *hw = &adapter->hw;
 122         u32 ctrl_ext;
 123
 124         /* Let firmware take over control of h/w */
 125         ctrl_ext = rd32(IGC_CTRL_EXT);
 126         wr32(IGC_CTRL_EXT,
 127              ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD);
 128 }
 129
 130 /**
 131  * igc_get_hw_control - get control of the h/w from f/w
 132  * @adapter: address of board private structure
 133  *
 134  * igc_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
 135  * For ASF and Pass Through versions of f/w this means that
 136  * the driver is loaded.
 137  */
 138 static void igc_get_hw_control(struct igc_adapter *adapter)
 139 {
 140         struct igc_hw *hw = &adapter->hw;
 141         u32 ctrl_ext;
 142
 143         /* Let firmware know the driver has taken over */
 144         ctrl_ext = rd32(IGC_CTRL_EXT);
 145         wr32(IGC_CTRL_EXT,
 146              ctrl_ext | IGC_CTRL_EXT_DRV_LOAD);
 147 }
 148
 149 /**
 150  * igc_free_tx_resources - Free Tx Resources per Queue
 151  * @tx_ring: Tx descriptor ring for a specific queue
 152  *
 153  * Free all transmit software resources
 154  */
 155 void igc_free_tx_resources(struct igc_ring *tx_ring)
 156 {
 157         igc_clean_tx_ring(tx_ring);
 158
 159         vfree(tx_ring->tx_buffer_info);
 160         tx_ring->tx_buffer_info = NULL;
 161
 162         /* if not set, then don't free */
 163         if (!tx_ring->desc)
 164                 return;
 165
 166         dma_free_coherent(tx_ring->dev, tx_ring->size,
 167                           tx_ring->desc, tx_ring->dma);
 168
 169         tx_ring->desc = NULL;
 170 }
 171
 172 /**
 173  * igc_free_all_tx_resources - Free Tx Resources for All Queues
 174  * @adapter: board private structure
 175  *
 176  * Free all transmit software resources
 177  */
 178 static void igc_free_all_tx_resources(struct igc_adapter *adapter)
 179 {
 180         int i;
 181
 182         for (i = 0; i < adapter->num_tx_queues; i++)
 183                 igc_free_tx_resources(adapter->tx_ring[i]);
 184 }
 185
 186 /**
 187  * igc_clean_tx_ring - Free Tx Buffers
 188  * @tx_ring: ring to be cleaned
 189  */
 190 static void igc_clean_tx_ring(struct igc_ring *tx_ring)
 191 {
 192         u16 i = tx_ring->next_to_clean;
 193         struct igc_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i];
 194
 195         while (i != tx_ring->next_to_use) {
 196                 union igc_adv_tx_desc *eop_desc, *tx_desc;
 197
 198                 /* Free all the Tx ring sk_buffs */
 199                 dev_kfree_skb_any(tx_buffer->skb);
 200
 201                 /* unmap skb header data */
 202                 dma_unmap_single(tx_ring->dev,
 203                                  dma_unmap_addr(tx_buffer, dma),
 204                                  dma_unmap_len(tx_buffer, len),
 205                                  DMA_TO_DEVICE);
 206
 207                 /* check for eop_desc to determine the end of the packet */
 208                 eop_desc = tx_buffer->next_to_watch;
 209                 tx_desc = IGC_TX_DESC(tx_ring, i);
 210
 211                 /* unmap remaining buffers */
 212                 while (tx_desc != eop_desc) {
 213                         tx_buffer++;
 214                         tx_desc++;
 215                         i++;
 216                         if (unlikely(i == tx_ring->count)) {
 217                                 i = 0;
 218                                 tx_buffer = tx_ring->tx_buffer_info;
 219                                 tx_desc = IGC_TX_DESC(tx_ring, 0);
 220                         }
 221
 222                         /* unmap any remaining paged data */
 223                         if (dma_unmap_len(tx_buffer, len))
 224                                 dma_unmap_page(tx_ring->dev,
 225                                                dma_unmap_addr(tx_buffer, dma),
 226                                                dma_unmap_len(tx_buffer, len),
 227                                                DMA_TO_DEVICE);
 228                 }
 229
 230                 /* move us one more past the eop_desc for start of next pkt */
 231                 tx_buffer++;
 232                 i++;
 233                 if (unlikely(i == tx_ring->count)) {
 234                         i = 0;
 235                         tx_buffer = tx_ring->tx_buffer_info;
 236                 }
 237         }
 238
 239         /* reset BQL for queue */
 240         netdev_tx_reset_queue(txring_txq(tx_ring));
 241
 242         /* reset next_to_use and next_to_clean */
 243         tx_ring->next_to_use = 0;
 244         tx_ring->next_to_clean = 0;
 245 }
 246
 247 /**
 248  * igc_clean_all_tx_rings - Free Tx Buffers for all queues
 249  * @adapter: board private structure
 250  */
 251 static void igc_clean_all_tx_rings(struct igc_adapter *adapter)
 252 {
 253         int i;
 254
 255         for (i = 0; i < adapter->num_tx_queues; i++)
 256                 if (adapter->tx_ring[i])
 257                         igc_clean_tx_ring(adapter->tx_ring[i]);
 258 }
 259
 260 /**
 261  * igc_setup_tx_resources - allocate Tx resources (Descriptors)
 262  * @tx_ring: tx descriptor ring (for a specific queue) to setup
 263  *
 264  * Return 0 on success, negative on failure
 265  */
 266 int igc_setup_tx_resources(struct igc_ring *tx_ring)
 267 {
 268         struct device *dev = tx_ring->dev;
 269         int size = 0;
 270
 271         size = sizeof(struct igc_tx_buffer) * tx_ring->count;
 272         tx_ring->tx_buffer_info = vzalloc(size);
 273         if (!tx_ring->tx_buffer_info)
 274                 goto err;
 275
 276         /* round up to nearest 4K */
 277         tx_ring->size = tx_ring->count * sizeof(union igc_adv_tx_desc);
 278         tx_ring->size = ALIGN(tx_ring->size, 4096);
 279
 280         tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
 281                                            &tx_ring->dma, GFP_KERNEL);
 282
 283         if (!tx_ring->desc)
 284                 goto err;
 285
 286         tx_ring->next_to_use = 0;
 287         tx_ring->next_to_clean = 0;
 288
 289         return 0;
 290
 291 err:
 292         vfree(tx_ring->tx_buffer_info);
 293         dev_err(dev,
 294                 "Unable to allocate memory for the transmit descriptor ring\n");
 295         return -ENOMEM;
 296 }
 297
 298 /**
 299  * igc_setup_all_tx_resources - wrapper to allocate Tx resources for all queues
 300  * @adapter: board private structure
 301  *
 302  * Return 0 on success, negative on failure
 303  */
 304 static int igc_setup_all_tx_resources(struct igc_adapter *adapter)
 305 {
 306         struct pci_dev *pdev = adapter->pdev;
 307         int i, err = 0;
 308
 309         for (i = 0; i < adapter->num_tx_queues; i++) {
 310                 err = igc_setup_tx_resources(adapter->tx_ring[i]);
 311                 if (err) {
 312                         dev_err(&pdev->dev,
 313                                 "Allocation for Tx Queue %u failed\n", i);
 314                         for (i--; i >= 0; i--)
 315                                 igc_free_tx_resources(adapter->tx_ring[i]);
 316                         break;
 317                 }
 318         }
 319
 320         return err;
 321 }
 322
 323 /**
 324  * igc_clean_rx_ring - Free Rx Buffers per Queue
 325  * @rx_ring: ring to free buffers from
 326  */
 327 static void igc_clean_rx_ring(struct igc_ring *rx_ring)
 328 {
 329         u16 i = rx_ring->next_to_clean;
 330
 331         if (rx_ring->skb)
 332                 dev_kfree_skb(rx_ring->skb);
 333         rx_ring->skb = NULL;
 334
 335         /* Free all the Rx ring sk_buffs */
 336         while (i != rx_ring->next_to_alloc) {
 337                 struct igc_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
 338
 339                 /* Invalidate cache lines that may have been written to by
 340                  * device so that we avoid corrupting memory.
 341                  */
 342                 dma_sync_single_range_for_cpu(rx_ring->dev,
 343                                               buffer_info->dma,
 344                                               buffer_info->page_offset,
 345                                               igc_rx_bufsz(rx_ring),
 346                                               DMA_FROM_DEVICE);
 347
 348                 /* free resources associated with mapping */
 349                 dma_unmap_page_attrs(rx_ring->dev,
 350                                      buffer_info->dma,
 351                                      igc_rx_pg_size(rx_ring),
 352                                      DMA_FROM_DEVICE,
 353                                      IGC_RX_DMA_ATTR);
 354                 __page_frag_cache_drain(buffer_info->page,
 355                                         buffer_info->pagecnt_bias);
 356
 357                 i++;
 358                 if (i == rx_ring->count)
 359                         i = 0;
 360         }
 361
 362         rx_ring->next_to_alloc = 0;
 363         rx_ring->next_to_clean = 0;
 364         rx_ring->next_to_use = 0;
 365 }
 366
 367 /**
 368  * igc_clean_all_rx_rings - Free Rx Buffers for all queues
 369  * @adapter: board private structure
 370  */
 371 static void igc_clean_all_rx_rings(struct igc_adapter *adapter)
 372 {
 373         int i;
 374
 375         for (i = 0; i < adapter->num_rx_queues; i++)
 376                 if (adapter->rx_ring[i])
 377                         igc_clean_rx_ring(adapter->rx_ring[i]);
 378 }
 379
 380 /**
 381  * igc_free_rx_resources - Free Rx Resources
 382  * @rx_ring: ring to clean the resources from
 383  *
 384  * Free all receive software resources
 385  */
 386 void igc_free_rx_resources(struct igc_ring *rx_ring)
 387 {
 388         igc_clean_rx_ring(rx_ring);
 389
 390         vfree(rx_ring->rx_buffer_info);
 391         rx_ring->rx_buffer_info = NULL;
 392
 393         /* if not set, then don't free */
 394         if (!rx_ring->desc)
 395                 return;
 396
 397         dma_free_coherent(rx_ring->dev, rx_ring->size,
 398                           rx_ring->desc, rx_ring->dma);
 399
 400         rx_ring->desc = NULL;
 401 }
 402
 403 /**
 404  * igc_free_all_rx_resources - Free Rx Resources for All Queues
 405  * @adapter: board private structure
 406  *
 407  * Free all receive software resources
 408  */
 409 static void igc_free_all_rx_resources(struct igc_adapter *adapter)
 410 {
 411         int i;
 412
 413         for (i = 0; i < adapter->num_rx_queues; i++)
 414                 igc_free_rx_resources(adapter->rx_ring[i]);
 415 }
 416
 417 /**
 418  * igc_setup_rx_resources - allocate Rx resources (Descriptors)
 419  * @rx_ring:    rx descriptor ring (for a specific queue) to setup
 420  *
 421  * Returns 0 on success, negative on failure
 422  */
 423 int igc_setup_rx_resources(struct igc_ring *rx_ring)
 424 {
 425         struct device *dev = rx_ring->dev;
 426         int size, desc_len;
 427
 428         size = sizeof(struct igc_rx_buffer) * rx_ring->count;
 429         rx_ring->rx_buffer_info = vzalloc(size);
 430         if (!rx_ring->rx_buffer_info)
 431                 goto err;
 432
 433         desc_len = sizeof(union igc_adv_rx_desc);
 434
 435         /* Round up to nearest 4K */
 436         rx_ring->size = rx_ring->count * desc_len;
 437         rx_ring->size = ALIGN(rx_ring->size, 4096);
 438
 439         rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
 440                                            &rx_ring->dma, GFP_KERNEL);
 441
 442         if (!rx_ring->desc)
 443                 goto err;
 444
 445         rx_ring->next_to_alloc = 0;
 446         rx_ring->next_to_clean = 0;
 447         rx_ring->next_to_use = 0;
 448
 449         return 0;
 450
 451 err:
 452         vfree(rx_ring->rx_buffer_info);
 453         rx_ring->rx_buffer_info = NULL;
 454         dev_err(dev,
 455                 "Unable to allocate memory for the receive descriptor ring\n");
 456         return -ENOMEM;
 457 }
 458
 459 /**
 460  * igc_setup_all_rx_resources - wrapper to allocate Rx resources
 461  *                                (Descriptors) for all queues
 462  * @adapter: board private structure
 463  *
 464  * Return 0 on success, negative on failure
 465  */
 466 static int igc_setup_all_rx_resources(struct igc_adapter *adapter)
 467 {
 468         struct pci_dev *pdev = adapter->pdev;
 469         int i, err = 0;
 470
 471         for (i = 0; i < adapter->num_rx_queues; i++) {
 472                 err = igc_setup_rx_resources(adapter->rx_ring[i]);
 473                 if (err) {
 474                         dev_err(&pdev->dev,
 475                                 "Allocation for Rx Queue %u failed\n", i);
 476                         for (i--; i >= 0; i--)
 477                                 igc_free_rx_resources(adapter->rx_ring[i]);
 478                         break;
 479                 }
 480         }
 481
 482         return err;
 483 }
 484
 485 /**
 486  * igc_configure_rx_ring - Configure a receive ring after Reset
 487  * @adapter: board private structure
 488  * @ring: receive ring to be configured
 489  *
 490  * Configure the Rx unit of the MAC after a reset.
 491  */
 492 static void igc_configure_rx_ring(struct igc_adapter *adapter,
 493                                   struct igc_ring *ring)
 494 {
 495         struct igc_hw *hw = &adapter->hw;
 496         union igc_adv_rx_desc *rx_desc;
 497         int reg_idx = ring->reg_idx;
 498         u32 srrctl = 0, rxdctl = 0;
 499         u64 rdba = ring->dma;
 500
 501         /* disable the queue */
 502         wr32(IGC_RXDCTL(reg_idx), 0);
 503
 504         /* Set DMA base address registers */
 505         wr32(IGC_RDBAL(reg_idx),
 506              rdba & 0x00000000ffffffffULL);
 507         wr32(IGC_RDBAH(reg_idx), rdba >> 32);
 508         wr32(IGC_RDLEN(reg_idx),
 509              ring->count * sizeof(union igc_adv_rx_desc));
 510
 511         /* initialize head and tail */
 512         ring->tail = adapter->io_addr + IGC_RDT(reg_idx);
 513         wr32(IGC_RDH(reg_idx), 0);
 514         writel(0, ring->tail);
 515
 516         /* reset next-to- use/clean to place SW in sync with hardware */
 517         ring->next_to_clean = 0;
 518         ring->next_to_use = 0;
 519
 520         /* set descriptor configuration */
 521         srrctl = IGC_RX_HDR_LEN << IGC_SRRCTL_BSIZEHDRSIZE_SHIFT;
 522         if (ring_uses_large_buffer(ring))
 523                 srrctl |= IGC_RXBUFFER_3072 >> IGC_SRRCTL_BSIZEPKT_SHIFT;
 524         else
 525                 srrctl |= IGC_RXBUFFER_2048 >> IGC_SRRCTL_BSIZEPKT_SHIFT;
 526         srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF;
 527
 528         wr32(IGC_SRRCTL(reg_idx), srrctl);
 529
 530         rxdctl |= IGC_RX_PTHRESH;
 531         rxdctl |= IGC_RX_HTHRESH << 8;
 532         rxdctl |= IGC_RX_WTHRESH << 16;
 533
 534         /* initialize rx_buffer_info */
 535         memset(ring->rx_buffer_info, 0,
 536                sizeof(struct igc_rx_buffer) * ring->count);
 537
 538         /* initialize Rx descriptor 0 */
 539         rx_desc = IGC_RX_DESC(ring, 0);
 540         rx_desc->wb.upper.length = 0;
 541
 542         /* enable receive descriptor fetching */
 543         rxdctl |= IGC_RXDCTL_QUEUE_ENABLE;
 544
 545         wr32(IGC_RXDCTL(reg_idx), rxdctl);
 546 }
 547
 548 /**
 549  * igc_configure_rx - Configure receive Unit after Reset
 550  * @adapter: board private structure
 551  *
 552  * Configure the Rx unit of the MAC after a reset.
 553  */
 554 static void igc_configure_rx(struct igc_adapter *adapter)
 555 {
 556         int i;
 557
 558         /* Setup the HW Rx Head and Tail Descriptor Pointers and
 559          * the Base and Length of the Rx Descriptor Ring
 560          */
 561         for (i = 0; i < adapter->num_rx_queues; i++)
 562                 igc_configure_rx_ring(adapter, adapter->rx_ring[i]);
 563 }
 564
 565 /**
 566  * igc_configure_tx_ring - Configure transmit ring after Reset
 567  * @adapter: board private structure
 568  * @ring: tx ring to configure
 569  *
 570  * Configure a transmit ring after a reset.
 571  */
 572 static void igc_configure_tx_ring(struct igc_adapter *adapter,
 573                                   struct igc_ring *ring)
 574 {
 575         struct igc_hw *hw = &adapter->hw;
 576         int reg_idx = ring->reg_idx;
 577         u64 tdba = ring->dma;
 578         u32 txdctl = 0;
 579
 580         /* disable the queue */
 581         wr32(IGC_TXDCTL(reg_idx), 0);
 582         wrfl();
 583         mdelay(10);
 584
 585         wr32(IGC_TDLEN(reg_idx),
 586              ring->count * sizeof(union igc_adv_tx_desc));
 587         wr32(IGC_TDBAL(reg_idx),
 588              tdba & 0x00000000ffffffffULL);
 589         wr32(IGC_TDBAH(reg_idx), tdba >> 32);
 590
 591         ring->tail = adapter->io_addr + IGC_TDT(reg_idx);
 592         wr32(IGC_TDH(reg_idx), 0);
 593         writel(0, ring->tail);
 594
 595         txdctl |= IGC_TX_PTHRESH;
 596         txdctl |= IGC_TX_HTHRESH << 8;
 597         txdctl |= IGC_TX_WTHRESH << 16;
 598
 599         txdctl |= IGC_TXDCTL_QUEUE_ENABLE;
 600         wr32(IGC_TXDCTL(reg_idx), txdctl);
 601 }
 602
 603 /**
 604  * igc_configure_tx - Configure transmit Unit after Reset
 605  * @adapter: board private structure
 606  *
 607  * Configure the Tx unit of the MAC after a reset.
 608  */
 609 static void igc_configure_tx(struct igc_adapter *adapter)
 610 {
 611         int i;
 612
 613         for (i = 0; i < adapter->num_tx_queues; i++)
 614                 igc_configure_tx_ring(adapter, adapter->tx_ring[i]);
 615 }
 616
 617 /**
 618  * igc_setup_mrqc - configure the multiple receive queue control registers
 619  * @adapter: Board private structure
 620  */
 621 static void igc_setup_mrqc(struct igc_adapter *adapter)
 622 {
 623         struct igc_hw *hw = &adapter->hw;
 624         u32 j, num_rx_queues;
 625         u32 mrqc, rxcsum;
 626         u32 rss_key[10];
 627
 628         netdev_rss_key_fill(rss_key, sizeof(rss_key));
 629         for (j = 0; j < 10; j++)
 630                 wr32(IGC_RSSRK(j), rss_key[j]);
 631
 632         num_rx_queues = adapter->rss_queues;
 633
 634         if (adapter->rss_indir_tbl_init != num_rx_queues) {
 635                 for (j = 0; j < IGC_RETA_SIZE; j++)
 636                         adapter->rss_indir_tbl[j] =
 637                         (j * num_rx_queues) / IGC_RETA_SIZE;
 638                 adapter->rss_indir_tbl_init = num_rx_queues;
 639         }
 640         igc_write_rss_indir_tbl(adapter);
 641
 642         /* Disable raw packet checksumming so that RSS hash is placed in
 643          * descriptor on writeback.  No need to enable TCP/UDP/IP checksum
 644          * offloads as they are enabled by default
 645          */
 646         rxcsum = rd32(IGC_RXCSUM);
 647         rxcsum |= IGC_RXCSUM_PCSD;
 648
 649         /* Enable Receive Checksum Offload for SCTP */
 650         rxcsum |= IGC_RXCSUM_CRCOFL;
 651
 652         /* Don't need to set TUOFL or IPOFL, they default to 1 */
 653         wr32(IGC_RXCSUM, rxcsum);
 654
 655         /* Generate RSS hash based on packet types, TCP/UDP
 656          * port numbers and/or IPv4/v6 src and dst addresses
 657          */
 658         mrqc = IGC_MRQC_RSS_FIELD_IPV4 |
 659                IGC_MRQC_RSS_FIELD_IPV4_TCP |
 660                IGC_MRQC_RSS_FIELD_IPV6 |
 661                IGC_MRQC_RSS_FIELD_IPV6_TCP |
 662                IGC_MRQC_RSS_FIELD_IPV6_TCP_EX;
 663
 664         if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV4_UDP)
 665                 mrqc |= IGC_MRQC_RSS_FIELD_IPV4_UDP;
 666         if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV6_UDP)
 667                 mrqc |= IGC_MRQC_RSS_FIELD_IPV6_UDP;
 668
 669         mrqc |= IGC_MRQC_ENABLE_RSS_MQ;
 670
 671         wr32(IGC_MRQC, mrqc);
 672 }
 673
 674 /**
 675  * igc_setup_rctl - configure the receive control registers
 676  * @adapter: Board private structure
 677  */
 678 static void igc_setup_rctl(struct igc_adapter *adapter)
 679 {
 680         struct igc_hw *hw = &adapter->hw;
 681         u32 rctl;
 682
 683         rctl = rd32(IGC_RCTL);
 684
 685         rctl &= ~(3 << IGC_RCTL_MO_SHIFT);
 686         rctl &= ~(IGC_RCTL_LBM_TCVR | IGC_RCTL_LBM_MAC);
 687
 688         rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_RDMTS_HALF |
 689                 (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT);
 690
 691         /* enable stripping of CRC. Newer features require
 692          * that the HW strips the CRC.
 693          */
 694         rctl |= IGC_RCTL_SECRC;
 695
 696         /* disable store bad packets and clear size bits. */
 697         rctl &= ~(IGC_RCTL_SBP | IGC_RCTL_SZ_256);
 698
 699         /* enable LPE to allow for reception of jumbo frames */
 700         rctl |= IGC_RCTL_LPE;
 701
 702         /* disable queue 0 to prevent tail write w/o re-config */
 703         wr32(IGC_RXDCTL(0), 0);
 704
 705         /* This is useful for sniffing bad packets. */
 706         if (adapter->netdev->features & NETIF_F_RXALL) {
 707                 /* UPE and MPE will be handled by normal PROMISC logic
 708                  * in set_rx_mode
 709                  */
 710                 rctl |= (IGC_RCTL_SBP | /* Receive bad packets */
 711                          IGC_RCTL_BAM | /* RX All Bcast Pkts */
 712                          IGC_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
 713
 714                 rctl &= ~(IGC_RCTL_DPF | /* Allow filtered pause */
 715                           IGC_RCTL_CFIEN); /* Disable VLAN CFIEN Filter */
 716         }
 717
 718         wr32(IGC_RCTL, rctl);
 719 }
 720
 721 /**
 722  * igc_setup_tctl - configure the transmit control registers
 723  * @adapter: Board private structure
 724  */
 725 static void igc_setup_tctl(struct igc_adapter *adapter)
 726 {
 727         struct igc_hw *hw = &adapter->hw;
 728         u32 tctl;
 729
 730         /* disable queue 0 which icould be enabled by default */
 731         wr32(IGC_TXDCTL(0), 0);
 732
 733         /* Program the Transmit Control Register */
 734         tctl = rd32(IGC_TCTL);
 735         tctl &= ~IGC_TCTL_CT;
 736         tctl |= IGC_TCTL_PSP | IGC_TCTL_RTLC |
 737                 (IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT);
 738
 739         /* Enable transmits */
 740         tctl |= IGC_TCTL_EN;
 741
 742         wr32(IGC_TCTL, tctl);
 743 }
 744
 745 /**
 746  * igc_set_mac - Change the Ethernet Address of the NIC
 747  * @netdev: network interface device structure
 748  * @p: pointer to an address structure
 749  *
 750  * Returns 0 on success, negative on failure
 751  */
 752 static int igc_set_mac(struct net_device *netdev, void *p)
 753 {
 754         struct igc_adapter *adapter = netdev_priv(netdev);
 755         struct igc_hw *hw = &adapter->hw;
 756         struct sockaddr *addr = p;
 757
 758         if (!is_valid_ether_addr(addr->sa_data))
 759                 return -EADDRNOTAVAIL;
 760
 761         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
 762         memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
 763
 764         /* set the correct pool for the new PF MAC address in entry 0 */
 765         igc_set_default_mac_filter(adapter);
 766
 767         return 0;
 768 }
 769
 770 static void igc_tx_csum(struct igc_ring *tx_ring, struct igc_tx_buffer *first)
 771 {
 772 }
 773
 774 static int __igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size)
 775 {
 776         struct net_device *netdev = tx_ring->netdev;
 777
 778         netif_stop_subqueue(netdev, tx_ring->queue_index);
 779
 780         /* memory barriier comment */
 781         smp_mb();
 782
 783         /* We need to check again in a case another CPU has just
 784          * made room available.
 785          */
 786         if (igc_desc_unused(tx_ring) < size)
 787                 return -EBUSY;
 788
 789         /* A reprieve! */
 790         netif_wake_subqueue(netdev, tx_ring->queue_index);
 791
 792         u64_stats_update_begin(&tx_ring->tx_syncp2);
 793         tx_ring->tx_stats.restart_queue2++;
 794         u64_stats_update_end(&tx_ring->tx_syncp2);
 795
 796         return 0;
 797 }
 798
 799 static inline int igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size)
 800 {
 801         if (igc_desc_unused(tx_ring) >= size)
 802                 return 0;
 803         return __igc_maybe_stop_tx(tx_ring, size);
 804 }
 805
 806 static u32 igc_tx_cmd_type(struct sk_buff *skb, u32 tx_flags)
 807 {
 808         /* set type for advanced descriptor with frame checksum insertion */
 809         u32 cmd_type = IGC_ADVTXD_DTYP_DATA |
 810                        IGC_ADVTXD_DCMD_DEXT |
 811                        IGC_ADVTXD_DCMD_IFCS;
 812
 813         return cmd_type;
 814 }
 815
 816 static void igc_tx_olinfo_status(struct igc_ring *tx_ring,
 817                                  union igc_adv_tx_desc *tx_desc,
 818                                  u32 tx_flags, unsigned int paylen)
 819 {
 820         u32 olinfo_status = paylen << IGC_ADVTXD_PAYLEN_SHIFT;
 821
 822         /* insert L4 checksum */
 823         olinfo_status |= (tx_flags & IGC_TX_FLAGS_CSUM) *
 824                           ((IGC_TXD_POPTS_TXSM << 8) /
 825                           IGC_TX_FLAGS_CSUM);
 826
 827         /* insert IPv4 checksum */
 828         olinfo_status |= (tx_flags & IGC_TX_FLAGS_IPV4) *
 829                           (((IGC_TXD_POPTS_IXSM << 8)) /
 830                           IGC_TX_FLAGS_IPV4);
 831
 832         tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
 833 }
 834
 835 static int igc_tx_map(struct igc_ring *tx_ring,
 836                       struct igc_tx_buffer *first,
 837                       const u8 hdr_len)
 838 {
 839         struct sk_buff *skb = first->skb;
 840         struct igc_tx_buffer *tx_buffer;
 841         union igc_adv_tx_desc *tx_desc;
 842         u32 tx_flags = first->tx_flags;
 843         struct skb_frag_struct *frag;
 844         u16 i = tx_ring->next_to_use;
 845         unsigned int data_len, size;
 846         dma_addr_t dma;
 847         u32 cmd_type = igc_tx_cmd_type(skb, tx_flags);
 848
 849         tx_desc = IGC_TX_DESC(tx_ring, i);
 850
 851         igc_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len);
 852
 853         size = skb_headlen(skb);
 854         data_len = skb->data_len;
 855
 856         dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
 857
 858         tx_buffer = first;
 859
 860         for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
 861                 if (dma_mapping_error(tx_ring->dev, dma))
 862                         goto dma_error;
 863
 864                 /* record length, and DMA address */
 865                 dma_unmap_len_set(tx_buffer, len, size);
 866                 dma_unmap_addr_set(tx_buffer, dma, dma);
 867
 868                 tx_desc->read.buffer_addr = cpu_to_le64(dma);
 869
 870                 while (unlikely(size > IGC_MAX_DATA_PER_TXD)) {
 871                         tx_desc->read.cmd_type_len =
 872                                 cpu_to_le32(cmd_type ^ IGC_MAX_DATA_PER_TXD);
 873
 874                         i++;
 875                         tx_desc++;
 876                         if (i == tx_ring->count) {
 877                                 tx_desc = IGC_TX_DESC(tx_ring, 0);
 878                                 i = 0;
 879                         }
 880                         tx_desc->read.olinfo_status = 0;
 881
 882                         dma += IGC_MAX_DATA_PER_TXD;
 883                         size -= IGC_MAX_DATA_PER_TXD;
 884
 885                         tx_desc->read.buffer_addr = cpu_to_le64(dma);
 886                 }
 887
 888                 if (likely(!data_len))
 889                         break;
 890
 891                 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size);
 892
 893                 i++;
 894                 tx_desc++;
 895                 if (i == tx_ring->count) {
 896                         tx_desc = IGC_TX_DESC(tx_ring, 0);
 897                         i = 0;
 898                 }
 899                 tx_desc->read.olinfo_status = 0;
 900
 901                 size = skb_frag_size(frag);
 902                 data_len -= size;
 903
 904                 dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
 905                                        size, DMA_TO_DEVICE);
 906
 907                 tx_buffer = &tx_ring->tx_buffer_info[i];
 908         }
 909
 910         /* write last descriptor with RS and EOP bits */
 911         cmd_type |= size | IGC_TXD_DCMD;
 912         tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
 913
 914         netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
 915
 916         /* set the timestamp */
 917         first->time_stamp = jiffies;
 918
 919         skb_tx_timestamp(skb);
 920
 921         /* Force memory writes to complete before letting h/w know there
 922          * are new descriptors to fetch.  (Only applicable for weak-ordered
 923          * memory model archs, such as IA-64).
 924          *
 925          * We also need this memory barrier to make certain all of the
 926          * status bits have been updated before next_to_watch is written.
 927          */
 928         wmb();
 929
 930         /* set next_to_watch value indicating a packet is present */
 931         first->next_to_watch = tx_desc;
 932
 933         i++;
 934         if (i == tx_ring->count)
 935                 i = 0;
 936
 937         tx_ring->next_to_use = i;
 938
 939         /* Make sure there is space in the ring for the next send. */
 940         igc_maybe_stop_tx(tx_ring, DESC_NEEDED);
 941
 942         if (netif_xmit_stopped(txring_txq(tx_ring)) || !skb->xmit_more) {
 943                 writel(i, tx_ring->tail);
 944
 945                 /* we need this if more than one processor can write to our tail
 946                  * at a time, it synchronizes IO on IA64/Altix systems
 947                  */
 948                 mmiowb();
 949         }
 950
 951         return 0;
 952 dma_error:
 953         dev_err(tx_ring->dev, "TX DMA map failed\n");
 954         tx_buffer = &tx_ring->tx_buffer_info[i];
 955
 956         /* clear dma mappings for failed tx_buffer_info map */
 957         while (tx_buffer != first) {
 958                 if (dma_unmap_len(tx_buffer, len))
 959                         dma_unmap_page(tx_ring->dev,
 960                                        dma_unmap_addr(tx_buffer, dma),
 961                                        dma_unmap_len(tx_buffer, len),
 962                                        DMA_TO_DEVICE);
 963                 dma_unmap_len_set(tx_buffer, len, 0);
 964
 965                 if (i-- == 0)
 966                         i += tx_ring->count;
 967                 tx_buffer = &tx_ring->tx_buffer_info[i];
 968         }
 969
 970         if (dma_unmap_len(tx_buffer, len))
 971                 dma_unmap_single(tx_ring->dev,
 972                                  dma_unmap_addr(tx_buffer, dma),
 973                                  dma_unmap_len(tx_buffer, len),
 974                                  DMA_TO_DEVICE);
 975         dma_unmap_len_set(tx_buffer, len, 0);
 976
 977         dev_kfree_skb_any(tx_buffer->skb);
 978         tx_buffer->skb = NULL;
 979
 980         tx_ring->next_to_use = i;
 981
 982         return -1;
 983 }
 984
 985 static netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb,
 986                                        struct igc_ring *tx_ring)
 987 {
 988         u16 count = TXD_USE_COUNT(skb_headlen(skb));
 989         __be16 protocol = vlan_get_protocol(skb);
 990         struct igc_tx_buffer *first;
 991         u32 tx_flags = 0;
 992         unsigned short f;
 993         u8 hdr_len = 0;
 994
 995         /* need: 1 descriptor per page * PAGE_SIZE/IGC_MAX_DATA_PER_TXD,
 996          *      + 1 desc for skb_headlen/IGC_MAX_DATA_PER_TXD,
 997          *      + 2 desc gap to keep tail from touching head,
 998          *      + 1 desc for context descriptor,
 999          * otherwise try next time
1000          */
1001         for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
1002                 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
1003
1004         if (igc_maybe_stop_tx(tx_ring, count + 3)) {
1005                 /* this is a hard error */
1006                 return NETDEV_TX_BUSY;
1007         }
1008
1009         /* record the location of the first descriptor for this packet */
1010         first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
1011         first->skb = skb;
1012         first->bytecount = skb->len;
1013         first->gso_segs = 1;
1014
1015         /* record initial flags and protocol */
1016         first->tx_flags = tx_flags;
1017         first->protocol = protocol;
1018
1019         igc_tx_csum(tx_ring, first);
1020
1021         igc_tx_map(tx_ring, first, hdr_len);
1022
1023         return NETDEV_TX_OK;
1024 }
1025
1026 static inline struct igc_ring *igc_tx_queue_mapping(struct igc_adapter *adapter,
1027                                                     struct sk_buff *skb)
1028 {
1029         unsigned int r_idx = skb->queue_mapping;
1030
1031         if (r_idx >= adapter->num_tx_queues)
1032                 r_idx = r_idx % adapter->num_tx_queues;
1033
1034         return adapter->tx_ring[r_idx];
1035 }
1036
1037 static netdev_tx_t igc_xmit_frame(struct sk_buff *skb,
1038                                   struct net_device *netdev)
1039 {
1040         struct igc_adapter *adapter = netdev_priv(netdev);
1041
1042         /* The minimum packet size with TCTL.PSP set is 17 so pad the skb
1043          * in order to meet this minimum size requirement.
1044          */
1045         if (skb->len < 17) {
1046                 if (skb_padto(skb, 17))
1047                         return NETDEV_TX_OK;
1048                 skb->len = 17;
1049         }
1050
1051         return igc_xmit_frame_ring(skb, igc_tx_queue_mapping(adapter, skb));
1052 }
1053
1054 static inline void igc_rx_hash(struct igc_ring *ring,
1055                                union igc_adv_rx_desc *rx_desc,
1056                                struct sk_buff *skb)
1057 {
1058         if (ring->netdev->features & NETIF_F_RXHASH)
1059                 skb_set_hash(skb,
1060                              le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),
1061                              PKT_HASH_TYPE_L3);
1062 }
1063
1064 /**
1065  * igc_process_skb_fields - Populate skb header fields from Rx descriptor
1066  * @rx_ring: rx descriptor ring packet is being transacted on
1067  * @rx_desc: pointer to the EOP Rx descriptor
1068  * @skb: pointer to current skb being populated
1069  *
1070  * This function checks the ring, descriptor, and packet information in
1071  * order to populate the hash, checksum, VLAN, timestamp, protocol, and
1072  * other fields within the skb.
1073  */
1074 static void igc_process_skb_fields(struct igc_ring *rx_ring,
1075                                    union igc_adv_rx_desc *rx_desc,
1076                                    struct sk_buff *skb)
1077 {
1078         igc_rx_hash(rx_ring, rx_desc, skb);
1079
1080         skb_record_rx_queue(skb, rx_ring->queue_index);
1081
1082         skb->protocol = eth_type_trans(skb, rx_ring->netdev);
1083 }
1084
1085 static struct igc_rx_buffer *igc_get_rx_buffer(struct igc_ring *rx_ring,
1086                                                const unsigned int size)
1087 {
1088         struct igc_rx_buffer *rx_buffer;
1089
1090         rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
1091         prefetchw(rx_buffer->page);
1092
1093         /* we are reusing so sync this buffer for CPU use */
1094         dma_sync_single_range_for_cpu(rx_ring->dev,
1095                                       rx_buffer->dma,
1096                                       rx_buffer->page_offset,
1097                                       size,
1098                                       DMA_FROM_DEVICE);
1099
1100         rx_buffer->pagecnt_bias--;
1101
1102         return rx_buffer;
1103 }
1104
1105 /**
1106  * igc_add_rx_frag - Add contents of Rx buffer to sk_buff
1107  * @rx_ring: rx descriptor ring to transact packets on
1108  * @rx_buffer: buffer containing page to add
1109  * @skb: sk_buff to place the data into
1110  * @size: size of buffer to be added
1111  *
1112  * This function will add the data contained in rx_buffer->page to the skb.
1113  */
1114 static void igc_add_rx_frag(struct igc_ring *rx_ring,
1115                             struct igc_rx_buffer *rx_buffer,
1116                             struct sk_buff *skb,
1117                             unsigned int size)
1118 {
1119 #if (PAGE_SIZE < 8192)
1120         unsigned int truesize = igc_rx_pg_size(rx_ring) / 2;
1121
1122         skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
1123                         rx_buffer->page_offset, size, truesize);
1124         rx_buffer->page_offset ^= truesize;
1125 #else
1126         unsigned int truesize = ring_uses_build_skb(rx_ring) ?
1127                                 SKB_DATA_ALIGN(IGC_SKB_PAD + size) :
1128                                 SKB_DATA_ALIGN(size);
1129         skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
1130                         rx_buffer->page_offset, size, truesize);
1131         rx_buffer->page_offset += truesize;
1132 #endif
1133 }
1134
1135 static struct sk_buff *igc_build_skb(struct igc_ring *rx_ring,
1136                                      struct igc_rx_buffer *rx_buffer,
1137                                      union igc_adv_rx_desc *rx_desc,
1138                                      unsigned int size)
1139 {
1140         void *va = page_address(rx_buffer->page) + rx_buffer->page_offset;
1141 #if (PAGE_SIZE < 8192)
1142         unsigned int truesize = igc_rx_pg_size(rx_ring) / 2;
1143 #else
1144         unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
1145                                 SKB_DATA_ALIGN(IGC_SKB_PAD + size);
1146 #endif
1147         struct sk_buff *skb;
1148
1149         /* prefetch first cache line of first page */
1150         prefetch(va);
1151 #if L1_CACHE_BYTES < 128
1152         prefetch(va + L1_CACHE_BYTES);
1153 #endif
1154
1155         /* build an skb around the page buffer */
1156         skb = build_skb(va - IGC_SKB_PAD, truesize);
1157         if (unlikely(!skb))
1158                 return NULL;
1159
1160         /* update pointers within the skb to store the data */
1161         skb_reserve(skb, IGC_SKB_PAD);
1162         __skb_put(skb, size);
1163
1164         /* update buffer offset */
1165 #if (PAGE_SIZE < 8192)
1166         rx_buffer->page_offset ^= truesize;
1167 #else
1168         rx_buffer->page_offset += truesize;
1169 #endif
1170
1171         return skb;
1172 }
1173
1174 static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring,
1175                                          struct igc_rx_buffer *rx_buffer,
1176                                          union igc_adv_rx_desc *rx_desc,
1177                                          unsigned int size)
1178 {
1179         void *va = page_address(rx_buffer->page) + rx_buffer->page_offset;
1180 #if (PAGE_SIZE < 8192)
1181         unsigned int truesize = igc_rx_pg_size(rx_ring) / 2;
1182 #else
1183         unsigned int truesize = SKB_DATA_ALIGN(size);
1184 #endif
1185         unsigned int headlen;
1186         struct sk_buff *skb;
1187
1188         /* prefetch first cache line of first page */
1189         prefetch(va);
1190 #if L1_CACHE_BYTES < 128
1191         prefetch(va + L1_CACHE_BYTES);
1192 #endif
1193
1194         /* allocate a skb to store the frags */
1195         skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGC_RX_HDR_LEN);
1196         if (unlikely(!skb))
1197                 return NULL;
1198
1199         /* Determine available headroom for copy */
1200         headlen = size;
1201         if (headlen > IGC_RX_HDR_LEN)
1202                 headlen = eth_get_headlen(va, IGC_RX_HDR_LEN);
1203
1204         /* align pull length to size of long to optimize memcpy performance */
1205         memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long)));
1206
1207         /* update all of the pointers */
1208         size -= headlen;
1209         if (size) {
1210                 skb_add_rx_frag(skb, 0, rx_buffer->page,
1211                                 (va + headlen) - page_address(rx_buffer->page),
1212                                 size, truesize);
1213 #if (PAGE_SIZE < 8192)
1214                 rx_buffer->page_offset ^= truesize;
1215 #else
1216                 rx_buffer->page_offset += truesize;
1217 #endif
1218         } else {
1219                 rx_buffer->pagecnt_bias++;
1220         }
1221
1222         return skb;
1223 }
1224
1225 /**
1226  * igc_reuse_rx_page - page flip buffer and store it back on the ring
1227  * @rx_ring: rx descriptor ring to store buffers on
1228  * @old_buff: donor buffer to have page reused
1229  *
1230  * Synchronizes page for reuse by the adapter
1231  */
1232 static void igc_reuse_rx_page(struct igc_ring *rx_ring,
1233                               struct igc_rx_buffer *old_buff)
1234 {
1235         u16 nta = rx_ring->next_to_alloc;
1236         struct igc_rx_buffer *new_buff;
1237
1238         new_buff = &rx_ring->rx_buffer_info[nta];
1239
1240         /* update, and store next to alloc */
1241         nta++;
1242         rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
1243
1244         /* Transfer page from old buffer to new buffer.
1245          * Move each member individually to avoid possible store
1246          * forwarding stalls.
1247          */
1248         new_buff->dma           = old_buff->dma;
1249         new_buff->page          = old_buff->page;
1250         new_buff->page_offset   = old_buff->page_offset;
1251         new_buff->pagecnt_bias  = old_buff->pagecnt_bias;
1252 }
1253
1254 static inline bool igc_page_is_reserved(struct page *page)
1255 {
1256         return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page);
1257 }
1258
1259 static bool igc_can_reuse_rx_page(struct igc_rx_buffer *rx_buffer)
1260 {
1261         unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
1262         struct page *page = rx_buffer->page;
1263
1264         /* avoid re-using remote pages */
1265         if (unlikely(igc_page_is_reserved(page)))
1266                 return false;
1267
1268 #if (PAGE_SIZE < 8192)
1269         /* if we are only owner of page we can reuse it */
1270         if (unlikely((page_ref_count(page) - pagecnt_bias) > 1))
1271                 return false;
1272 #else
1273 #define IGC_LAST_OFFSET \
1274         (SKB_WITH_OVERHEAD(PAGE_SIZE) - IGC_RXBUFFER_2048)
1275
1276         if (rx_buffer->page_offset > IGC_LAST_OFFSET)
1277                 return false;
1278 #endif
1279
1280         /* If we have drained the page fragment pool we need to update
1281          * the pagecnt_bias and page count so that we fully restock the
1282          * number of references the driver holds.
1283          */
1284         if (unlikely(!pagecnt_bias)) {
1285                 page_ref_add(page, USHRT_MAX);
1286                 rx_buffer->pagecnt_bias = USHRT_MAX;
1287         }
1288
1289         return true;
1290 }
1291
1292 /**
1293  * igc_is_non_eop - process handling of non-EOP buffers
1294  * @rx_ring: Rx ring being processed
1295  * @rx_desc: Rx descriptor for current buffer
1296  * @skb: current socket buffer containing buffer in progress
1297  *
1298  * This function updates next to clean.  If the buffer is an EOP buffer
1299  * this function exits returning false, otherwise it will place the
1300  * sk_buff in the next buffer to be chained and return true indicating
1301  * that this is in fact a non-EOP buffer.
1302  */
1303 static bool igc_is_non_eop(struct igc_ring *rx_ring,
1304                            union igc_adv_rx_desc *rx_desc)
1305 {
1306         u32 ntc = rx_ring->next_to_clean + 1;
1307
1308         /* fetch, update, and store next to clean */
1309         ntc = (ntc < rx_ring->count) ? ntc : 0;
1310         rx_ring->next_to_clean = ntc;
1311
1312         prefetch(IGC_RX_DESC(rx_ring, ntc));
1313
1314         if (likely(igc_test_staterr(rx_desc, IGC_RXD_STAT_EOP)))
1315                 return false;
1316
1317         return true;
1318 }
1319
1320 /**
1321  * igc_cleanup_headers - Correct corrupted or empty headers
1322  * @rx_ring: rx descriptor ring packet is being transacted on
1323  * @rx_desc: pointer to the EOP Rx descriptor
1324  * @skb: pointer to current skb being fixed
1325  *
1326  * Address the case where we are pulling data in on pages only
1327  * and as such no data is present in the skb header.
1328  *
1329  * In addition if skb is not at least 60 bytes we need to pad it so that
1330  * it is large enough to qualify as a valid Ethernet frame.
1331  *
1332  * Returns true if an error was encountered and skb was freed.
1333  */
1334 static bool igc_cleanup_headers(struct igc_ring *rx_ring,
1335                                 union igc_adv_rx_desc *rx_desc,
1336                                 struct sk_buff *skb)
1337 {
1338         if (unlikely((igc_test_staterr(rx_desc,
1339                                        IGC_RXDEXT_ERR_FRAME_ERR_MASK)))) {
1340                 struct net_device *netdev = rx_ring->netdev;
1341
1342                 if (!(netdev->features & NETIF_F_RXALL)) {
1343                         dev_kfree_skb_any(skb);
1344                         return true;
1345                 }
1346         }
1347
1348         /* if eth_skb_pad returns an error the skb was freed */
1349         if (eth_skb_pad(skb))
1350                 return true;
1351
1352         return false;
1353 }
1354
1355 static void igc_put_rx_buffer(struct igc_ring *rx_ring,
1356                               struct igc_rx_buffer *rx_buffer)
1357 {
1358         if (igc_can_reuse_rx_page(rx_buffer)) {
1359                 /* hand second half of page back to the ring */
1360                 igc_reuse_rx_page(rx_ring, rx_buffer);
1361         } else {
1362                 /* We are not reusing the buffer so unmap it and free
1363                  * any references we are holding to it
1364                  */
1365                 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
1366                                      igc_rx_pg_size(rx_ring), DMA_FROM_DEVICE,
1367                                      IGC_RX_DMA_ATTR);
1368                 __page_frag_cache_drain(rx_buffer->page,
1369                                         rx_buffer->pagecnt_bias);
1370         }
1371
1372         /* clear contents of rx_buffer */
1373         rx_buffer->page = NULL;
1374 }
1375
1376 /**
1377  * igc_alloc_rx_buffers - Replace used receive buffers; packet split
1378  * @adapter: address of board private structure
1379  */
1380 static void igc_alloc_rx_buffers(struct igc_ring *rx_ring, u16 cleaned_count)
1381 {
1382         union igc_adv_rx_desc *rx_desc;
1383         u16 i = rx_ring->next_to_use;
1384         struct igc_rx_buffer *bi;
1385         u16 bufsz;
1386
1387         /* nothing to do */
1388         if (!cleaned_count)
1389                 return;
1390
1391         rx_desc = IGC_RX_DESC(rx_ring, i);
1392         bi = &rx_ring->rx_buffer_info[i];
1393         i -= rx_ring->count;
1394
1395         bufsz = igc_rx_bufsz(rx_ring);
1396
1397         do {
1398                 if (!igc_alloc_mapped_page(rx_ring, bi))
1399                         break;
1400
1401                 /* sync the buffer for use by the device */
1402                 dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
1403                                                  bi->page_offset, bufsz,
1404                                                  DMA_FROM_DEVICE);
1405
1406                 /* Refresh the desc even if buffer_addrs didn't change
1407                  * because each write-back erases this info.
1408                  */
1409                 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
1410
1411                 rx_desc++;
1412                 bi++;
1413                 i++;
1414                 if (unlikely(!i)) {
1415                         rx_desc = IGC_RX_DESC(rx_ring, 0);
1416                         bi = rx_ring->rx_buffer_info;
1417                         i -= rx_ring->count;
1418                 }
1419
1420                 /* clear the length for the next_to_use descriptor */
1421                 rx_desc->wb.upper.length = 0;
1422
1423                 cleaned_count--;
1424         } while (cleaned_count);
1425
1426         i += rx_ring->count;
1427
1428         if (rx_ring->next_to_use != i) {
1429                 /* record the next descriptor to use */
1430                 rx_ring->next_to_use = i;
1431
1432                 /* update next to alloc since we have filled the ring */
1433                 rx_ring->next_to_alloc = i;
1434
1435                 /* Force memory writes to complete before letting h/w
1436                  * know there are new descriptors to fetch.  (Only
1437                  * applicable for weak-ordered memory model archs,
1438                  * such as IA-64).
1439                  */
1440                 wmb();
1441                 writel(i, rx_ring->tail);
1442         }
1443 }
1444
1445 static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget)
1446 {
1447         unsigned int total_bytes = 0, total_packets = 0;
1448         struct igc_ring *rx_ring = q_vector->rx.ring;
1449         struct sk_buff *skb = rx_ring->skb;
1450         u16 cleaned_count = igc_desc_unused(rx_ring);
1451
1452         while (likely(total_packets < budget)) {
1453                 union igc_adv_rx_desc *rx_desc;
1454                 struct igc_rx_buffer *rx_buffer;
1455                 unsigned int size;
1456
1457                 /* return some buffers to hardware, one at a time is too slow */
1458                 if (cleaned_count >= IGC_RX_BUFFER_WRITE) {
1459                         igc_alloc_rx_buffers(rx_ring, cleaned_count);
1460                         cleaned_count = 0;
1461                 }
1462
1463                 rx_desc = IGC_RX_DESC(rx_ring, rx_ring->next_to_clean);
1464                 size = le16_to_cpu(rx_desc->wb.upper.length);
1465                 if (!size)
1466                         break;
1467
1468                 /* This memory barrier is needed to keep us from reading
1469                  * any other fields out of the rx_desc until we know the
1470                  * descriptor has been written back
1471                  */
1472                 dma_rmb();
1473
1474                 rx_buffer = igc_get_rx_buffer(rx_ring, size);
1475
1476                 /* retrieve a buffer from the ring */
1477                 if (skb)
1478                         igc_add_rx_frag(rx_ring, rx_buffer, skb, size);
1479                 else if (ring_uses_build_skb(rx_ring))
1480                         skb = igc_build_skb(rx_ring, rx_buffer, rx_desc, size);
1481                 else
1482                         skb = igc_construct_skb(rx_ring, rx_buffer,
1483                                                 rx_desc, size);
1484
1485                 /* exit if we failed to retrieve a buffer */
1486                 if (!skb) {
1487                         rx_ring->rx_stats.alloc_failed++;
1488                         rx_buffer->pagecnt_bias++;
1489                         break;
1490                 }
1491
1492                 igc_put_rx_buffer(rx_ring, rx_buffer);
1493                 cleaned_count++;
1494
1495                 /* fetch next buffer in frame if non-eop */
1496                 if (igc_is_non_eop(rx_ring, rx_desc))
1497                         continue;
1498
1499                 /* verify the packet layout is correct */
1500                 if (igc_cleanup_headers(rx_ring, rx_desc, skb)) {
1501                         skb = NULL;
1502                         continue;
1503                 }
1504
1505                 /* probably a little skewed due to removing CRC */
1506                 total_bytes += skb->len;
1507
1508                 /* populate checksum, timestamp, VLAN, and protocol */
1509                 igc_process_skb_fields(rx_ring, rx_desc, skb);
1510
1511                 napi_gro_receive(&q_vector->napi, skb);
1512
1513                 /* reset skb pointer */
1514                 skb = NULL;
1515
1516                 /* update budget accounting */
1517                 total_packets++;
1518         }
1519
1520         /* place incomplete frames back on ring for completion */
1521         rx_ring->skb = skb;
1522
1523         u64_stats_update_begin(&rx_ring->rx_syncp);
1524         rx_ring->rx_stats.packets += total_packets;
1525         rx_ring->rx_stats.bytes += total_bytes;
1526         u64_stats_update_end(&rx_ring->rx_syncp);
1527         q_vector->rx.total_packets += total_packets;
1528         q_vector->rx.total_bytes += total_bytes;
1529
1530         if (cleaned_count)
1531                 igc_alloc_rx_buffers(rx_ring, cleaned_count);
1532
1533         return total_packets;
1534 }
1535
1536 static inline unsigned int igc_rx_offset(struct igc_ring *rx_ring)
1537 {
1538         return ring_uses_build_skb(rx_ring) ? IGC_SKB_PAD : 0;
1539 }
1540
1541 static bool igc_alloc_mapped_page(struct igc_ring *rx_ring,
1542                                   struct igc_rx_buffer *bi)
1543 {
1544         struct page *page = bi->page;
1545         dma_addr_t dma;
1546
1547         /* since we are recycling buffers we should seldom need to alloc */
1548         if (likely(page))
1549                 return true;
1550
1551         /* alloc new page for storage */
1552         page = dev_alloc_pages(igc_rx_pg_order(rx_ring));
1553         if (unlikely(!page)) {
1554                 rx_ring->rx_stats.alloc_failed++;
1555                 return false;
1556         }
1557
1558         /* map page for use */
1559         dma = dma_map_page_attrs(rx_ring->dev, page, 0,
1560                                  igc_rx_pg_size(rx_ring),
1561                                  DMA_FROM_DEVICE,
1562                                  IGC_RX_DMA_ATTR);
1563
1564         /* if mapping failed free memory back to system since
1565          * there isn't much point in holding memory we can't use
1566          */
1567         if (dma_mapping_error(rx_ring->dev, dma)) {
1568                 __free_page(page);
1569
1570                 rx_ring->rx_stats.alloc_failed++;
1571                 return false;
1572         }
1573
1574         bi->dma = dma;
1575         bi->page = page;
1576         bi->page_offset = igc_rx_offset(rx_ring);
1577         bi->pagecnt_bias = 1;
1578
1579         return true;
1580 }
1581
1582 /**
1583  * igc_clean_tx_irq - Reclaim resources after transmit completes
1584  * @q_vector: pointer to q_vector containing needed info
1585  * @napi_budget: Used to determine if we are in netpoll
1586  *
1587  * returns true if ring is completely cleaned
1588  */
1589 static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget)
1590 {
1591         struct igc_adapter *adapter = q_vector->adapter;
1592         unsigned int total_bytes = 0, total_packets = 0;
1593         unsigned int budget = q_vector->tx.work_limit;
1594         struct igc_ring *tx_ring = q_vector->tx.ring;
1595         unsigned int i = tx_ring->next_to_clean;
1596         struct igc_tx_buffer *tx_buffer;
1597         union igc_adv_tx_desc *tx_desc;
1598
1599         if (test_bit(__IGC_DOWN, &adapter->state))
1600                 return true;
1601
1602         tx_buffer = &tx_ring->tx_buffer_info[i];
1603         tx_desc = IGC_TX_DESC(tx_ring, i);
1604         i -= tx_ring->count;
1605
1606         do {
1607                 union igc_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
1608
1609                 /* if next_to_watch is not set then there is no work pending */
1610                 if (!eop_desc)
1611                         break;
1612
1613                 /* prevent any other reads prior to eop_desc */
1614                 smp_rmb();
1615
1616                 /* if DD is not set pending work has not been completed */
1617                 if (!(eop_desc->wb.status & cpu_to_le32(IGC_TXD_STAT_DD)))
1618                         break;
1619
1620                 /* clear next_to_watch to prevent false hangs */
1621                 tx_buffer->next_to_watch = NULL;
1622
1623                 /* update the statistics for this packet */
1624                 total_bytes += tx_buffer->bytecount;
1625                 total_packets += tx_buffer->gso_segs;
1626
1627                 /* free the skb */
1628                 napi_consume_skb(tx_buffer->skb, napi_budget);
1629
1630                 /* unmap skb header data */
1631                 dma_unmap_single(tx_ring->dev,
1632                                  dma_unmap_addr(tx_buffer, dma),
1633                                  dma_unmap_len(tx_buffer, len),
1634                                  DMA_TO_DEVICE);
1635
1636                 /* clear tx_buffer data */
1637                 dma_unmap_len_set(tx_buffer, len, 0);
1638
1639                 /* clear last DMA location and unmap remaining buffers */
1640                 while (tx_desc != eop_desc) {
1641                         tx_buffer++;
1642                         tx_desc++;
1643                         i++;
1644                         if (unlikely(!i)) {
1645                                 i -= tx_ring->count;
1646                                 tx_buffer = tx_ring->tx_buffer_info;
1647                                 tx_desc = IGC_TX_DESC(tx_ring, 0);
1648                         }
1649
1650                         /* unmap any remaining paged data */
1651                         if (dma_unmap_len(tx_buffer, len)) {
1652                                 dma_unmap_page(tx_ring->dev,
1653                                                dma_unmap_addr(tx_buffer, dma),
1654                                                dma_unmap_len(tx_buffer, len),
1655                                                DMA_TO_DEVICE);
1656                                 dma_unmap_len_set(tx_buffer, len, 0);
1657                         }
1658                 }
1659
1660                 /* move us one more past the eop_desc for start of next pkt */
1661                 tx_buffer++;
1662                 tx_desc++;
1663                 i++;
1664                 if (unlikely(!i)) {
1665                         i -= tx_ring->count;
1666                         tx_buffer = tx_ring->tx_buffer_info;
1667                         tx_desc = IGC_TX_DESC(tx_ring, 0);
1668                 }
1669
1670                 /* issue prefetch for next Tx descriptor */
1671                 prefetch(tx_desc);
1672
1673                 /* update budget accounting */
1674                 budget--;
1675         } while (likely(budget));
1676
1677         netdev_tx_completed_queue(txring_txq(tx_ring),
1678                                   total_packets, total_bytes);
1679
1680         i += tx_ring->count;
1681         tx_ring->next_to_clean = i;
1682         u64_stats_update_begin(&tx_ring->tx_syncp);
1683         tx_ring->tx_stats.bytes += total_bytes;
1684         tx_ring->tx_stats.packets += total_packets;
1685         u64_stats_update_end(&tx_ring->tx_syncp);
1686         q_vector->tx.total_bytes += total_bytes;
1687         q_vector->tx.total_packets += total_packets;
1688
1689         if (test_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
1690                 struct igc_hw *hw = &adapter->hw;
1691
1692                 /* Detect a transmit hang in hardware, this serializes the
1693                  * check with the clearing of time_stamp and movement of i
1694                  */
1695                 clear_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
1696                 if (tx_buffer->next_to_watch &&
1697                     time_after(jiffies, tx_buffer->time_stamp +
1698                     (adapter->tx_timeout_factor * HZ)) &&
1699                     !(rd32(IGC_STATUS) & IGC_STATUS_TXOFF)) {
1700                         /* detected Tx unit hang */
1701                         dev_err(tx_ring->dev,
1702                                 "Detected Tx Unit Hang\n"
1703                                 "  Tx Queue             <%d>\n"
1704                                 "  TDH                  <%x>\n"
1705                                 "  TDT                  <%x>\n"
1706                                 "  next_to_use          <%x>\n"
1707                                 "  next_to_clean        <%x>\n"
1708                                 "buffer_info[next_to_clean]\n"
1709                                 "  time_stamp           <%lx>\n"
1710                                 "  next_to_watch        <%p>\n"
1711                                 "  jiffies              <%lx>\n"
1712                                 "  desc.status          <%x>\n",
1713                                 tx_ring->queue_index,
1714                                 rd32(IGC_TDH(tx_ring->reg_idx)),
1715                                 readl(tx_ring->tail),
1716                                 tx_ring->next_to_use,
1717                                 tx_ring->next_to_clean,
1718                                 tx_buffer->time_stamp,
1719                                 tx_buffer->next_to_watch,
1720                                 jiffies,
1721                                 tx_buffer->next_to_watch->wb.status);
1722                         netif_stop_subqueue(tx_ring->netdev,
1723                                             tx_ring->queue_index);
1724
1725                         /* we are about to reset, no point in enabling stuff */
1726                         return true;
1727                 }
1728         }
1729
1730 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
1731         if (unlikely(total_packets &&
1732                      netif_carrier_ok(tx_ring->netdev) &&
1733                      igc_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) {
1734                 /* Make sure that anybody stopping the queue after this
1735                  * sees the new next_to_clean.
1736                  */
1737                 smp_mb();
1738                 if (__netif_subqueue_stopped(tx_ring->netdev,
1739                                              tx_ring->queue_index) &&
1740                     !(test_bit(__IGC_DOWN, &adapter->state))) {
1741                         netif_wake_subqueue(tx_ring->netdev,
1742                                             tx_ring->queue_index);
1743
1744                         u64_stats_update_begin(&tx_ring->tx_syncp);
1745                         tx_ring->tx_stats.restart_queue++;
1746                         u64_stats_update_end(&tx_ring->tx_syncp);
1747                 }
1748         }
1749
1750         return !!budget;
1751 }
1752
1753 /**
1754  * igc_up - Open the interface and prepare it to handle traffic
1755  * @adapter: board private structure
1756  */
1757 void igc_up(struct igc_adapter *adapter)
1758 {
1759         struct igc_hw *hw = &adapter->hw;
1760         int i = 0;
1761
1762         /* hardware has been reset, we need to reload some things */
1763         igc_configure(adapter);
1764
1765         clear_bit(__IGC_DOWN, &adapter->state);
1766
1767         for (i = 0; i < adapter->num_q_vectors; i++)
1768                 napi_enable(&adapter->q_vector[i]->napi);
1769
1770         if (adapter->msix_entries)
1771                 igc_configure_msix(adapter);
1772         else
1773                 igc_assign_vector(adapter->q_vector[0], 0);
1774
1775         /* Clear any pending interrupts. */
1776         rd32(IGC_ICR);
1777         igc_irq_enable(adapter);
1778
1779         netif_tx_start_all_queues(adapter->netdev);
1780
1781         /* start the watchdog. */
1782         hw->mac.get_link_status = 1;
1783         schedule_work(&adapter->watchdog_task);
1784 }
1785
1786 /**
1787  * igc_update_stats - Update the board statistics counters
1788  * @adapter: board private structure
1789  */
1790 void igc_update_stats(struct igc_adapter *adapter)
1791 {
1792         struct rtnl_link_stats64 *net_stats = &adapter->stats64;
1793         struct pci_dev *pdev = adapter->pdev;
1794         struct igc_hw *hw = &adapter->hw;
1795         u64 _bytes, _packets;
1796         u64 bytes, packets;
1797         unsigned int start;
1798         u32 mpc;
1799         int i;
1800
1801         /* Prevent stats update while adapter is being reset, or if the pci
1802          * connection is down.
1803          */
1804         if (adapter->link_speed == 0)
1805                 return;
1806         if (pci_channel_offline(pdev))
1807                 return;
1808
1809         packets = 0;
1810         bytes = 0;
1811
1812         rcu_read_lock();
1813         for (i = 0; i < adapter->num_rx_queues; i++) {
1814                 struct igc_ring *ring = adapter->rx_ring[i];
1815                 u32 rqdpc = rd32(IGC_RQDPC(i));
1816
1817                 if (hw->mac.type >= igc_i225)
1818                         wr32(IGC_RQDPC(i), 0);
1819
1820                 if (rqdpc) {
1821                         ring->rx_stats.drops += rqdpc;
1822                         net_stats->rx_fifo_errors += rqdpc;
1823                 }
1824
1825                 do {
1826                         start = u64_stats_fetch_begin_irq(&ring->rx_syncp);
1827                         _bytes = ring->rx_stats.bytes;
1828                         _packets = ring->rx_stats.packets;
1829                 } while (u64_stats_fetch_retry_irq(&ring->rx_syncp, start));
1830                 bytes += _bytes;
1831                 packets += _packets;
1832         }
1833
1834         net_stats->rx_bytes = bytes;
1835         net_stats->rx_packets = packets;
1836
1837         packets = 0;
1838         bytes = 0;
1839         for (i = 0; i < adapter->num_tx_queues; i++) {
1840                 struct igc_ring *ring = adapter->tx_ring[i];
1841
1842                 do {
1843                         start = u64_stats_fetch_begin_irq(&ring->tx_syncp);
1844                         _bytes = ring->tx_stats.bytes;
1845                         _packets = ring->tx_stats.packets;
1846                 } while (u64_stats_fetch_retry_irq(&ring->tx_syncp, start));
1847                 bytes += _bytes;
1848                 packets += _packets;
1849         }
1850         net_stats->tx_bytes = bytes;
1851         net_stats->tx_packets = packets;
1852         rcu_read_unlock();
1853
1854         /* read stats registers */
1855         adapter->stats.crcerrs += rd32(IGC_CRCERRS);
1856         adapter->stats.gprc += rd32(IGC_GPRC);
1857         adapter->stats.gorc += rd32(IGC_GORCL);
1858         rd32(IGC_GORCH); /* clear GORCL */
1859         adapter->stats.bprc += rd32(IGC_BPRC);
1860         adapter->stats.mprc += rd32(IGC_MPRC);
1861         adapter->stats.roc += rd32(IGC_ROC);
1862
1863         adapter->stats.prc64 += rd32(IGC_PRC64);
1864         adapter->stats.prc127 += rd32(IGC_PRC127);
1865         adapter->stats.prc255 += rd32(IGC_PRC255);
1866         adapter->stats.prc511 += rd32(IGC_PRC511);
1867         adapter->stats.prc1023 += rd32(IGC_PRC1023);
1868         adapter->stats.prc1522 += rd32(IGC_PRC1522);
1869         adapter->stats.symerrs += rd32(IGC_SYMERRS);
1870         adapter->stats.sec += rd32(IGC_SEC);
1871
1872         mpc = rd32(IGC_MPC);
1873         adapter->stats.mpc += mpc;
1874         net_stats->rx_fifo_errors += mpc;
1875         adapter->stats.scc += rd32(IGC_SCC);
1876         adapter->stats.ecol += rd32(IGC_ECOL);
1877         adapter->stats.mcc += rd32(IGC_MCC);
1878         adapter->stats.latecol += rd32(IGC_LATECOL);
1879         adapter->stats.dc += rd32(IGC_DC);
1880         adapter->stats.rlec += rd32(IGC_RLEC);
1881         adapter->stats.xonrxc += rd32(IGC_XONRXC);
1882         adapter->stats.xontxc += rd32(IGC_XONTXC);
1883         adapter->stats.xoffrxc += rd32(IGC_XOFFRXC);
1884         adapter->stats.xofftxc += rd32(IGC_XOFFTXC);
1885         adapter->stats.fcruc += rd32(IGC_FCRUC);
1886         adapter->stats.gptc += rd32(IGC_GPTC);
1887         adapter->stats.gotc += rd32(IGC_GOTCL);
1888         rd32(IGC_GOTCH); /* clear GOTCL */
1889         adapter->stats.rnbc += rd32(IGC_RNBC);
1890         adapter->stats.ruc += rd32(IGC_RUC);
1891         adapter->stats.rfc += rd32(IGC_RFC);
1892         adapter->stats.rjc += rd32(IGC_RJC);
1893         adapter->stats.tor += rd32(IGC_TORH);
1894         adapter->stats.tot += rd32(IGC_TOTH);
1895         adapter->stats.tpr += rd32(IGC_TPR);
1896
1897         adapter->stats.ptc64 += rd32(IGC_PTC64);
1898         adapter->stats.ptc127 += rd32(IGC_PTC127);
1899         adapter->stats.ptc255 += rd32(IGC_PTC255);
1900         adapter->stats.ptc511 += rd32(IGC_PTC511);
1901         adapter->stats.ptc1023 += rd32(IGC_PTC1023);
1902         adapter->stats.ptc1522 += rd32(IGC_PTC1522);
1903
1904         adapter->stats.mptc += rd32(IGC_MPTC);
1905         adapter->stats.bptc += rd32(IGC_BPTC);
1906
1907         adapter->stats.tpt += rd32(IGC_TPT);
1908         adapter->stats.colc += rd32(IGC_COLC);
1909
1910         adapter->stats.algnerrc += rd32(IGC_ALGNERRC);
1911
1912         adapter->stats.tsctc += rd32(IGC_TSCTC);
1913         adapter->stats.tsctfc += rd32(IGC_TSCTFC);
1914
1915         adapter->stats.iac += rd32(IGC_IAC);
1916         adapter->stats.icrxoc += rd32(IGC_ICRXOC);
1917         adapter->stats.icrxptc += rd32(IGC_ICRXPTC);
1918         adapter->stats.icrxatc += rd32(IGC_ICRXATC);
1919         adapter->stats.ictxptc += rd32(IGC_ICTXPTC);
1920         adapter->stats.ictxatc += rd32(IGC_ICTXATC);
1921         adapter->stats.ictxqec += rd32(IGC_ICTXQEC);
1922         adapter->stats.ictxqmtc += rd32(IGC_ICTXQMTC);
1923         adapter->stats.icrxdmtc += rd32(IGC_ICRXDMTC);
1924
1925         /* Fill out the OS statistics structure */
1926         net_stats->multicast = adapter->stats.mprc;
1927         net_stats->collisions = adapter->stats.colc;
1928
1929         /* Rx Errors */
1930
1931         /* RLEC on some newer hardware can be incorrect so build
1932          * our own version based on RUC and ROC
1933          */
1934         net_stats->rx_errors = adapter->stats.rxerrc +
1935                 adapter->stats.crcerrs + adapter->stats.algnerrc +
1936                 adapter->stats.ruc + adapter->stats.roc +
1937                 adapter->stats.cexterr;
1938         net_stats->rx_length_errors = adapter->stats.ruc +
1939                                       adapter->stats.roc;
1940         net_stats->rx_crc_errors = adapter->stats.crcerrs;
1941         net_stats->rx_frame_errors = adapter->stats.algnerrc;
1942         net_stats->rx_missed_errors = adapter->stats.mpc;
1943
1944         /* Tx Errors */
1945         net_stats->tx_errors = adapter->stats.ecol +
1946                                adapter->stats.latecol;
1947         net_stats->tx_aborted_errors = adapter->stats.ecol;
1948         net_stats->tx_window_errors = adapter->stats.latecol;
1949         net_stats->tx_carrier_errors = adapter->stats.tncrs;
1950
1951         /* Tx Dropped needs to be maintained elsewhere */
1952
1953         /* Management Stats */
1954         adapter->stats.mgptc += rd32(IGC_MGTPTC);
1955         adapter->stats.mgprc += rd32(IGC_MGTPRC);
1956         adapter->stats.mgpdc += rd32(IGC_MGTPDC);
1957 }
1958
1959 static void igc_nfc_filter_exit(struct igc_adapter *adapter)
1960 {
1961         struct igc_nfc_filter *rule;
1962
1963         spin_lock(&adapter->nfc_lock);
1964
1965         hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node)
1966                 igc_erase_filter(adapter, rule);
1967
1968         hlist_for_each_entry(rule, &adapter->cls_flower_list, nfc_node)
1969                 igc_erase_filter(adapter, rule);
1970
1971         spin_unlock(&adapter->nfc_lock);
1972 }
1973
1974 static void igc_nfc_filter_restore(struct igc_adapter *adapter)
1975 {
1976         struct igc_nfc_filter *rule;
1977
1978         spin_lock(&adapter->nfc_lock);
1979
1980         hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node)
1981                 igc_add_filter(adapter, rule);
1982
1983         spin_unlock(&adapter->nfc_lock);
1984 }
1985
1986 /**
1987  * igc_down - Close the interface
1988  * @adapter: board private structure
1989  */
1990 void igc_down(struct igc_adapter *adapter)
1991 {
1992         struct net_device *netdev = adapter->netdev;
1993         struct igc_hw *hw = &adapter->hw;
1994         u32 tctl, rctl;
1995         int i = 0;
1996
1997         set_bit(__IGC_DOWN, &adapter->state);
1998
1999         /* disable receives in the hardware */
2000         rctl = rd32(IGC_RCTL);
2001         wr32(IGC_RCTL, rctl & ~IGC_RCTL_EN);
2002         /* flush and sleep below */
2003
2004         igc_nfc_filter_exit(adapter);
2005
2006         /* set trans_start so we don't get spurious watchdogs during reset */
2007         netif_trans_update(netdev);
2008
2009         netif_carrier_off(netdev);
2010         netif_tx_stop_all_queues(netdev);
2011
2012         /* disable transmits in the hardware */
2013         tctl = rd32(IGC_TCTL);
2014         tctl &= ~IGC_TCTL_EN;
2015         wr32(IGC_TCTL, tctl);
2016         /* flush both disables and wait for them to finish */
2017         wrfl();
2018         usleep_range(10000, 20000);
2019
2020         igc_irq_disable(adapter);
2021
2022         adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
2023
2024         for (i = 0; i < adapter->num_q_vectors; i++) {
2025                 if (adapter->q_vector[i]) {
2026                         napi_synchronize(&adapter->q_vector[i]->napi);
2027                         napi_disable(&adapter->q_vector[i]->napi);
2028                 }
2029         }
2030
2031         del_timer_sync(&adapter->watchdog_timer);
2032         del_timer_sync(&adapter->phy_info_timer);
2033
2034         /* record the stats before reset*/
2035         spin_lock(&adapter->stats64_lock);
2036         igc_update_stats(adapter);
2037         spin_unlock(&adapter->stats64_lock);
2038
2039         adapter->link_speed = 0;
2040         adapter->link_duplex = 0;
2041
2042         if (!pci_channel_offline(adapter->pdev))
2043                 igc_reset(adapter);
2044
2045         /* clear VLAN promisc flag so VFTA will be updated if necessary */
2046         adapter->flags &= ~IGC_FLAG_VLAN_PROMISC;
2047
2048         igc_clean_all_tx_rings(adapter);
2049         igc_clean_all_rx_rings(adapter);
2050 }
2051
2052 void igc_reinit_locked(struct igc_adapter *adapter)
2053 {
2054         WARN_ON(in_interrupt());
2055         while (test_and_set_bit(__IGC_RESETTING, &adapter->state))
2056                 usleep_range(1000, 2000);
2057         igc_down(adapter);
2058         igc_up(adapter);
2059         clear_bit(__IGC_RESETTING, &adapter->state);
2060 }
2061
2062 static void igc_reset_task(struct work_struct *work)
2063 {
2064         struct igc_adapter *adapter;
2065
2066         adapter = container_of(work, struct igc_adapter, reset_task);
2067
2068         netdev_err(adapter->netdev, "Reset adapter\n");
2069         igc_reinit_locked(adapter);
2070 }
2071
2072 /**
2073  * igc_change_mtu - Change the Maximum Transfer Unit
2074  * @netdev: network interface device structure
2075  * @new_mtu: new value for maximum frame size
2076  *
2077  * Returns 0 on success, negative on failure
2078  */
2079 static int igc_change_mtu(struct net_device *netdev, int new_mtu)
2080 {
2081         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
2082         struct igc_adapter *adapter = netdev_priv(netdev);
2083         struct pci_dev *pdev = adapter->pdev;
2084
2085         /* adjust max frame to be at least the size of a standard frame */
2086         if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN))
2087                 max_frame = ETH_FRAME_LEN + ETH_FCS_LEN;
2088
2089         while (test_and_set_bit(__IGC_RESETTING, &adapter->state))
2090                 usleep_range(1000, 2000);
2091
2092         /* igc_down has a dependency on max_frame_size */
2093         adapter->max_frame_size = max_frame;
2094
2095         if (netif_running(netdev))
2096                 igc_down(adapter);
2097
2098         dev_info(&pdev->dev, "changing MTU from %d to %d\n",
2099                  netdev->mtu, new_mtu);
2100         netdev->mtu = new_mtu;
2101
2102         if (netif_running(netdev))
2103                 igc_up(adapter);
2104         else
2105                 igc_reset(adapter);
2106
2107         clear_bit(__IGC_RESETTING, &adapter->state);
2108
2109         return 0;
2110 }
2111
2112 /**
2113  * igc_get_stats - Get System Network Statistics
2114  * @netdev: network interface device structure
2115  *
2116  * Returns the address of the device statistics structure.
2117  * The statistics are updated here and also from the timer callback.
2118  */
2119 static struct net_device_stats *igc_get_stats(struct net_device *netdev)
2120 {
2121         struct igc_adapter *adapter = netdev_priv(netdev);
2122
2123         if (!test_bit(__IGC_RESETTING, &adapter->state))
2124                 igc_update_stats(adapter);
2125
2126         /* only return the current stats */
2127         return &netdev->stats;
2128 }
2129
2130 static netdev_features_t igc_fix_features(struct net_device *netdev,
2131                                           netdev_features_t features)
2132 {
2133         /* Since there is no support for separate Rx/Tx vlan accel
2134          * enable/disable make sure Tx flag is always in same state as Rx.
2135          */
2136         if (features & NETIF_F_HW_VLAN_CTAG_RX)
2137                 features |= NETIF_F_HW_VLAN_CTAG_TX;
2138         else
2139                 features &= ~NETIF_F_HW_VLAN_CTAG_TX;
2140
2141         return features;
2142 }
2143
2144 static int igc_set_features(struct net_device *netdev,
2145                             netdev_features_t features)
2146 {
2147         netdev_features_t changed = netdev->features ^ features;
2148         struct igc_adapter *adapter = netdev_priv(netdev);
2149
2150         /* Add VLAN support */
2151         if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE)))
2152                 return 0;
2153
2154         if (!(features & NETIF_F_NTUPLE)) {
2155                 struct hlist_node *node2;
2156                 struct igc_nfc_filter *rule;
2157
2158                 spin_lock(&adapter->nfc_lock);
2159                 hlist_for_each_entry_safe(rule, node2,
2160                                           &adapter->nfc_filter_list, nfc_node) {
2161                         igc_erase_filter(adapter, rule);
2162                         hlist_del(&rule->nfc_node);
2163                         kfree(rule);
2164                 }
2165                 spin_unlock(&adapter->nfc_lock);
2166                 adapter->nfc_filter_count = 0;
2167         }
2168
2169         netdev->features = features;
2170
2171         if (netif_running(netdev))
2172                 igc_reinit_locked(adapter);
2173         else
2174                 igc_reset(adapter);
2175
2176         return 1;
2177 }
2178
2179 static netdev_features_t
2180 igc_features_check(struct sk_buff *skb, struct net_device *dev,
2181                    netdev_features_t features)
2182 {
2183         unsigned int network_hdr_len, mac_hdr_len;
2184
2185         /* Make certain the headers can be described by a context descriptor */
2186         mac_hdr_len = skb_network_header(skb) - skb->data;
2187         if (unlikely(mac_hdr_len > IGC_MAX_MAC_HDR_LEN))
2188                 return features & ~(NETIF_F_HW_CSUM |
2189                                     NETIF_F_SCTP_CRC |
2190                                     NETIF_F_HW_VLAN_CTAG_TX |
2191                                     NETIF_F_TSO |
2192                                     NETIF_F_TSO6);
2193
2194         network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb);
2195         if (unlikely(network_hdr_len >  IGC_MAX_NETWORK_HDR_LEN))
2196                 return features & ~(NETIF_F_HW_CSUM |
2197                                     NETIF_F_SCTP_CRC |
2198                                     NETIF_F_TSO |
2199                                     NETIF_F_TSO6);
2200
2201         /* We can only support IPv4 TSO in tunnels if we can mangle the
2202          * inner IP ID field, so strip TSO if MANGLEID is not supported.
2203          */
2204         if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID))
2205                 features &= ~NETIF_F_TSO;
2206
2207         return features;
2208 }
2209
2210 /**
2211  * igc_configure - configure the hardware for RX and TX
2212  * @adapter: private board structure
2213  */
2214 static void igc_configure(struct igc_adapter *adapter)
2215 {
2216         struct net_device *netdev = adapter->netdev;
2217         int i = 0;
2218
2219         igc_get_hw_control(adapter);
2220         igc_set_rx_mode(netdev);
2221
2222         igc_setup_tctl(adapter);
2223         igc_setup_mrqc(adapter);
2224         igc_setup_rctl(adapter);
2225
2226         igc_nfc_filter_restore(adapter);
2227         igc_configure_tx(adapter);
2228         igc_configure_rx(adapter);
2229
2230         igc_rx_fifo_flush_base(&adapter->hw);
2231
2232         /* call igc_desc_unused which always leaves
2233          * at least 1 descriptor unused to make sure
2234          * next_to_use != next_to_clean
2235          */
2236         for (i = 0; i < adapter->num_rx_queues; i++) {
2237                 struct igc_ring *ring = adapter->rx_ring[i];
2238
2239                 igc_alloc_rx_buffers(ring, igc_desc_unused(ring));
2240         }
2241 }
2242
2243 /**
2244  * igc_rar_set_index - Sync RAL[index] and RAH[index] registers with MAC table
2245  * @adapter: address of board private structure
2246  * @index: Index of the RAR entry which need to be synced with MAC table
2247  */
2248 static void igc_rar_set_index(struct igc_adapter *adapter, u32 index)
2249 {
2250         u8 *addr = adapter->mac_table[index].addr;
2251         struct igc_hw *hw = &adapter->hw;
2252         u32 rar_low, rar_high;
2253
2254         /* HW expects these to be in network order when they are plugged
2255          * into the registers which are little endian.  In order to guarantee
2256          * that ordering we need to do an leXX_to_cpup here in order to be
2257          * ready for the byteswap that occurs with writel
2258          */
2259         rar_low = le32_to_cpup((__le32 *)(addr));
2260         rar_high = le16_to_cpup((__le16 *)(addr + 4));
2261
2262         /* Indicate to hardware the Address is Valid. */
2263         if (adapter->mac_table[index].state & IGC_MAC_STATE_IN_USE) {
2264                 if (is_valid_ether_addr(addr))
2265                         rar_high |= IGC_RAH_AV;
2266
2267                 rar_high |= IGC_RAH_POOL_1 <<
2268                         adapter->mac_table[index].queue;
2269         }
2270
2271         wr32(IGC_RAL(index), rar_low);
2272         wrfl();
2273         wr32(IGC_RAH(index), rar_high);
2274         wrfl();
2275 }
2276
2277 /* Set default MAC address for the PF in the first RAR entry */
2278 static void igc_set_default_mac_filter(struct igc_adapter *adapter)
2279 {
2280         struct igc_mac_addr *mac_table = &adapter->mac_table[0];
2281
2282         ether_addr_copy(mac_table->addr, adapter->hw.mac.addr);
2283         mac_table->state = IGC_MAC_STATE_DEFAULT | IGC_MAC_STATE_IN_USE;
2284
2285         igc_rar_set_index(adapter, 0);
2286 }
2287
2288 /* If the filter to be added and an already existing filter express
2289  * the same address and address type, it should be possible to only
2290  * override the other configurations, for example the queue to steer
2291  * traffic.
2292  */
2293 static bool igc_mac_entry_can_be_used(const struct igc_mac_addr *entry,
2294                                       const u8 *addr, const u8 flags)
2295 {
2296         if (!(entry->state & IGC_MAC_STATE_IN_USE))
2297                 return true;
2298
2299         if ((entry->state & IGC_MAC_STATE_SRC_ADDR) !=
2300             (flags & IGC_MAC_STATE_SRC_ADDR))
2301                 return false;
2302
2303         if (!ether_addr_equal(addr, entry->addr))
2304                 return false;
2305
2306         return true;
2307 }
2308
2309 /* Add a MAC filter for 'addr' directing matching traffic to 'queue',
2310  * 'flags' is used to indicate what kind of match is made, match is by
2311  * default for the destination address, if matching by source address
2312  * is desired the flag IGC_MAC_STATE_SRC_ADDR can be used.
2313  */
2314 static int igc_add_mac_filter_flags(struct igc_adapter *adapter,
2315                                     const u8 *addr, const u8 queue,
2316                                     const u8 flags)
2317 {
2318         struct igc_hw *hw = &adapter->hw;
2319         int rar_entries = hw->mac.rar_entry_count;
2320         int i;
2321
2322         if (is_zero_ether_addr(addr))
2323                 return -EINVAL;
2324
2325         /* Search for the first empty entry in the MAC table.
2326          * Do not touch entries at the end of the table reserved for the VF MAC
2327          * addresses.
2328          */
2329         for (i = 0; i < rar_entries; i++) {
2330                 if (!igc_mac_entry_can_be_used(&adapter->mac_table[i],
2331                                                addr, flags))
2332                         continue;
2333
2334                 ether_addr_copy(adapter->mac_table[i].addr, addr);
2335                 adapter->mac_table[i].queue = queue;
2336                 adapter->mac_table[i].state |= IGC_MAC_STATE_IN_USE | flags;
2337
2338                 igc_rar_set_index(adapter, i);
2339                 return i;
2340         }
2341
2342         return -ENOSPC;
2343 }
2344
2345 int igc_add_mac_steering_filter(struct igc_adapter *adapter,
2346                                 const u8 *addr, u8 queue, u8 flags)
2347 {
2348         return igc_add_mac_filter_flags(adapter, addr, queue,
2349                                         IGC_MAC_STATE_QUEUE_STEERING | flags);
2350 }
2351
2352 /* Remove a MAC filter for 'addr' directing matching traffic to
2353  * 'queue', 'flags' is used to indicate what kind of match need to be
2354  * removed, match is by default for the destination address, if
2355  * matching by source address is to be removed the flag
2356  * IGC_MAC_STATE_SRC_ADDR can be used.
2357  */
2358 static int igc_del_mac_filter_flags(struct igc_adapter *adapter,
2359                                     const u8 *addr, const u8 queue,
2360                                     const u8 flags)
2361 {
2362         struct igc_hw *hw = &adapter->hw;
2363         int rar_entries = hw->mac.rar_entry_count;
2364         int i;
2365
2366         if (is_zero_ether_addr(addr))
2367                 return -EINVAL;
2368
2369         /* Search for matching entry in the MAC table based on given address
2370          * and queue. Do not touch entries at the end of the table reserved
2371          * for the VF MAC addresses.
2372          */
2373         for (i = 0; i < rar_entries; i++) {
2374                 if (!(adapter->mac_table[i].state & IGC_MAC_STATE_IN_USE))
2375                         continue;
2376                 if ((adapter->mac_table[i].state & flags) != flags)
2377                         continue;
2378                 if (adapter->mac_table[i].queue != queue)
2379                         continue;
2380                 if (!ether_addr_equal(adapter->mac_table[i].addr, addr))
2381                         continue;
2382
2383                 /* When a filter for the default address is "deleted",
2384                  * we return it to its initial configuration
2385                  */
2386                 if (adapter->mac_table[i].state & IGC_MAC_STATE_DEFAULT) {
2387                         adapter->mac_table[i].state =
2388                                 IGC_MAC_STATE_DEFAULT | IGC_MAC_STATE_IN_USE;
2389                 } else {
2390                         adapter->mac_table[i].state = 0;
2391                         adapter->mac_table[i].queue = 0;
2392                         memset(adapter->mac_table[i].addr, 0, ETH_ALEN);
2393                 }
2394
2395                 igc_rar_set_index(adapter, i);
2396                 return 0;
2397         }
2398
2399         return -ENOENT;
2400 }
2401
2402 int igc_del_mac_steering_filter(struct igc_adapter *adapter,
2403                                 const u8 *addr, u8 queue, u8 flags)
2404 {
2405         return igc_del_mac_filter_flags(adapter, addr, queue,
2406                                         IGC_MAC_STATE_QUEUE_STEERING | flags);
2407 }
2408
2409 /**
2410  * igc_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
2411  * @netdev: network interface device structure
2412  *
2413  * The set_rx_mode entry point is called whenever the unicast or multicast
2414  * address lists or the network interface flags are updated.  This routine is
2415  * responsible for configuring the hardware for proper unicast, multicast,
2416  * promiscuous mode, and all-multi behavior.
2417  */
2418 static void igc_set_rx_mode(struct net_device *netdev)
2419 {
2420 }
2421
2422 /**
2423  * igc_msix_other - msix other interrupt handler
2424  * @irq: interrupt number
2425  * @data: pointer to a q_vector
2426  */
2427 static irqreturn_t igc_msix_other(int irq, void *data)
2428 {
2429         struct igc_adapter *adapter = data;
2430         struct igc_hw *hw = &adapter->hw;
2431         u32 icr = rd32(IGC_ICR);
2432
2433         /* reading ICR causes bit 31 of EICR to be cleared */
2434         if (icr & IGC_ICR_DRSTA)
2435                 schedule_work(&adapter->reset_task);
2436
2437         if (icr & IGC_ICR_DOUTSYNC) {
2438                 /* HW is reporting DMA is out of sync */
2439                 adapter->stats.doosync++;
2440         }
2441
2442         if (icr & IGC_ICR_LSC) {
2443                 hw->mac.get_link_status = 1;
2444                 /* guard against interrupt when we're going down */
2445                 if (!test_bit(__IGC_DOWN, &adapter->state))
2446                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
2447         }
2448
2449         wr32(IGC_EIMS, adapter->eims_other);
2450
2451         return IRQ_HANDLED;
2452 }
2453
2454 /**
2455  * igc_write_ivar - configure ivar for given MSI-X vector
2456  * @hw: pointer to the HW structure
2457  * @msix_vector: vector number we are allocating to a given ring
2458  * @index: row index of IVAR register to write within IVAR table
2459  * @offset: column offset of in IVAR, should be multiple of 8
2460  *
2461  * The IVAR table consists of 2 columns,
2462  * each containing an cause allocation for an Rx and Tx ring, and a
2463  * variable number of rows depending on the number of queues supported.
2464  */
2465 static void igc_write_ivar(struct igc_hw *hw, int msix_vector,
2466                            int index, int offset)
2467 {
2468         u32 ivar = array_rd32(IGC_IVAR0, index);
2469
2470         /* clear any bits that are currently set */
2471         ivar &= ~((u32)0xFF << offset);
2472
2473         /* write vector and valid bit */
2474         ivar |= (msix_vector | IGC_IVAR_VALID) << offset;
2475
2476         array_wr32(IGC_IVAR0, index, ivar);
2477 }
2478
2479 static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector)
2480 {
2481         struct igc_adapter *adapter = q_vector->adapter;
2482         struct igc_hw *hw = &adapter->hw;
2483         int rx_queue = IGC_N0_QUEUE;
2484         int tx_queue = IGC_N0_QUEUE;
2485
2486         if (q_vector->rx.ring)
2487                 rx_queue = q_vector->rx.ring->reg_idx;
2488         if (q_vector->tx.ring)
2489                 tx_queue = q_vector->tx.ring->reg_idx;
2490
2491         switch (hw->mac.type) {
2492         case igc_i225:
2493                 if (rx_queue > IGC_N0_QUEUE)
2494                         igc_write_ivar(hw, msix_vector,
2495                                        rx_queue >> 1,
2496                                        (rx_queue & 0x1) << 4);
2497                 if (tx_queue > IGC_N0_QUEUE)
2498                         igc_write_ivar(hw, msix_vector,
2499                                        tx_queue >> 1,
2500                                        ((tx_queue & 0x1) << 4) + 8);
2501                 q_vector->eims_value = BIT(msix_vector);
2502                 break;
2503         default:
2504                 WARN_ONCE(hw->mac.type != igc_i225, "Wrong MAC type\n");
2505                 break;
2506         }
2507
2508         /* add q_vector eims value to global eims_enable_mask */
2509         adapter->eims_enable_mask |= q_vector->eims_value;
2510
2511         /* configure q_vector to set itr on first interrupt */
2512         q_vector->set_itr = 1;
2513 }
2514
2515 /**
2516  * igc_configure_msix - Configure MSI-X hardware
2517  * @adapter: Pointer to adapter structure
2518  *
2519  * igc_configure_msix sets up the hardware to properly
2520  * generate MSI-X interrupts.
2521  */
2522 static void igc_configure_msix(struct igc_adapter *adapter)
2523 {
2524         struct igc_hw *hw = &adapter->hw;
2525         int i, vector = 0;
2526         u32 tmp;
2527
2528         adapter->eims_enable_mask = 0;
2529
2530         /* set vector for other causes, i.e. link changes */
2531         switch (hw->mac.type) {
2532         case igc_i225:
2533                 /* Turn on MSI-X capability first, or our settings
2534                  * won't stick.  And it will take days to debug.
2535                  */
2536                 wr32(IGC_GPIE, IGC_GPIE_MSIX_MODE |
2537                      IGC_GPIE_PBA | IGC_GPIE_EIAME |
2538                      IGC_GPIE_NSICR);
2539
2540                 /* enable msix_other interrupt */
2541                 adapter->eims_other = BIT(vector);
2542                 tmp = (vector++ | IGC_IVAR_VALID) << 8;
2543
2544                 wr32(IGC_IVAR_MISC, tmp);
2545                 break;
2546         default:
2547                 /* do nothing, since nothing else supports MSI-X */
2548                 break;
2549         } /* switch (hw->mac.type) */
2550
2551         adapter->eims_enable_mask |= adapter->eims_other;
2552
2553         for (i = 0; i < adapter->num_q_vectors; i++)
2554                 igc_assign_vector(adapter->q_vector[i], vector++);
2555
2556         wrfl();
2557 }
2558
2559 static irqreturn_t igc_msix_ring(int irq, void *data)
2560 {
2561         struct igc_q_vector *q_vector = data;
2562
2563         /* Write the ITR value calculated from the previous interrupt. */
2564         igc_write_itr(q_vector);
2565
2566         napi_schedule(&q_vector->napi);
2567
2568         return IRQ_HANDLED;
2569 }
2570
2571 /**
2572  * igc_request_msix - Initialize MSI-X interrupts
2573  * @adapter: Pointer to adapter structure
2574  *
2575  * igc_request_msix allocates MSI-X vectors and requests interrupts from the
2576  * kernel.
2577  */
2578 static int igc_request_msix(struct igc_adapter *adapter)
2579 {
2580         int i = 0, err = 0, vector = 0, free_vector = 0;
2581         struct net_device *netdev = adapter->netdev;
2582
2583         err = request_irq(adapter->msix_entries[vector].vector,
2584                           &igc_msix_other, 0, netdev->name, adapter);
2585         if (err)
2586                 goto err_out;
2587
2588         for (i = 0; i < adapter->num_q_vectors; i++) {
2589                 struct igc_q_vector *q_vector = adapter->q_vector[i];
2590
2591                 vector++;
2592
2593                 q_vector->itr_register = adapter->io_addr + IGC_EITR(vector);
2594
2595                 if (q_vector->rx.ring && q_vector->tx.ring)
2596                         sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
2597                                 q_vector->rx.ring->queue_index);
2598                 else if (q_vector->tx.ring)
2599                         sprintf(q_vector->name, "%s-tx-%u", netdev->name,
2600                                 q_vector->tx.ring->queue_index);
2601                 else if (q_vector->rx.ring)
2602                         sprintf(q_vector->name, "%s-rx-%u", netdev->name,
2603                                 q_vector->rx.ring->queue_index);
2604                 else
2605                         sprintf(q_vector->name, "%s-unused", netdev->name);
2606
2607                 err = request_irq(adapter->msix_entries[vector].vector,
2608                                   igc_msix_ring, 0, q_vector->name,
2609                                   q_vector);
2610                 if (err)
2611                         goto err_free;
2612         }
2613
2614         igc_configure_msix(adapter);
2615         return 0;
2616
2617 err_free:
2618         /* free already assigned IRQs */
2619         free_irq(adapter->msix_entries[free_vector++].vector, adapter);
2620
2621         vector--;
2622         for (i = 0; i < vector; i++) {
2623                 free_irq(adapter->msix_entries[free_vector++].vector,
2624                          adapter->q_vector[i]);
2625         }
2626 err_out:
2627         return err;
2628 }
2629
2630 /**
2631  * igc_reset_q_vector - Reset config for interrupt vector
2632  * @adapter: board private structure to initialize
2633  * @v_idx: Index of vector to be reset
2634  *
2635  * If NAPI is enabled it will delete any references to the
2636  * NAPI struct. This is preparation for igc_free_q_vector.
2637  */
2638 static void igc_reset_q_vector(struct igc_adapter *adapter, int v_idx)
2639 {
2640         struct igc_q_vector *q_vector = adapter->q_vector[v_idx];
2641
2642         /* if we're coming from igc_set_interrupt_capability, the vectors are
2643          * not yet allocated
2644          */
2645         if (!q_vector)
2646                 return;
2647
2648         if (q_vector->tx.ring)
2649                 adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL;
2650
2651         if (q_vector->rx.ring)
2652                 adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL;
2653
2654         netif_napi_del(&q_vector->napi);
2655 }
2656
2657 static void igc_reset_interrupt_capability(struct igc_adapter *adapter)
2658 {
2659         int v_idx = adapter->num_q_vectors;
2660
2661         if (adapter->msix_entries) {
2662                 pci_disable_msix(adapter->pdev);
2663                 kfree(adapter->msix_entries);
2664                 adapter->msix_entries = NULL;
2665         } else if (adapter->flags & IGC_FLAG_HAS_MSI) {
2666                 pci_disable_msi(adapter->pdev);
2667         }
2668
2669         while (v_idx--)
2670                 igc_reset_q_vector(adapter, v_idx);
2671 }
2672
2673 /**
2674  * igc_clear_interrupt_scheme - reset the device to a state of no interrupts
2675  * @adapter: Pointer to adapter structure
2676  *
2677  * This function resets the device so that it has 0 rx queues, tx queues, and
2678  * MSI-X interrupts allocated.
2679  */
2680 static void igc_clear_interrupt_scheme(struct igc_adapter *adapter)
2681 {
2682         igc_free_q_vectors(adapter);
2683         igc_reset_interrupt_capability(adapter);
2684 }
2685
2686 /**
2687  * igc_free_q_vectors - Free memory allocated for interrupt vectors
2688  * @adapter: board private structure to initialize
2689  *
2690  * This function frees the memory allocated to the q_vectors.  In addition if
2691  * NAPI is enabled it will delete any references to the NAPI struct prior
2692  * to freeing the q_vector.
2693  */
2694 static void igc_free_q_vectors(struct igc_adapter *adapter)
2695 {
2696         int v_idx = adapter->num_q_vectors;
2697
2698         adapter->num_tx_queues = 0;
2699         adapter->num_rx_queues = 0;
2700         adapter->num_q_vectors = 0;
2701
2702         while (v_idx--) {
2703                 igc_reset_q_vector(adapter, v_idx);
2704                 igc_free_q_vector(adapter, v_idx);
2705         }
2706 }
2707
2708 /**
2709  * igc_free_q_vector - Free memory allocated for specific interrupt vector
2710  * @adapter: board private structure to initialize
2711  * @v_idx: Index of vector to be freed
2712  *
2713  * This function frees the memory allocated to the q_vector.
2714  */
2715 static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx)
2716 {
2717         struct igc_q_vector *q_vector = adapter->q_vector[v_idx];
2718
2719         adapter->q_vector[v_idx] = NULL;
2720
2721         /* igc_get_stats64() might access the rings on this vector,
2722          * we must wait a grace period before freeing it.
2723          */
2724         if (q_vector)
2725                 kfree_rcu(q_vector, rcu);
2726 }
2727
2728 /* Need to wait a few seconds after link up to get diagnostic information from
2729  * the phy
2730  */
2731 static void igc_update_phy_info(struct timer_list *t)
2732 {
2733         struct igc_adapter *adapter = from_timer(adapter, t, phy_info_timer);
2734
2735         igc_get_phy_info(&adapter->hw);
2736 }
2737
2738 /**
2739  * igc_has_link - check shared code for link and determine up/down
2740  * @adapter: pointer to driver private info
2741  */
2742 bool igc_has_link(struct igc_adapter *adapter)
2743 {
2744         struct igc_hw *hw = &adapter->hw;
2745         bool link_active = false;
2746
2747         /* get_link_status is set on LSC (link status) interrupt or
2748          * rx sequence error interrupt.  get_link_status will stay
2749          * false until the igc_check_for_link establishes link
2750          * for copper adapters ONLY
2751          */
2752         switch (hw->phy.media_type) {
2753         case igc_media_type_copper:
2754                 if (!hw->mac.get_link_status)
2755                         return true;
2756                 hw->mac.ops.check_for_link(hw);
2757                 link_active = !hw->mac.get_link_status;
2758                 break;
2759         default:
2760         case igc_media_type_unknown:
2761                 break;
2762         }
2763
2764         if (hw->mac.type == igc_i225 &&
2765             hw->phy.id == I225_I_PHY_ID) {
2766                 if (!netif_carrier_ok(adapter->netdev)) {
2767                         adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
2768                 } else if (!(adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)) {
2769                         adapter->flags |= IGC_FLAG_NEED_LINK_UPDATE;
2770                         adapter->link_check_timeout = jiffies;
2771                 }
2772         }
2773
2774         return link_active;
2775 }
2776
2777 /**
2778  * igc_watchdog - Timer Call-back
2779  * @data: pointer to adapter cast into an unsigned long
2780  */
2781 static void igc_watchdog(struct timer_list *t)
2782 {
2783         struct igc_adapter *adapter = from_timer(adapter, t, watchdog_timer);
2784         /* Do the rest outside of interrupt context */
2785         schedule_work(&adapter->watchdog_task);
2786 }
2787
2788 static void igc_watchdog_task(struct work_struct *work)
2789 {
2790         struct igc_adapter *adapter = container_of(work,
2791                                                    struct igc_adapter,
2792                                                    watchdog_task);
2793         struct net_device *netdev = adapter->netdev;
2794         struct igc_hw *hw = &adapter->hw;
2795         struct igc_phy_info *phy = &hw->phy;
2796         u16 phy_data, retry_count = 20;
2797         u32 connsw;
2798         u32 link;
2799         int i;
2800
2801         link = igc_has_link(adapter);
2802
2803         if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) {
2804                 if (time_after(jiffies, (adapter->link_check_timeout + HZ)))
2805                         adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
2806                 else
2807                         link = false;
2808         }
2809
2810         /* Force link down if we have fiber to swap to */
2811         if (adapter->flags & IGC_FLAG_MAS_ENABLE) {
2812                 if (hw->phy.media_type == igc_media_type_copper) {
2813                         connsw = rd32(IGC_CONNSW);
2814                         if (!(connsw & IGC_CONNSW_AUTOSENSE_EN))
2815                                 link = 0;
2816                 }
2817         }
2818         if (link) {
2819                 if (!netif_carrier_ok(netdev)) {
2820                         u32 ctrl;
2821
2822                         hw->mac.ops.get_speed_and_duplex(hw,
2823                                                          &adapter->link_speed,
2824                                                          &adapter->link_duplex);
2825
2826                         ctrl = rd32(IGC_CTRL);
2827                         /* Link status message must follow this format */
2828                         netdev_info(netdev,
2829                                     "igc: %s NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n",
2830                                     netdev->name,
2831                                     adapter->link_speed,
2832                                     adapter->link_duplex == FULL_DUPLEX ?
2833                                     "Full" : "Half",
2834                                     (ctrl & IGC_CTRL_TFCE) &&
2835                                     (ctrl & IGC_CTRL_RFCE) ? "RX/TX" :
2836                                     (ctrl & IGC_CTRL_RFCE) ?  "RX" :
2837                                     (ctrl & IGC_CTRL_TFCE) ?  "TX" : "None");
2838
2839                         /* check if SmartSpeed worked */
2840                         igc_check_downshift(hw);
2841                         if (phy->speed_downgraded)
2842                                 netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n");
2843
2844                         /* adjust timeout factor according to speed/duplex */
2845                         adapter->tx_timeout_factor = 1;
2846                         switch (adapter->link_speed) {
2847                         case SPEED_10:
2848                                 adapter->tx_timeout_factor = 14;
2849                                 break;
2850                         case SPEED_100:
2851                                 /* maybe add some timeout factor ? */
2852                                 break;
2853                         }
2854
2855                         if (adapter->link_speed != SPEED_1000)
2856                                 goto no_wait;
2857
2858                         /* wait for Remote receiver status OK */
2859 retry_read_status:
2860                         if (!igc_read_phy_reg(hw, PHY_1000T_STATUS,
2861                                               &phy_data)) {
2862                                 if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) &&
2863                                     retry_count) {
2864                                         msleep(100);
2865                                         retry_count--;
2866                                         goto retry_read_status;
2867                                 } else if (!retry_count) {
2868                                         dev_err(&adapter->pdev->dev, "exceed max 2 second\n");
2869                                 }
2870                         } else {
2871                                 dev_err(&adapter->pdev->dev, "read 1000Base-T Status Reg\n");
2872                         }
2873 no_wait:
2874                         netif_carrier_on(netdev);
2875
2876                         /* link state has changed, schedule phy info update */
2877                         if (!test_bit(__IGC_DOWN, &adapter->state))
2878                                 mod_timer(&adapter->phy_info_timer,
2879                                           round_jiffies(jiffies + 2 * HZ));
2880                 }
2881         } else {
2882                 if (netif_carrier_ok(netdev)) {
2883                         adapter->link_speed = 0;
2884                         adapter->link_duplex = 0;
2885
2886                         /* Links status message must follow this format */
2887                         netdev_info(netdev, "igc: %s NIC Link is Down\n",
2888                                     netdev->name);
2889                         netif_carrier_off(netdev);
2890
2891                         /* link state has changed, schedule phy info update */
2892                         if (!test_bit(__IGC_DOWN, &adapter->state))
2893                                 mod_timer(&adapter->phy_info_timer,
2894                                           round_jiffies(jiffies + 2 * HZ));
2895
2896                         /* link is down, time to check for alternate media */
2897                         if (adapter->flags & IGC_FLAG_MAS_ENABLE) {
2898                                 if (adapter->flags & IGC_FLAG_MEDIA_RESET) {
2899                                         schedule_work(&adapter->reset_task);
2900                                         /* return immediately */
2901                                         return;
2902                                 }
2903                         }
2904
2905                 /* also check for alternate media here */
2906                 } else if (!netif_carrier_ok(netdev) &&
2907                            (adapter->flags & IGC_FLAG_MAS_ENABLE)) {
2908                         if (adapter->flags & IGC_FLAG_MEDIA_RESET) {
2909                                 schedule_work(&adapter->reset_task);
2910                                 /* return immediately */
2911                                 return;
2912                         }
2913                 }
2914         }
2915
2916         spin_lock(&adapter->stats64_lock);
2917         igc_update_stats(adapter);
2918         spin_unlock(&adapter->stats64_lock);
2919
2920         for (i = 0; i < adapter->num_tx_queues; i++) {
2921                 struct igc_ring *tx_ring = adapter->tx_ring[i];
2922
2923                 if (!netif_carrier_ok(netdev)) {
2924                         /* We've lost link, so the controller stops DMA,
2925                          * but we've got queued Tx work that's never going
2926                          * to get done, so reset controller to flush Tx.
2927                          * (Do the reset outside of interrupt context).
2928                          */
2929                         if (igc_desc_unused(tx_ring) + 1 < tx_ring->count) {
2930                                 adapter->tx_timeout_count++;
2931                                 schedule_work(&adapter->reset_task);
2932                                 /* return immediately since reset is imminent */
2933                                 return;
2934                         }
2935                 }
2936
2937                 /* Force detection of hung controller every watchdog period */
2938                 set_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
2939         }
2940
2941         /* Cause software interrupt to ensure Rx ring is cleaned */
2942         if (adapter->flags & IGC_FLAG_HAS_MSIX) {
2943                 u32 eics = 0;
2944
2945                 for (i = 0; i < adapter->num_q_vectors; i++)
2946                         eics |= adapter->q_vector[i]->eims_value;
2947                 wr32(IGC_EICS, eics);
2948         } else {
2949                 wr32(IGC_ICS, IGC_ICS_RXDMT0);
2950         }
2951
2952         /* Reset the timer */
2953         if (!test_bit(__IGC_DOWN, &adapter->state)) {
2954                 if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)
2955                         mod_timer(&adapter->watchdog_timer,
2956                                   round_jiffies(jiffies +  HZ));
2957                 else
2958                         mod_timer(&adapter->watchdog_timer,
2959                                   round_jiffies(jiffies + 2 * HZ));
2960         }
2961 }
2962
2963 /**
2964  * igc_update_ring_itr - update the dynamic ITR value based on packet size
2965  * @q_vector: pointer to q_vector
2966  *
2967  * Stores a new ITR value based on strictly on packet size.  This
2968  * algorithm is less sophisticated than that used in igc_update_itr,
2969  * due to the difficulty of synchronizing statistics across multiple
2970  * receive rings.  The divisors and thresholds used by this function
2971  * were determined based on theoretical maximum wire speed and testing
2972  * data, in order to minimize response time while increasing bulk
2973  * throughput.
2974  * NOTE: This function is called only when operating in a multiqueue
2975  * receive environment.
2976  */
2977 static void igc_update_ring_itr(struct igc_q_vector *q_vector)
2978 {
2979         struct igc_adapter *adapter = q_vector->adapter;
2980         int new_val = q_vector->itr_val;
2981         int avg_wire_size = 0;
2982         unsigned int packets;
2983
2984         /* For non-gigabit speeds, just fix the interrupt rate at 4000
2985          * ints/sec - ITR timer value of 120 ticks.
2986          */
2987         switch (adapter->link_speed) {
2988         case SPEED_10:
2989         case SPEED_100:
2990                 new_val = IGC_4K_ITR;
2991                 goto set_itr_val;
2992         default:
2993                 break;
2994         }
2995
2996         packets = q_vector->rx.total_packets;
2997         if (packets)
2998                 avg_wire_size = q_vector->rx.total_bytes / packets;
2999
3000         packets = q_vector->tx.total_packets;
3001         if (packets)
3002                 avg_wire_size = max_t(u32, avg_wire_size,
3003                                       q_vector->tx.total_bytes / packets);
3004
3005         /* if avg_wire_size isn't set no work was done */
3006         if (!avg_wire_size)
3007                 goto clear_counts;
3008
3009         /* Add 24 bytes to size to account for CRC, preamble, and gap */
3010         avg_wire_size += 24;
3011
3012         /* Don't starve jumbo frames */
3013         avg_wire_size = min(avg_wire_size, 3000);
3014
3015         /* Give a little boost to mid-size frames */
3016         if (avg_wire_size > 300 && avg_wire_size < 1200)
3017                 new_val = avg_wire_size / 3;
3018         else
3019                 new_val = avg_wire_size / 2;
3020
3021         /* conservative mode (itr 3) eliminates the lowest_latency setting */
3022         if (new_val < IGC_20K_ITR &&
3023             ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
3024             (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
3025                 new_val = IGC_20K_ITR;
3026
3027 set_itr_val:
3028         if (new_val != q_vector->itr_val) {
3029                 q_vector->itr_val = new_val;
3030                 q_vector->set_itr = 1;
3031         }
3032 clear_counts:
3033         q_vector->rx.total_bytes = 0;
3034         q_vector->rx.total_packets = 0;
3035         q_vector->tx.total_bytes = 0;
3036         q_vector->tx.total_packets = 0;
3037 }
3038
3039 /**
3040  * igc_update_itr - update the dynamic ITR value based on statistics
3041  * @q_vector: pointer to q_vector
3042  * @ring_container: ring info to update the itr for
3043  *
3044  * Stores a new ITR value based on packets and byte
3045  * counts during the last interrupt.  The advantage of per interrupt
3046  * computation is faster updates and more accurate ITR for the current
3047  * traffic pattern.  Constants in this function were computed
3048  * based on theoretical maximum wire speed and thresholds were set based
3049  * on testing data as well as attempting to minimize response time
3050  * while increasing bulk throughput.
3051  * NOTE: These calculations are only valid when operating in a single-
3052  * queue environment.
3053  */
3054 static void igc_update_itr(struct igc_q_vector *q_vector,
3055                            struct igc_ring_container *ring_container)
3056 {
3057         unsigned int packets = ring_container->total_packets;
3058         unsigned int bytes = ring_container->total_bytes;
3059         u8 itrval = ring_container->itr;
3060
3061         /* no packets, exit with status unchanged */
3062         if (packets == 0)
3063                 return;
3064
3065         switch (itrval) {
3066         case lowest_latency:
3067                 /* handle TSO and jumbo frames */
3068                 if (bytes / packets > 8000)
3069                         itrval = bulk_latency;
3070                 else if ((packets < 5) && (bytes > 512))
3071                         itrval = low_latency;
3072                 break;
3073         case low_latency:  /* 50 usec aka 20000 ints/s */
3074                 if (bytes > 10000) {
3075                         /* this if handles the TSO accounting */
3076                         if (bytes / packets > 8000)
3077                                 itrval = bulk_latency;
3078                         else if ((packets < 10) || ((bytes / packets) > 1200))
3079                                 itrval = bulk_latency;
3080                         else if ((packets > 35))
3081                                 itrval = lowest_latency;
3082                 } else if (bytes / packets > 2000) {
3083                         itrval = bulk_latency;
3084                 } else if (packets <= 2 && bytes < 512) {
3085                         itrval = lowest_latency;
3086                 }
3087                 break;
3088         case bulk_latency: /* 250 usec aka 4000 ints/s */
3089                 if (bytes > 25000) {
3090                         if (packets > 35)
3091                                 itrval = low_latency;
3092                 } else if (bytes < 1500) {
3093                         itrval = low_latency;
3094                 }
3095                 break;
3096         }
3097
3098         /* clear work counters since we have the values we need */
3099         ring_container->total_bytes = 0;
3100         ring_container->total_packets = 0;
3101
3102         /* write updated itr to ring container */
3103         ring_container->itr = itrval;
3104 }
3105
3106 /**
3107  * igc_intr_msi - Interrupt Handler
3108  * @irq: interrupt number
3109  * @data: pointer to a network interface device structure
3110  */
3111 static irqreturn_t igc_intr_msi(int irq, void *data)
3112 {
3113         struct igc_adapter *adapter = data;
3114         struct igc_q_vector *q_vector = adapter->q_vector[0];
3115         struct igc_hw *hw = &adapter->hw;
3116         /* read ICR disables interrupts using IAM */
3117         u32 icr = rd32(IGC_ICR);
3118
3119         igc_write_itr(q_vector);
3120
3121         if (icr & IGC_ICR_DRSTA)
3122                 schedule_work(&adapter->reset_task);
3123
3124         if (icr & IGC_ICR_DOUTSYNC) {
3125                 /* HW is reporting DMA is out of sync */
3126                 adapter->stats.doosync++;
3127         }
3128
3129         if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) {
3130                 hw->mac.get_link_status = 1;
3131                 if (!test_bit(__IGC_DOWN, &adapter->state))
3132                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
3133         }
3134
3135         napi_schedule(&q_vector->napi);
3136
3137         return IRQ_HANDLED;
3138 }
3139
3140 /**
3141  * igc_intr - Legacy Interrupt Handler
3142  * @irq: interrupt number
3143  * @data: pointer to a network interface device structure
3144  */
3145 static irqreturn_t igc_intr(int irq, void *data)
3146 {
3147         struct igc_adapter *adapter = data;
3148         struct igc_q_vector *q_vector = adapter->q_vector[0];
3149         struct igc_hw *hw = &adapter->hw;
3150         /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked.  No
3151          * need for the IMC write
3152          */
3153         u32 icr = rd32(IGC_ICR);
3154
3155         /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
3156          * not set, then the adapter didn't send an interrupt
3157          */
3158         if (!(icr & IGC_ICR_INT_ASSERTED))
3159                 return IRQ_NONE;
3160
3161         igc_write_itr(q_vector);
3162
3163         if (icr & IGC_ICR_DRSTA)
3164                 schedule_work(&adapter->reset_task);
3165
3166         if (icr & IGC_ICR_DOUTSYNC) {
3167                 /* HW is reporting DMA is out of sync */
3168                 adapter->stats.doosync++;
3169         }
3170
3171         if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) {
3172                 hw->mac.get_link_status = 1;
3173                 /* guard against interrupt when we're going down */
3174                 if (!test_bit(__IGC_DOWN, &adapter->state))
3175                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
3176         }
3177
3178         napi_schedule(&q_vector->napi);
3179
3180         return IRQ_HANDLED;
3181 }
3182
3183 static void igc_set_itr(struct igc_q_vector *q_vector)
3184 {
3185         struct igc_adapter *adapter = q_vector->adapter;
3186         u32 new_itr = q_vector->itr_val;
3187         u8 current_itr = 0;
3188
3189         /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3190         switch (adapter->link_speed) {
3191         case SPEED_10:
3192         case SPEED_100:
3193                 current_itr = 0;
3194                 new_itr = IGC_4K_ITR;
3195                 goto set_itr_now;
3196         default:
3197                 break;
3198         }
3199
3200         igc_update_itr(q_vector, &q_vector->tx);
3201         igc_update_itr(q_vector, &q_vector->rx);
3202
3203         current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
3204
3205         /* conservative mode (itr 3) eliminates the lowest_latency setting */
3206         if (current_itr == lowest_latency &&
3207             ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
3208             (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
3209                 current_itr = low_latency;
3210
3211         switch (current_itr) {
3212         /* counts and packets in update_itr are dependent on these numbers */
3213         case lowest_latency:
3214                 new_itr = IGC_70K_ITR; /* 70,000 ints/sec */
3215                 break;
3216         case low_latency:
3217                 new_itr = IGC_20K_ITR; /* 20,000 ints/sec */
3218                 break;
3219         case bulk_latency:
3220                 new_itr = IGC_4K_ITR;  /* 4,000 ints/sec */
3221                 break;
3222         default:
3223                 break;
3224         }
3225
3226 set_itr_now:
3227         if (new_itr != q_vector->itr_val) {
3228                 /* this attempts to bias the interrupt rate towards Bulk
3229                  * by adding intermediate steps when interrupt rate is
3230                  * increasing
3231                  */
3232                 new_itr = new_itr > q_vector->itr_val ?
3233                           max((new_itr * q_vector->itr_val) /
3234                           (new_itr + (q_vector->itr_val >> 2)),
3235                           new_itr) : new_itr;
3236                 /* Don't write the value here; it resets the adapter's
3237                  * internal timer, and causes us to delay far longer than
3238                  * we should between interrupts.  Instead, we write the ITR
3239                  * value at the beginning of the next interrupt so the timing
3240                  * ends up being correct.
3241                  */
3242                 q_vector->itr_val = new_itr;
3243                 q_vector->set_itr = 1;
3244         }
3245 }
3246
3247 static void igc_ring_irq_enable(struct igc_q_vector *q_vector)
3248 {
3249         struct igc_adapter *adapter = q_vector->adapter;
3250         struct igc_hw *hw = &adapter->hw;
3251
3252         if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
3253             (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
3254                 if (adapter->num_q_vectors == 1)
3255                         igc_set_itr(q_vector);
3256                 else
3257                         igc_update_ring_itr(q_vector);
3258         }
3259
3260         if (!test_bit(__IGC_DOWN, &adapter->state)) {
3261                 if (adapter->msix_entries)
3262                         wr32(IGC_EIMS, q_vector->eims_value);
3263                 else
3264                         igc_irq_enable(adapter);
3265         }
3266 }
3267
3268 /**
3269  * igc_poll - NAPI Rx polling callback
3270  * @napi: napi polling structure
3271  * @budget: count of how many packets we should handle
3272  */
3273 static int igc_poll(struct napi_struct *napi, int budget)
3274 {
3275         struct igc_q_vector *q_vector = container_of(napi,
3276                                                      struct igc_q_vector,
3277                                                      napi);
3278         bool clean_complete = true;
3279         int work_done = 0;
3280
3281         if (q_vector->tx.ring)
3282                 clean_complete = igc_clean_tx_irq(q_vector, budget);
3283
3284         if (q_vector->rx.ring) {
3285                 int cleaned = igc_clean_rx_irq(q_vector, budget);
3286
3287                 work_done += cleaned;
3288                 if (cleaned >= budget)
3289                         clean_complete = false;
3290         }
3291
3292         /* If all work not completed, return budget and keep polling */
3293         if (!clean_complete)
3294                 return budget;
3295
3296         /* Exit the polling mode, but don't re-enable interrupts if stack might
3297          * poll us due to busy-polling
3298          */
3299         if (likely(napi_complete_done(napi, work_done)))
3300                 igc_ring_irq_enable(q_vector);
3301
3302         return min(work_done, budget - 1);
3303 }
3304
3305 /**
3306  * igc_set_interrupt_capability - set MSI or MSI-X if supported
3307  * @adapter: Pointer to adapter structure
3308  *
3309  * Attempt to configure interrupts using the best available
3310  * capabilities of the hardware and kernel.
3311  */
3312 static void igc_set_interrupt_capability(struct igc_adapter *adapter,
3313                                          bool msix)
3314 {
3315         int numvecs, i;
3316         int err;
3317
3318         if (!msix)
3319                 goto msi_only;
3320         adapter->flags |= IGC_FLAG_HAS_MSIX;
3321
3322         /* Number of supported queues. */
3323         adapter->num_rx_queues = adapter->rss_queues;
3324
3325         adapter->num_tx_queues = adapter->rss_queues;
3326
3327         /* start with one vector for every Rx queue */
3328         numvecs = adapter->num_rx_queues;
3329
3330         /* if Tx handler is separate add 1 for every Tx queue */
3331         if (!(adapter->flags & IGC_FLAG_QUEUE_PAIRS))
3332                 numvecs += adapter->num_tx_queues;
3333
3334         /* store the number of vectors reserved for queues */
3335         adapter->num_q_vectors = numvecs;
3336
3337         /* add 1 vector for link status interrupts */
3338         numvecs++;
3339
3340         adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
3341                                         GFP_KERNEL);
3342
3343         if (!adapter->msix_entries)
3344                 return;
3345
3346         /* populate entry values */
3347         for (i = 0; i < numvecs; i++)
3348                 adapter->msix_entries[i].entry = i;
3349
3350         err = pci_enable_msix_range(adapter->pdev,
3351                                     adapter->msix_entries,
3352                                     numvecs,
3353                                     numvecs);
3354         if (err > 0)
3355                 return;
3356
3357         kfree(adapter->msix_entries);
3358         adapter->msix_entries = NULL;
3359
3360         igc_reset_interrupt_capability(adapter);
3361
3362 msi_only:
3363         adapter->flags &= ~IGC_FLAG_HAS_MSIX;
3364
3365         adapter->rss_queues = 1;
3366         adapter->flags |= IGC_FLAG_QUEUE_PAIRS;
3367         adapter->num_rx_queues = 1;
3368         adapter->num_tx_queues = 1;
3369         adapter->num_q_vectors = 1;
3370         if (!pci_enable_msi(adapter->pdev))
3371                 adapter->flags |= IGC_FLAG_HAS_MSI;
3372 }
3373
3374 static void igc_add_ring(struct igc_ring *ring,
3375                          struct igc_ring_container *head)
3376 {
3377         head->ring = ring;
3378         head->count++;
3379 }
3380
3381 /**
3382  * igc_alloc_q_vector - Allocate memory for a single interrupt vector
3383  * @adapter: board private structure to initialize
3384  * @v_count: q_vectors allocated on adapter, used for ring interleaving
3385  * @v_idx: index of vector in adapter struct
3386  * @txr_count: total number of Tx rings to allocate
3387  * @txr_idx: index of first Tx ring to allocate
3388  * @rxr_count: total number of Rx rings to allocate
3389  * @rxr_idx: index of first Rx ring to allocate
3390  *
3391  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
3392  */
3393 static int igc_alloc_q_vector(struct igc_adapter *adapter,
3394                               unsigned int v_count, unsigned int v_idx,
3395                               unsigned int txr_count, unsigned int txr_idx,
3396                               unsigned int rxr_count, unsigned int rxr_idx)
3397 {
3398         struct igc_q_vector *q_vector;
3399         struct igc_ring *ring;
3400         int ring_count;
3401
3402         /* igc only supports 1 Tx and/or 1 Rx queue per vector */
3403         if (txr_count > 1 || rxr_count > 1)
3404                 return -ENOMEM;
3405
3406         ring_count = txr_count + rxr_count;
3407
3408         /* allocate q_vector and rings */
3409         q_vector = adapter->q_vector[v_idx];
3410         if (!q_vector)
3411                 q_vector = kzalloc(struct_size(q_vector, ring, ring_count),
3412                                    GFP_KERNEL);
3413         else
3414                 memset(q_vector, 0, struct_size(q_vector, ring, ring_count));
3415         if (!q_vector)
3416                 return -ENOMEM;
3417
3418         /* initialize NAPI */
3419         netif_napi_add(adapter->netdev, &q_vector->napi,
3420                        igc_poll, 64);
3421
3422         /* tie q_vector and adapter together */
3423         adapter->q_vector[v_idx] = q_vector;
3424         q_vector->adapter = adapter;
3425
3426         /* initialize work limits */
3427         q_vector->tx.work_limit = adapter->tx_work_limit;
3428
3429         /* initialize ITR configuration */
3430         q_vector->itr_register = adapter->io_addr + IGC_EITR(0);
3431         q_vector->itr_val = IGC_START_ITR;
3432
3433         /* initialize pointer to rings */
3434         ring = q_vector->ring;
3435
3436         /* initialize ITR */
3437         if (rxr_count) {
3438                 /* rx or rx/tx vector */
3439                 if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3)
3440                         q_vector->itr_val = adapter->rx_itr_setting;
3441         } else {
3442                 /* tx only vector */
3443                 if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3)
3444                         q_vector->itr_val = adapter->tx_itr_setting;
3445         }
3446
3447         if (txr_count) {
3448                 /* assign generic ring traits */
3449                 ring->dev = &adapter->pdev->dev;
3450                 ring->netdev = adapter->netdev;
3451
3452                 /* configure backlink on ring */
3453                 ring->q_vector = q_vector;
3454
3455                 /* update q_vector Tx values */
3456                 igc_add_ring(ring, &q_vector->tx);
3457
3458                 /* apply Tx specific ring traits */
3459                 ring->count = adapter->tx_ring_count;
3460                 ring->queue_index = txr_idx;
3461
3462                 /* assign ring to adapter */
3463                 adapter->tx_ring[txr_idx] = ring;
3464
3465                 /* push pointer to next ring */
3466                 ring++;
3467         }
3468
3469         if (rxr_count) {
3470                 /* assign generic ring traits */
3471                 ring->dev = &adapter->pdev->dev;
3472                 ring->netdev = adapter->netdev;
3473
3474                 /* configure backlink on ring */
3475                 ring->q_vector = q_vector;
3476
3477                 /* update q_vector Rx values */
3478                 igc_add_ring(ring, &q_vector->rx);
3479
3480                 /* apply Rx specific ring traits */
3481                 ring->count = adapter->rx_ring_count;
3482                 ring->queue_index = rxr_idx;
3483
3484                 /* assign ring to adapter */
3485                 adapter->rx_ring[rxr_idx] = ring;
3486         }
3487
3488         return 0;
3489 }
3490
3491 /**
3492  * igc_alloc_q_vectors - Allocate memory for interrupt vectors
3493  * @adapter: board private structure to initialize
3494  *
3495  * We allocate one q_vector per queue interrupt.  If allocation fails we
3496  * return -ENOMEM.
3497  */
3498 static int igc_alloc_q_vectors(struct igc_adapter *adapter)
3499 {
3500         int rxr_remaining = adapter->num_rx_queues;
3501         int txr_remaining = adapter->num_tx_queues;
3502         int rxr_idx = 0, txr_idx = 0, v_idx = 0;
3503         int q_vectors = adapter->num_q_vectors;
3504         int err;
3505
3506         if (q_vectors >= (rxr_remaining + txr_remaining)) {
3507                 for (; rxr_remaining; v_idx++) {
3508                         err = igc_alloc_q_vector(adapter, q_vectors, v_idx,
3509                                                  0, 0, 1, rxr_idx);
3510
3511                         if (err)
3512                                 goto err_out;
3513
3514                         /* update counts and index */
3515                         rxr_remaining--;
3516                         rxr_idx++;
3517                 }
3518         }
3519
3520         for (; v_idx < q_vectors; v_idx++) {
3521                 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx);
3522                 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx);
3523
3524                 err = igc_alloc_q_vector(adapter, q_vectors, v_idx,
3525                                          tqpv, txr_idx, rqpv, rxr_idx);
3526
3527                 if (err)
3528                         goto err_out;
3529
3530                 /* update counts and index */
3531                 rxr_remaining -= rqpv;
3532                 txr_remaining -= tqpv;
3533                 rxr_idx++;
3534                 txr_idx++;
3535         }
3536
3537         return 0;
3538
3539 err_out:
3540         adapter->num_tx_queues = 0;
3541         adapter->num_rx_queues = 0;
3542         adapter->num_q_vectors = 0;
3543
3544         while (v_idx--)
3545                 igc_free_q_vector(adapter, v_idx);
3546
3547         return -ENOMEM;
3548 }
3549
3550 /**
3551  * igc_cache_ring_register - Descriptor ring to register mapping
3552  * @adapter: board private structure to initialize
3553  *
3554  * Once we know the feature-set enabled for the device, we'll cache
3555  * the register offset the descriptor ring is assigned to.
3556  */
3557 static void igc_cache_ring_register(struct igc_adapter *adapter)
3558 {
3559         int i = 0, j = 0;
3560
3561         switch (adapter->hw.mac.type) {
3562         case igc_i225:
3563         /* Fall through */
3564         default:
3565                 for (; i < adapter->num_rx_queues; i++)
3566                         adapter->rx_ring[i]->reg_idx = i;
3567                 for (; j < adapter->num_tx_queues; j++)
3568                         adapter->tx_ring[j]->reg_idx = j;
3569                 break;
3570         }
3571 }
3572
3573 /**
3574  * igc_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
3575  * @adapter: Pointer to adapter structure
3576  *
3577  * This function initializes the interrupts and allocates all of the queues.
3578  */
3579 static int igc_init_interrupt_scheme(struct igc_adapter *adapter, bool msix)
3580 {
3581         struct pci_dev *pdev = adapter->pdev;
3582         int err = 0;
3583
3584         igc_set_interrupt_capability(adapter, msix);
3585
3586         err = igc_alloc_q_vectors(adapter);
3587         if (err) {
3588                 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
3589                 goto err_alloc_q_vectors;
3590         }
3591
3592         igc_cache_ring_register(adapter);
3593
3594         return 0;
3595
3596 err_alloc_q_vectors:
3597         igc_reset_interrupt_capability(adapter);
3598         return err;
3599 }
3600
3601 static void igc_free_irq(struct igc_adapter *adapter)
3602 {
3603         if (adapter->msix_entries) {
3604                 int vector = 0, i;
3605
3606                 free_irq(adapter->msix_entries[vector++].vector, adapter);
3607
3608                 for (i = 0; i < adapter->num_q_vectors; i++)
3609                         free_irq(adapter->msix_entries[vector++].vector,
3610                                  adapter->q_vector[i]);
3611         } else {
3612                 free_irq(adapter->pdev->irq, adapter);
3613         }
3614 }
3615
3616 /**
3617  * igc_irq_disable - Mask off interrupt generation on the NIC
3618  * @adapter: board private structure
3619  */
3620 static void igc_irq_disable(struct igc_adapter *adapter)
3621 {
3622         struct igc_hw *hw = &adapter->hw;
3623
3624         if (adapter->msix_entries) {
3625                 u32 regval = rd32(IGC_EIAM);
3626
3627                 wr32(IGC_EIAM, regval & ~adapter->eims_enable_mask);
3628                 wr32(IGC_EIMC, adapter->eims_enable_mask);
3629                 regval = rd32(IGC_EIAC);
3630                 wr32(IGC_EIAC, regval & ~adapter->eims_enable_mask);
3631         }
3632
3633         wr32(IGC_IAM, 0);
3634         wr32(IGC_IMC, ~0);
3635         wrfl();
3636
3637         if (adapter->msix_entries) {
3638                 int vector = 0, i;
3639
3640                 synchronize_irq(adapter->msix_entries[vector++].vector);
3641
3642                 for (i = 0; i < adapter->num_q_vectors; i++)
3643                         synchronize_irq(adapter->msix_entries[vector++].vector);
3644         } else {
3645                 synchronize_irq(adapter->pdev->irq);
3646         }
3647 }
3648
3649 /**
3650  * igc_irq_enable - Enable default interrupt generation settings
3651  * @adapter: board private structure
3652  */
3653 static void igc_irq_enable(struct igc_adapter *adapter)
3654 {
3655         struct igc_hw *hw = &adapter->hw;
3656
3657         if (adapter->msix_entries) {
3658                 u32 ims = IGC_IMS_LSC | IGC_IMS_DOUTSYNC | IGC_IMS_DRSTA;
3659                 u32 regval = rd32(IGC_EIAC);
3660
3661                 wr32(IGC_EIAC, regval | adapter->eims_enable_mask);
3662                 regval = rd32(IGC_EIAM);
3663                 wr32(IGC_EIAM, regval | adapter->eims_enable_mask);
3664                 wr32(IGC_EIMS, adapter->eims_enable_mask);
3665                 wr32(IGC_IMS, ims);
3666         } else {
3667                 wr32(IGC_IMS, IMS_ENABLE_MASK | IGC_IMS_DRSTA);
3668                 wr32(IGC_IAM, IMS_ENABLE_MASK | IGC_IMS_DRSTA);
3669         }
3670 }
3671
3672 /**
3673  * igc_request_irq - initialize interrupts
3674  * @adapter: Pointer to adapter structure
3675  *
3676  * Attempts to configure interrupts using the best available
3677  * capabilities of the hardware and kernel.
3678  */
3679 static int igc_request_irq(struct igc_adapter *adapter)
3680 {
3681         struct net_device *netdev = adapter->netdev;
3682         struct pci_dev *pdev = adapter->pdev;
3683         int err = 0;
3684
3685         if (adapter->flags & IGC_FLAG_HAS_MSIX) {
3686                 err = igc_request_msix(adapter);
3687                 if (!err)
3688                         goto request_done;
3689                 /* fall back to MSI */
3690                 igc_free_all_tx_resources(adapter);
3691                 igc_free_all_rx_resources(adapter);
3692
3693                 igc_clear_interrupt_scheme(adapter);
3694                 err = igc_init_interrupt_scheme(adapter, false);
3695                 if (err)
3696                         goto request_done;
3697                 igc_setup_all_tx_resources(adapter);
3698                 igc_setup_all_rx_resources(adapter);
3699                 igc_configure(adapter);
3700         }
3701
3702         igc_assign_vector(adapter->q_vector[0], 0);
3703
3704         if (adapter->flags & IGC_FLAG_HAS_MSI) {
3705                 err = request_irq(pdev->irq, &igc_intr_msi, 0,
3706                                   netdev->name, adapter);
3707                 if (!err)
3708                         goto request_done;
3709
3710                 /* fall back to legacy interrupts */
3711                 igc_reset_interrupt_capability(adapter);
3712                 adapter->flags &= ~IGC_FLAG_HAS_MSI;
3713         }
3714
3715         err = request_irq(pdev->irq, &igc_intr, IRQF_SHARED,
3716                           netdev->name, adapter);
3717
3718         if (err)
3719                 dev_err(&pdev->dev, "Error %d getting interrupt\n",
3720                         err);
3721
3722 request_done:
3723         return err;
3724 }
3725
3726 static void igc_write_itr(struct igc_q_vector *q_vector)
3727 {
3728         u32 itr_val = q_vector->itr_val & IGC_QVECTOR_MASK;
3729
3730         if (!q_vector->set_itr)
3731                 return;
3732
3733         if (!itr_val)
3734                 itr_val = IGC_ITR_VAL_MASK;
3735
3736         itr_val |= IGC_EITR_CNT_IGNR;
3737
3738         writel(itr_val, q_vector->itr_register);
3739         q_vector->set_itr = 0;
3740 }
3741
3742 /**
3743  * igc_open - Called when a network interface is made active
3744  * @netdev: network interface device structure
3745  *
3746  * Returns 0 on success, negative value on failure
3747  *
3748  * The open entry point is called when a network interface is made
3749  * active by the system (IFF_UP).  At this point all resources needed
3750  * for transmit and receive operations are allocated, the interrupt
3751  * handler is registered with the OS, the watchdog timer is started,
3752  * and the stack is notified that the interface is ready.
3753  */
3754 static int __igc_open(struct net_device *netdev, bool resuming)
3755 {
3756         struct igc_adapter *adapter = netdev_priv(netdev);
3757         struct igc_hw *hw = &adapter->hw;
3758         int err = 0;
3759         int i = 0;
3760
3761         /* disallow open during test */
3762
3763         if (test_bit(__IGC_TESTING, &adapter->state)) {
3764                 WARN_ON(resuming);
3765                 return -EBUSY;
3766         }
3767
3768         netif_carrier_off(netdev);
3769
3770         /* allocate transmit descriptors */
3771         err = igc_setup_all_tx_resources(adapter);
3772         if (err)
3773                 goto err_setup_tx;
3774
3775         /* allocate receive descriptors */
3776         err = igc_setup_all_rx_resources(adapter);
3777         if (err)
3778                 goto err_setup_rx;
3779
3780         igc_power_up_link(adapter);
3781
3782         igc_configure(adapter);
3783
3784         err = igc_request_irq(adapter);
3785         if (err)
3786                 goto err_req_irq;
3787
3788         /* Notify the stack of the actual queue counts. */
3789         err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues);
3790         if (err)
3791                 goto err_set_queues;
3792
3793         err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues);
3794         if (err)
3795                 goto err_set_queues;
3796
3797         clear_bit(__IGC_DOWN, &adapter->state);
3798
3799         for (i = 0; i < adapter->num_q_vectors; i++)
3800                 napi_enable(&adapter->q_vector[i]->napi);
3801
3802         /* Clear any pending interrupts. */
3803         rd32(IGC_ICR);
3804         igc_irq_enable(adapter);
3805
3806         netif_tx_start_all_queues(netdev);
3807
3808         /* start the watchdog. */
3809         hw->mac.get_link_status = 1;
3810         schedule_work(&adapter->watchdog_task);
3811
3812         return IGC_SUCCESS;
3813
3814 err_set_queues:
3815         igc_free_irq(adapter);
3816 err_req_irq:
3817         igc_release_hw_control(adapter);
3818         igc_power_down_link(adapter);
3819         igc_free_all_rx_resources(adapter);
3820 err_setup_rx:
3821         igc_free_all_tx_resources(adapter);
3822 err_setup_tx:
3823         igc_reset(adapter);
3824
3825         return err;
3826 }
3827
3828 static int igc_open(struct net_device *netdev)
3829 {
3830         return __igc_open(netdev, false);
3831 }
3832
3833 /**
3834  * igc_close - Disables a network interface
3835  * @netdev: network interface device structure
3836  *
3837  * Returns 0, this is not allowed to fail
3838  *
3839  * The close entry point is called when an interface is de-activated
3840  * by the OS.  The hardware is still under the driver's control, but
3841  * needs to be disabled.  A global MAC reset is issued to stop the
3842  * hardware, and all transmit and receive resources are freed.
3843  */
3844 static int __igc_close(struct net_device *netdev, bool suspending)
3845 {
3846         struct igc_adapter *adapter = netdev_priv(netdev);
3847
3848         WARN_ON(test_bit(__IGC_RESETTING, &adapter->state));
3849
3850         igc_down(adapter);
3851
3852         igc_release_hw_control(adapter);
3853
3854         igc_free_irq(adapter);
3855
3856         igc_free_all_tx_resources(adapter);
3857         igc_free_all_rx_resources(adapter);
3858
3859         return 0;
3860 }
3861
3862 static int igc_close(struct net_device *netdev)
3863 {
3864         if (netif_device_present(netdev) || netdev->dismantle)
3865                 return __igc_close(netdev, false);
3866         return 0;
3867 }
3868
3869 static const struct net_device_ops igc_netdev_ops = {
3870         .ndo_open               = igc_open,
3871         .ndo_stop               = igc_close,
3872         .ndo_start_xmit         = igc_xmit_frame,
3873         .ndo_set_mac_address    = igc_set_mac,
3874         .ndo_change_mtu         = igc_change_mtu,
3875         .ndo_get_stats          = igc_get_stats,
3876         .ndo_fix_features       = igc_fix_features,
3877         .ndo_set_features       = igc_set_features,
3878         .ndo_features_check     = igc_features_check,
3879 };
3880
3881 /* PCIe configuration access */
3882 void igc_read_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value)
3883 {
3884         struct igc_adapter *adapter = hw->back;
3885
3886         pci_read_config_word(adapter->pdev, reg, value);
3887 }
3888
3889 void igc_write_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value)
3890 {
3891         struct igc_adapter *adapter = hw->back;
3892
3893         pci_write_config_word(adapter->pdev, reg, *value);
3894 }
3895
3896 s32 igc_read_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value)
3897 {
3898         struct igc_adapter *adapter = hw->back;
3899         u16 cap_offset;
3900
3901         cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
3902         if (!cap_offset)
3903                 return -IGC_ERR_CONFIG;
3904
3905         pci_read_config_word(adapter->pdev, cap_offset + reg, value);
3906
3907         return IGC_SUCCESS;
3908 }
3909
3910 s32 igc_write_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value)
3911 {
3912         struct igc_adapter *adapter = hw->back;
3913         u16 cap_offset;
3914
3915         cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
3916         if (!cap_offset)
3917                 return -IGC_ERR_CONFIG;
3918
3919         pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
3920
3921         return IGC_SUCCESS;
3922 }
3923
3924 u32 igc_rd32(struct igc_hw *hw, u32 reg)
3925 {
3926         struct igc_adapter *igc = container_of(hw, struct igc_adapter, hw);
3927         u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr);
3928         u32 value = 0;
3929
3930         if (IGC_REMOVED(hw_addr))
3931                 return ~value;
3932
3933         value = readl(&hw_addr[reg]);
3934
3935         /* reads should not return all F's */
3936         if (!(~value) && (!reg || !(~readl(hw_addr)))) {
3937                 struct net_device *netdev = igc->netdev;
3938
3939                 hw->hw_addr = NULL;
3940                 netif_device_detach(netdev);
3941                 netdev_err(netdev, "PCIe link lost, device now detached\n");
3942         }
3943
3944         return value;
3945 }
3946
3947 int igc_set_spd_dplx(struct igc_adapter *adapter, u32 spd, u8 dplx)
3948 {
3949         struct pci_dev *pdev = adapter->pdev;
3950         struct igc_mac_info *mac = &adapter->hw.mac;
3951
3952         mac->autoneg = 0;
3953
3954         /* Make sure dplx is at most 1 bit and lsb of speed is not set
3955          * for the switch() below to work
3956          */
3957         if ((spd & 1) || (dplx & ~1))
3958                 goto err_inval;
3959
3960         switch (spd + dplx) {
3961         case SPEED_10 + DUPLEX_HALF:
3962                 mac->forced_speed_duplex = ADVERTISE_10_HALF;
3963                 break;
3964         case SPEED_10 + DUPLEX_FULL:
3965                 mac->forced_speed_duplex = ADVERTISE_10_FULL;
3966                 break;
3967         case SPEED_100 + DUPLEX_HALF:
3968                 mac->forced_speed_duplex = ADVERTISE_100_HALF;
3969                 break;
3970         case SPEED_100 + DUPLEX_FULL:
3971                 mac->forced_speed_duplex = ADVERTISE_100_FULL;
3972                 break;
3973         case SPEED_1000 + DUPLEX_FULL:
3974                 mac->autoneg = 1;
3975                 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
3976                 break;
3977         case SPEED_1000 + DUPLEX_HALF: /* not supported */
3978                 goto err_inval;
3979         case SPEED_2500 + DUPLEX_FULL:
3980                 mac->autoneg = 1;
3981                 adapter->hw.phy.autoneg_advertised = ADVERTISE_2500_FULL;
3982                 break;
3983         case SPEED_2500 + DUPLEX_HALF: /* not supported */
3984         default:
3985                 goto err_inval;
3986         }
3987
3988         /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */
3989         adapter->hw.phy.mdix = AUTO_ALL_MODES;
3990
3991         return 0;
3992
3993 err_inval:
3994         dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
3995         return -EINVAL;
3996 }
3997
3998 /**
3999  * igc_probe - Device Initialization Routine
4000  * @pdev: PCI device information struct
4001  * @ent: entry in igc_pci_tbl
4002  *
4003  * Returns 0 on success, negative on failure
4004  *
4005  * igc_probe initializes an adapter identified by a pci_dev structure.
4006  * The OS initialization, configuring the adapter private structure,
4007  * and a hardware reset occur.
4008  */
4009 static int igc_probe(struct pci_dev *pdev,
4010                      const struct pci_device_id *ent)
4011 {
4012         struct igc_adapter *adapter;
4013         struct net_device *netdev;
4014         struct igc_hw *hw;
4015         const struct igc_info *ei = igc_info_tbl[ent->driver_data];
4016         int err;
4017
4018         err = pci_enable_device_mem(pdev);
4019         if (err)
4020                 return err;
4021
4022         err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
4023         if (!err) {
4024                 err = dma_set_coherent_mask(&pdev->dev,
4025                                             DMA_BIT_MASK(64));
4026         } else {
4027                 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
4028                 if (err) {
4029                         err = dma_set_coherent_mask(&pdev->dev,
4030                                                     DMA_BIT_MASK(32));
4031                         if (err) {
4032                                 dev_err(&pdev->dev, "igc: Wrong DMA config\n");
4033                                 goto err_dma;
4034                         }
4035                 }
4036         }
4037
4038         err = pci_request_selected_regions(pdev,
4039                                            pci_select_bars(pdev,
4040                                                            IORESOURCE_MEM),
4041                                            igc_driver_name);
4042         if (err)
4043                 goto err_pci_reg;
4044
4045         pci_enable_pcie_error_reporting(pdev);
4046
4047         pci_set_master(pdev);
4048
4049         err = -ENOMEM;
4050         netdev = alloc_etherdev_mq(sizeof(struct igc_adapter),
4051                                    IGC_MAX_TX_QUEUES);
4052
4053         if (!netdev)
4054                 goto err_alloc_etherdev;
4055
4056         SET_NETDEV_DEV(netdev, &pdev->dev);
4057
4058         pci_set_drvdata(pdev, netdev);
4059         adapter = netdev_priv(netdev);
4060         adapter->netdev = netdev;
4061         adapter->pdev = pdev;
4062         hw = &adapter->hw;
4063         hw->back = adapter;
4064         adapter->port_num = hw->bus.func;
4065         adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
4066
4067         err = pci_save_state(pdev);
4068         if (err)
4069                 goto err_ioremap;
4070
4071         err = -EIO;
4072         adapter->io_addr = ioremap(pci_resource_start(pdev, 0),
4073                                    pci_resource_len(pdev, 0));
4074         if (!adapter->io_addr)
4075                 goto err_ioremap;
4076
4077         /* hw->hw_addr can be zeroed, so use adapter->io_addr for unmap */
4078         hw->hw_addr = adapter->io_addr;
4079
4080         netdev->netdev_ops = &igc_netdev_ops;
4081         igc_set_ethtool_ops(netdev);
4082         netdev->watchdog_timeo = 5 * HZ;
4083
4084         netdev->mem_start = pci_resource_start(pdev, 0);
4085         netdev->mem_end = pci_resource_end(pdev, 0);
4086
4087         /* PCI config space info */
4088         hw->vendor_id = pdev->vendor;
4089         hw->device_id = pdev->device;
4090         hw->revision_id = pdev->revision;
4091         hw->subsystem_vendor_id = pdev->subsystem_vendor;
4092         hw->subsystem_device_id = pdev->subsystem_device;
4093
4094         /* Copy the default MAC and PHY function pointers */
4095         memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
4096         memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
4097
4098         /* Initialize skew-specific constants */
4099         err = ei->get_invariants(hw);
4100         if (err)
4101                 goto err_sw_init;
4102
4103         /* setup the private structure */
4104         err = igc_sw_init(adapter);
4105         if (err)
4106                 goto err_sw_init;
4107
4108         /* copy netdev features into list of user selectable features */
4109         netdev->hw_features |= NETIF_F_NTUPLE;
4110
4111         /* MTU range: 68 - 9216 */
4112         netdev->min_mtu = ETH_MIN_MTU;
4113         netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE;
4114
4115         /* before reading the NVM, reset the controller to put the device in a
4116          * known good starting state
4117          */
4118         hw->mac.ops.reset_hw(hw);
4119
4120         if (eth_platform_get_mac_address(&pdev->dev, hw->mac.addr)) {
4121                 /* copy the MAC address out of the NVM */
4122                 if (hw->mac.ops.read_mac_addr(hw))
4123                         dev_err(&pdev->dev, "NVM Read Error\n");
4124         }
4125
4126         memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
4127
4128         if (!is_valid_ether_addr(netdev->dev_addr)) {
4129                 dev_err(&pdev->dev, "Invalid MAC Address\n");
4130                 err = -EIO;
4131                 goto err_eeprom;
4132         }
4133
4134         /* configure RXPBSIZE and TXPBSIZE */
4135         wr32(IGC_RXPBS, I225_RXPBSIZE_DEFAULT);
4136         wr32(IGC_TXPBS, I225_TXPBSIZE_DEFAULT);
4137
4138         timer_setup(&adapter->watchdog_timer, igc_watchdog, 0);
4139         timer_setup(&adapter->phy_info_timer, igc_update_phy_info, 0);
4140
4141         INIT_WORK(&adapter->reset_task, igc_reset_task);
4142         INIT_WORK(&adapter->watchdog_task, igc_watchdog_task);
4143
4144         /* Initialize link properties that are user-changeable */
4145         adapter->fc_autoneg = true;
4146         hw->mac.autoneg = true;
4147         hw->phy.autoneg_advertised = 0xaf;
4148
4149         hw->fc.requested_mode = igc_fc_default;
4150         hw->fc.current_mode = igc_fc_default;
4151
4152         /* reset the hardware with the new settings */
4153         igc_reset(adapter);
4154
4155         /* let the f/w know that the h/w is now under the control of the
4156          * driver.
4157          */
4158         igc_get_hw_control(adapter);
4159
4160         strncpy(netdev->name, "eth%d", IFNAMSIZ);
4161         err = register_netdev(netdev);
4162         if (err)
4163                 goto err_register;
4164
4165          /* carrier off reporting is important to ethtool even BEFORE open */
4166         netif_carrier_off(netdev);
4167
4168         /* Check if Media Autosense is enabled */
4169         adapter->ei = *ei;
4170
4171         /* print pcie link status and MAC address */
4172         pcie_print_link_status(pdev);
4173         netdev_info(netdev, "MAC: %pM\n", netdev->dev_addr);
4174
4175         return 0;
4176
4177 err_register:
4178         igc_release_hw_control(adapter);
4179 err_eeprom:
4180         if (!igc_check_reset_block(hw))
4181                 igc_reset_phy(hw);
4182 err_sw_init:
4183         igc_clear_interrupt_scheme(adapter);
4184         iounmap(adapter->io_addr);
4185 err_ioremap:
4186         free_netdev(netdev);
4187 err_alloc_etherdev:
4188         pci_release_selected_regions(pdev,
4189                                      pci_select_bars(pdev, IORESOURCE_MEM));
4190 err_pci_reg:
4191 err_dma:
4192         pci_disable_device(pdev);
4193         return err;
4194 }
4195
4196 /**
4197  * igc_remove - Device Removal Routine
4198  * @pdev: PCI device information struct
4199  *
4200  * igc_remove is called by the PCI subsystem to alert the driver
4201  * that it should release a PCI device.  This could be caused by a
4202  * Hot-Plug event, or because the driver is going to be removed from
4203  * memory.
4204  */
4205 static void igc_remove(struct pci_dev *pdev)
4206 {
4207         struct net_device *netdev = pci_get_drvdata(pdev);
4208         struct igc_adapter *adapter = netdev_priv(netdev);
4209
4210         set_bit(__IGC_DOWN, &adapter->state);
4211
4212         del_timer_sync(&adapter->watchdog_timer);
4213         del_timer_sync(&adapter->phy_info_timer);
4214
4215         cancel_work_sync(&adapter->reset_task);
4216         cancel_work_sync(&adapter->watchdog_task);
4217
4218         /* Release control of h/w to f/w.  If f/w is AMT enabled, this
4219          * would have already happened in close and is redundant.
4220          */
4221         igc_release_hw_control(adapter);
4222         unregister_netdev(netdev);
4223
4224         igc_clear_interrupt_scheme(adapter);
4225         pci_iounmap(pdev, adapter->io_addr);
4226         pci_release_mem_regions(pdev);
4227
4228         kfree(adapter->mac_table);
4229         kfree(adapter->shadow_vfta);
4230         free_netdev(netdev);
4231
4232         pci_disable_pcie_error_reporting(pdev);
4233
4234         pci_disable_device(pdev);
4235 }
4236
4237 static struct pci_driver igc_driver = {
4238         .name     = igc_driver_name,
4239         .id_table = igc_pci_tbl,
4240         .probe    = igc_probe,
4241         .remove   = igc_remove,
4242 };
4243
4244 void igc_set_flag_queue_pairs(struct igc_adapter *adapter,
4245                               const u32 max_rss_queues)
4246 {
4247         /* Determine if we need to pair queues. */
4248         /* If rss_queues > half of max_rss_queues, pair the queues in
4249          * order to conserve interrupts due to limited supply.
4250          */
4251         if (adapter->rss_queues > (max_rss_queues / 2))
4252                 adapter->flags |= IGC_FLAG_QUEUE_PAIRS;
4253         else
4254                 adapter->flags &= ~IGC_FLAG_QUEUE_PAIRS;
4255 }
4256
4257 unsigned int igc_get_max_rss_queues(struct igc_adapter *adapter)
4258 {
4259         unsigned int max_rss_queues;
4260
4261         /* Determine the maximum number of RSS queues supported. */
4262         max_rss_queues = IGC_MAX_RX_QUEUES;
4263
4264         return max_rss_queues;
4265 }
4266
4267 static void igc_init_queue_configuration(struct igc_adapter *adapter)
4268 {
4269         u32 max_rss_queues;
4270
4271         max_rss_queues = igc_get_max_rss_queues(adapter);
4272         adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus());
4273
4274         igc_set_flag_queue_pairs(adapter, max_rss_queues);
4275 }
4276
4277 /**
4278  * igc_sw_init - Initialize general software structures (struct igc_adapter)
4279  * @adapter: board private structure to initialize
4280  *
4281  * igc_sw_init initializes the Adapter private data structure.
4282  * Fields are initialized based on PCI device information and
4283  * OS network device settings (MTU size).
4284  */
4285 static int igc_sw_init(struct igc_adapter *adapter)
4286 {
4287         struct net_device *netdev = adapter->netdev;
4288         struct pci_dev *pdev = adapter->pdev;
4289         struct igc_hw *hw = &adapter->hw;
4290
4291         int size = sizeof(struct igc_mac_addr) * hw->mac.rar_entry_count;
4292
4293         pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
4294
4295         /* set default ring sizes */
4296         adapter->tx_ring_count = IGC_DEFAULT_TXD;
4297         adapter->rx_ring_count = IGC_DEFAULT_RXD;
4298
4299         /* set default ITR values */
4300         adapter->rx_itr_setting = IGC_DEFAULT_ITR;
4301         adapter->tx_itr_setting = IGC_DEFAULT_ITR;
4302
4303         /* set default work limits */
4304         adapter->tx_work_limit = IGC_DEFAULT_TX_WORK;
4305
4306         /* adjust max frame to be at least the size of a standard frame */
4307         adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
4308                                 VLAN_HLEN;
4309         adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
4310
4311         spin_lock_init(&adapter->nfc_lock);
4312         spin_lock_init(&adapter->stats64_lock);
4313         /* Assume MSI-X interrupts, will be checked during IRQ allocation */
4314         adapter->flags |= IGC_FLAG_HAS_MSIX;
4315
4316         adapter->mac_table = kzalloc(size, GFP_ATOMIC);
4317         if (!adapter->mac_table)
4318                 return -ENOMEM;
4319
4320         igc_init_queue_configuration(adapter);
4321
4322         /* This call may decrease the number of queues */
4323         if (igc_init_interrupt_scheme(adapter, true)) {
4324                 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
4325                 return -ENOMEM;
4326         }
4327
4328         /* Explicitly disable IRQ since the NIC can be in any state. */
4329         igc_irq_disable(adapter);
4330
4331         set_bit(__IGC_DOWN, &adapter->state);
4332
4333         return 0;
4334 }
4335
4336 /**
4337  * igc_reinit_queues - return error
4338  * @adapter: pointer to adapter structure
4339  */
4340 int igc_reinit_queues(struct igc_adapter *adapter)
4341 {
4342         struct net_device *netdev = adapter->netdev;
4343         struct pci_dev *pdev = adapter->pdev;
4344         int err = 0;
4345
4346         if (netif_running(netdev))
4347                 igc_close(netdev);
4348
4349         igc_reset_interrupt_capability(adapter);
4350
4351         if (igc_init_interrupt_scheme(adapter, true)) {
4352                 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
4353                 return -ENOMEM;
4354         }
4355
4356         if (netif_running(netdev))
4357                 err = igc_open(netdev);
4358
4359         return err;
4360 }
4361
4362 /**
4363  * igc_get_hw_dev - return device
4364  * @hw: pointer to hardware structure
4365  *
4366  * used by hardware layer to print debugging information
4367  */
4368 struct net_device *igc_get_hw_dev(struct igc_hw *hw)
4369 {
4370         struct igc_adapter *adapter = hw->back;
4371
4372         return adapter->netdev;
4373 }
4374
4375 /**
4376  * igc_init_module - Driver Registration Routine
4377  *
4378  * igc_init_module is the first routine called when the driver is
4379  * loaded. All it does is register with the PCI subsystem.
4380  */
4381 static int __init igc_init_module(void)
4382 {
4383         int ret;
4384
4385         pr_info("%s - version %s\n",
4386                 igc_driver_string, igc_driver_version);
4387
4388         pr_info("%s\n", igc_copyright);
4389
4390         ret = pci_register_driver(&igc_driver);
4391         return ret;
4392 }
4393
4394 module_init(igc_init_module);
4395
4396 /**
4397  * igc_exit_module - Driver Exit Cleanup Routine
4398  *
4399  * igc_exit_module is called just before the driver is removed
4400  * from memory.
4401  */
4402 static void __exit igc_exit_module(void)
4403 {
4404         pci_unregister_driver(&igc_driver);
4405 }
4406
4407 module_exit(igc_exit_module);
4408 /* igc_main.c */