1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2018 Intel Corporation */
4 #include <linux/module.h>
5 #include <linux/types.h>
6 #include <linux/if_vlan.h>
12 #define DRV_VERSION "0.0.1-k"
13 #define DRV_SUMMARY "Intel(R) 2.5G Ethernet Linux Driver"
15 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
17 static int debug = -1;
19 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
20 MODULE_DESCRIPTION(DRV_SUMMARY);
21 MODULE_LICENSE("GPL v2");
22 MODULE_VERSION(DRV_VERSION);
23 module_param(debug, int, 0);
24 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
26 char igc_driver_name[] = "igc";
27 char igc_driver_version[] = DRV_VERSION;
28 static const char igc_driver_string[] = DRV_SUMMARY;
29 static const char igc_copyright[] =
30 "Copyright(c) 2018 Intel Corporation.";
32 static const struct igc_info *igc_info_tbl[] = {
33 [board_base] = &igc_base_info,
36 static const struct pci_device_id igc_pci_tbl[] = {
37 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LM), board_base },
38 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_V), board_base },
39 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_I), board_base },
40 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I220_V), board_base },
41 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K), board_base },
42 /* required last entry */
46 MODULE_DEVICE_TABLE(pci, igc_pci_tbl);
48 /* forward declaration */
49 static void igc_clean_tx_ring(struct igc_ring *tx_ring);
50 static int igc_sw_init(struct igc_adapter *);
51 static void igc_configure(struct igc_adapter *adapter);
52 static void igc_power_down_link(struct igc_adapter *adapter);
53 static void igc_set_default_mac_filter(struct igc_adapter *adapter);
54 static void igc_set_rx_mode(struct net_device *netdev);
55 static void igc_write_itr(struct igc_q_vector *q_vector);
56 static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector);
57 static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx);
58 static void igc_set_interrupt_capability(struct igc_adapter *adapter,
60 static void igc_free_q_vectors(struct igc_adapter *adapter);
61 static void igc_irq_disable(struct igc_adapter *adapter);
62 static void igc_irq_enable(struct igc_adapter *adapter);
63 static void igc_configure_msix(struct igc_adapter *adapter);
64 static bool igc_alloc_mapped_page(struct igc_ring *rx_ring,
65 struct igc_rx_buffer *bi);
74 void igc_reset(struct igc_adapter *adapter)
76 struct pci_dev *pdev = adapter->pdev;
77 struct igc_hw *hw = &adapter->hw;
78 struct igc_fc_info *fc = &hw->fc;
81 /* Repartition PBA for greater than 9k MTU if required */
84 /* flow control settings
85 * The high water mark must be low enough to fit one full frame
86 * after transmitting the pause frame. As such we must have enough
87 * space to allow for us to complete our current transmit and then
88 * receive the frame that is in progress from the link partner.
90 * - the full Rx FIFO size minus one full Tx plus one full Rx frame
92 hwm = (pba << 10) - (adapter->max_frame_size + MAX_JUMBO_FRAME_SIZE);
94 fc->high_water = hwm & 0xFFFFFFF0; /* 16-byte granularity */
95 fc->low_water = fc->high_water - 16;
96 fc->pause_time = 0xFFFF;
98 fc->current_mode = fc->requested_mode;
100 hw->mac.ops.reset_hw(hw);
102 if (hw->mac.ops.init_hw(hw))
103 dev_err(&pdev->dev, "Hardware Error\n");
105 if (!netif_running(adapter->netdev))
106 igc_power_down_link(adapter);
108 igc_get_phy_info(hw);
112 * igc_power_up_link - Power up the phy/serdes link
113 * @adapter: address of board private structure
115 static void igc_power_up_link(struct igc_adapter *adapter)
117 igc_reset_phy(&adapter->hw);
119 if (adapter->hw.phy.media_type == igc_media_type_copper)
120 igc_power_up_phy_copper(&adapter->hw);
122 igc_setup_link(&adapter->hw);
126 * igc_power_down_link - Power down the phy/serdes link
127 * @adapter: address of board private structure
129 static void igc_power_down_link(struct igc_adapter *adapter)
131 if (adapter->hw.phy.media_type == igc_media_type_copper)
132 igc_power_down_phy_copper_base(&adapter->hw);
136 * igc_release_hw_control - release control of the h/w to f/w
137 * @adapter: address of board private structure
139 * igc_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
140 * For ASF and Pass Through versions of f/w this means that the
141 * driver is no longer loaded.
143 static void igc_release_hw_control(struct igc_adapter *adapter)
145 struct igc_hw *hw = &adapter->hw;
148 /* Let firmware take over control of h/w */
149 ctrl_ext = rd32(IGC_CTRL_EXT);
151 ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD);
155 * igc_get_hw_control - get control of the h/w from f/w
156 * @adapter: address of board private structure
158 * igc_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
159 * For ASF and Pass Through versions of f/w this means that
160 * the driver is loaded.
162 static void igc_get_hw_control(struct igc_adapter *adapter)
164 struct igc_hw *hw = &adapter->hw;
167 /* Let firmware know the driver has taken over */
168 ctrl_ext = rd32(IGC_CTRL_EXT);
170 ctrl_ext | IGC_CTRL_EXT_DRV_LOAD);
174 * igc_free_tx_resources - Free Tx Resources per Queue
175 * @tx_ring: Tx descriptor ring for a specific queue
177 * Free all transmit software resources
179 void igc_free_tx_resources(struct igc_ring *tx_ring)
181 igc_clean_tx_ring(tx_ring);
183 vfree(tx_ring->tx_buffer_info);
184 tx_ring->tx_buffer_info = NULL;
186 /* if not set, then don't free */
190 dma_free_coherent(tx_ring->dev, tx_ring->size,
191 tx_ring->desc, tx_ring->dma);
193 tx_ring->desc = NULL;
197 * igc_free_all_tx_resources - Free Tx Resources for All Queues
198 * @adapter: board private structure
200 * Free all transmit software resources
202 static void igc_free_all_tx_resources(struct igc_adapter *adapter)
206 for (i = 0; i < adapter->num_tx_queues; i++)
207 igc_free_tx_resources(adapter->tx_ring[i]);
211 * igc_clean_tx_ring - Free Tx Buffers
212 * @tx_ring: ring to be cleaned
214 static void igc_clean_tx_ring(struct igc_ring *tx_ring)
216 u16 i = tx_ring->next_to_clean;
217 struct igc_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i];
219 while (i != tx_ring->next_to_use) {
220 union igc_adv_tx_desc *eop_desc, *tx_desc;
222 /* Free all the Tx ring sk_buffs */
223 dev_kfree_skb_any(tx_buffer->skb);
225 /* unmap skb header data */
226 dma_unmap_single(tx_ring->dev,
227 dma_unmap_addr(tx_buffer, dma),
228 dma_unmap_len(tx_buffer, len),
231 /* check for eop_desc to determine the end of the packet */
232 eop_desc = tx_buffer->next_to_watch;
233 tx_desc = IGC_TX_DESC(tx_ring, i);
235 /* unmap remaining buffers */
236 while (tx_desc != eop_desc) {
240 if (unlikely(i == tx_ring->count)) {
242 tx_buffer = tx_ring->tx_buffer_info;
243 tx_desc = IGC_TX_DESC(tx_ring, 0);
246 /* unmap any remaining paged data */
247 if (dma_unmap_len(tx_buffer, len))
248 dma_unmap_page(tx_ring->dev,
249 dma_unmap_addr(tx_buffer, dma),
250 dma_unmap_len(tx_buffer, len),
254 /* move us one more past the eop_desc for start of next pkt */
257 if (unlikely(i == tx_ring->count)) {
259 tx_buffer = tx_ring->tx_buffer_info;
263 /* reset BQL for queue */
264 netdev_tx_reset_queue(txring_txq(tx_ring));
266 /* reset next_to_use and next_to_clean */
267 tx_ring->next_to_use = 0;
268 tx_ring->next_to_clean = 0;
272 * igc_clean_all_tx_rings - Free Tx Buffers for all queues
273 * @adapter: board private structure
275 static void igc_clean_all_tx_rings(struct igc_adapter *adapter)
279 for (i = 0; i < adapter->num_tx_queues; i++)
280 if (adapter->tx_ring[i])
281 igc_clean_tx_ring(adapter->tx_ring[i]);
285 * igc_setup_tx_resources - allocate Tx resources (Descriptors)
286 * @tx_ring: tx descriptor ring (for a specific queue) to setup
288 * Return 0 on success, negative on failure
290 int igc_setup_tx_resources(struct igc_ring *tx_ring)
292 struct device *dev = tx_ring->dev;
295 size = sizeof(struct igc_tx_buffer) * tx_ring->count;
296 tx_ring->tx_buffer_info = vzalloc(size);
297 if (!tx_ring->tx_buffer_info)
300 /* round up to nearest 4K */
301 tx_ring->size = tx_ring->count * sizeof(union igc_adv_tx_desc);
302 tx_ring->size = ALIGN(tx_ring->size, 4096);
304 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
305 &tx_ring->dma, GFP_KERNEL);
310 tx_ring->next_to_use = 0;
311 tx_ring->next_to_clean = 0;
316 vfree(tx_ring->tx_buffer_info);
318 "Unable to allocate memory for the transmit descriptor ring\n");
323 * igc_setup_all_tx_resources - wrapper to allocate Tx resources for all queues
324 * @adapter: board private structure
326 * Return 0 on success, negative on failure
328 static int igc_setup_all_tx_resources(struct igc_adapter *adapter)
330 struct pci_dev *pdev = adapter->pdev;
333 for (i = 0; i < adapter->num_tx_queues; i++) {
334 err = igc_setup_tx_resources(adapter->tx_ring[i]);
337 "Allocation for Tx Queue %u failed\n", i);
338 for (i--; i >= 0; i--)
339 igc_free_tx_resources(adapter->tx_ring[i]);
348 * igc_clean_rx_ring - Free Rx Buffers per Queue
349 * @rx_ring: ring to free buffers from
351 static void igc_clean_rx_ring(struct igc_ring *rx_ring)
353 u16 i = rx_ring->next_to_clean;
355 dev_kfree_skb(rx_ring->skb);
358 /* Free all the Rx ring sk_buffs */
359 while (i != rx_ring->next_to_alloc) {
360 struct igc_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
362 /* Invalidate cache lines that may have been written to by
363 * device so that we avoid corrupting memory.
365 dma_sync_single_range_for_cpu(rx_ring->dev,
367 buffer_info->page_offset,
368 igc_rx_bufsz(rx_ring),
371 /* free resources associated with mapping */
372 dma_unmap_page_attrs(rx_ring->dev,
374 igc_rx_pg_size(rx_ring),
377 __page_frag_cache_drain(buffer_info->page,
378 buffer_info->pagecnt_bias);
381 if (i == rx_ring->count)
385 rx_ring->next_to_alloc = 0;
386 rx_ring->next_to_clean = 0;
387 rx_ring->next_to_use = 0;
391 * igc_clean_all_rx_rings - Free Rx Buffers for all queues
392 * @adapter: board private structure
394 static void igc_clean_all_rx_rings(struct igc_adapter *adapter)
398 for (i = 0; i < adapter->num_rx_queues; i++)
399 if (adapter->rx_ring[i])
400 igc_clean_rx_ring(adapter->rx_ring[i]);
404 * igc_free_rx_resources - Free Rx Resources
405 * @rx_ring: ring to clean the resources from
407 * Free all receive software resources
409 void igc_free_rx_resources(struct igc_ring *rx_ring)
411 igc_clean_rx_ring(rx_ring);
413 vfree(rx_ring->rx_buffer_info);
414 rx_ring->rx_buffer_info = NULL;
416 /* if not set, then don't free */
420 dma_free_coherent(rx_ring->dev, rx_ring->size,
421 rx_ring->desc, rx_ring->dma);
423 rx_ring->desc = NULL;
427 * igc_free_all_rx_resources - Free Rx Resources for All Queues
428 * @adapter: board private structure
430 * Free all receive software resources
432 static void igc_free_all_rx_resources(struct igc_adapter *adapter)
436 for (i = 0; i < adapter->num_rx_queues; i++)
437 igc_free_rx_resources(adapter->rx_ring[i]);
441 * igc_setup_rx_resources - allocate Rx resources (Descriptors)
442 * @rx_ring: rx descriptor ring (for a specific queue) to setup
444 * Returns 0 on success, negative on failure
446 int igc_setup_rx_resources(struct igc_ring *rx_ring)
448 struct device *dev = rx_ring->dev;
451 size = sizeof(struct igc_rx_buffer) * rx_ring->count;
452 rx_ring->rx_buffer_info = vzalloc(size);
453 if (!rx_ring->rx_buffer_info)
456 desc_len = sizeof(union igc_adv_rx_desc);
458 /* Round up to nearest 4K */
459 rx_ring->size = rx_ring->count * desc_len;
460 rx_ring->size = ALIGN(rx_ring->size, 4096);
462 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
463 &rx_ring->dma, GFP_KERNEL);
468 rx_ring->next_to_alloc = 0;
469 rx_ring->next_to_clean = 0;
470 rx_ring->next_to_use = 0;
475 vfree(rx_ring->rx_buffer_info);
476 rx_ring->rx_buffer_info = NULL;
478 "Unable to allocate memory for the receive descriptor ring\n");
483 * igc_setup_all_rx_resources - wrapper to allocate Rx resources
484 * (Descriptors) for all queues
485 * @adapter: board private structure
487 * Return 0 on success, negative on failure
489 static int igc_setup_all_rx_resources(struct igc_adapter *adapter)
491 struct pci_dev *pdev = adapter->pdev;
494 for (i = 0; i < adapter->num_rx_queues; i++) {
495 err = igc_setup_rx_resources(adapter->rx_ring[i]);
498 "Allocation for Rx Queue %u failed\n", i);
499 for (i--; i >= 0; i--)
500 igc_free_rx_resources(adapter->rx_ring[i]);
509 * igc_configure_rx_ring - Configure a receive ring after Reset
510 * @adapter: board private structure
511 * @ring: receive ring to be configured
513 * Configure the Rx unit of the MAC after a reset.
515 static void igc_configure_rx_ring(struct igc_adapter *adapter,
516 struct igc_ring *ring)
518 struct igc_hw *hw = &adapter->hw;
519 union igc_adv_rx_desc *rx_desc;
520 int reg_idx = ring->reg_idx;
521 u32 srrctl = 0, rxdctl = 0;
522 u64 rdba = ring->dma;
524 /* disable the queue */
525 wr32(IGC_RXDCTL(reg_idx), 0);
527 /* Set DMA base address registers */
528 wr32(IGC_RDBAL(reg_idx),
529 rdba & 0x00000000ffffffffULL);
530 wr32(IGC_RDBAH(reg_idx), rdba >> 32);
531 wr32(IGC_RDLEN(reg_idx),
532 ring->count * sizeof(union igc_adv_rx_desc));
534 /* initialize head and tail */
535 ring->tail = adapter->io_addr + IGC_RDT(reg_idx);
536 wr32(IGC_RDH(reg_idx), 0);
537 writel(0, ring->tail);
539 /* reset next-to- use/clean to place SW in sync with hardware */
540 ring->next_to_clean = 0;
541 ring->next_to_use = 0;
543 /* set descriptor configuration */
544 srrctl = IGC_RX_HDR_LEN << IGC_SRRCTL_BSIZEHDRSIZE_SHIFT;
545 if (ring_uses_large_buffer(ring))
546 srrctl |= IGC_RXBUFFER_3072 >> IGC_SRRCTL_BSIZEPKT_SHIFT;
548 srrctl |= IGC_RXBUFFER_2048 >> IGC_SRRCTL_BSIZEPKT_SHIFT;
549 srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF;
551 wr32(IGC_SRRCTL(reg_idx), srrctl);
553 rxdctl |= IGC_RX_PTHRESH;
554 rxdctl |= IGC_RX_HTHRESH << 8;
555 rxdctl |= IGC_RX_WTHRESH << 16;
557 /* initialize rx_buffer_info */
558 memset(ring->rx_buffer_info, 0,
559 sizeof(struct igc_rx_buffer) * ring->count);
561 /* initialize Rx descriptor 0 */
562 rx_desc = IGC_RX_DESC(ring, 0);
563 rx_desc->wb.upper.length = 0;
565 /* enable receive descriptor fetching */
566 rxdctl |= IGC_RXDCTL_QUEUE_ENABLE;
568 wr32(IGC_RXDCTL(reg_idx), rxdctl);
572 * igc_configure_rx - Configure receive Unit after Reset
573 * @adapter: board private structure
575 * Configure the Rx unit of the MAC after a reset.
577 static void igc_configure_rx(struct igc_adapter *adapter)
581 /* Setup the HW Rx Head and Tail Descriptor Pointers and
582 * the Base and Length of the Rx Descriptor Ring
584 for (i = 0; i < adapter->num_rx_queues; i++)
585 igc_configure_rx_ring(adapter, adapter->rx_ring[i]);
589 * igc_configure_tx_ring - Configure transmit ring after Reset
590 * @adapter: board private structure
591 * @ring: tx ring to configure
593 * Configure a transmit ring after a reset.
595 static void igc_configure_tx_ring(struct igc_adapter *adapter,
596 struct igc_ring *ring)
598 struct igc_hw *hw = &adapter->hw;
599 int reg_idx = ring->reg_idx;
600 u64 tdba = ring->dma;
603 /* disable the queue */
604 wr32(IGC_TXDCTL(reg_idx), 0);
608 wr32(IGC_TDLEN(reg_idx),
609 ring->count * sizeof(union igc_adv_tx_desc));
610 wr32(IGC_TDBAL(reg_idx),
611 tdba & 0x00000000ffffffffULL);
612 wr32(IGC_TDBAH(reg_idx), tdba >> 32);
614 ring->tail = adapter->io_addr + IGC_TDT(reg_idx);
615 wr32(IGC_TDH(reg_idx), 0);
616 writel(0, ring->tail);
618 txdctl |= IGC_TX_PTHRESH;
619 txdctl |= IGC_TX_HTHRESH << 8;
620 txdctl |= IGC_TX_WTHRESH << 16;
622 txdctl |= IGC_TXDCTL_QUEUE_ENABLE;
623 wr32(IGC_TXDCTL(reg_idx), txdctl);
627 * igc_configure_tx - Configure transmit Unit after Reset
628 * @adapter: board private structure
630 * Configure the Tx unit of the MAC after a reset.
632 static void igc_configure_tx(struct igc_adapter *adapter)
636 for (i = 0; i < adapter->num_tx_queues; i++)
637 igc_configure_tx_ring(adapter, adapter->tx_ring[i]);
641 * igc_setup_mrqc - configure the multiple receive queue control registers
642 * @adapter: Board private structure
644 static void igc_setup_mrqc(struct igc_adapter *adapter)
646 struct igc_hw *hw = &adapter->hw;
647 u32 j, num_rx_queues;
651 netdev_rss_key_fill(rss_key, sizeof(rss_key));
652 for (j = 0; j < 10; j++)
653 wr32(IGC_RSSRK(j), rss_key[j]);
655 num_rx_queues = adapter->rss_queues;
657 if (adapter->rss_indir_tbl_init != num_rx_queues) {
658 for (j = 0; j < IGC_RETA_SIZE; j++)
659 adapter->rss_indir_tbl[j] =
660 (j * num_rx_queues) / IGC_RETA_SIZE;
661 adapter->rss_indir_tbl_init = num_rx_queues;
663 igc_write_rss_indir_tbl(adapter);
665 /* Disable raw packet checksumming so that RSS hash is placed in
666 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
667 * offloads as they are enabled by default
669 rxcsum = rd32(IGC_RXCSUM);
670 rxcsum |= IGC_RXCSUM_PCSD;
672 /* Enable Receive Checksum Offload for SCTP */
673 rxcsum |= IGC_RXCSUM_CRCOFL;
675 /* Don't need to set TUOFL or IPOFL, they default to 1 */
676 wr32(IGC_RXCSUM, rxcsum);
678 /* Generate RSS hash based on packet types, TCP/UDP
679 * port numbers and/or IPv4/v6 src and dst addresses
681 mrqc = IGC_MRQC_RSS_FIELD_IPV4 |
682 IGC_MRQC_RSS_FIELD_IPV4_TCP |
683 IGC_MRQC_RSS_FIELD_IPV6 |
684 IGC_MRQC_RSS_FIELD_IPV6_TCP |
685 IGC_MRQC_RSS_FIELD_IPV6_TCP_EX;
687 if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV4_UDP)
688 mrqc |= IGC_MRQC_RSS_FIELD_IPV4_UDP;
689 if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV6_UDP)
690 mrqc |= IGC_MRQC_RSS_FIELD_IPV6_UDP;
692 mrqc |= IGC_MRQC_ENABLE_RSS_MQ;
694 wr32(IGC_MRQC, mrqc);
698 * igc_setup_rctl - configure the receive control registers
699 * @adapter: Board private structure
701 static void igc_setup_rctl(struct igc_adapter *adapter)
703 struct igc_hw *hw = &adapter->hw;
706 rctl = rd32(IGC_RCTL);
708 rctl &= ~(3 << IGC_RCTL_MO_SHIFT);
709 rctl &= ~(IGC_RCTL_LBM_TCVR | IGC_RCTL_LBM_MAC);
711 rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_RDMTS_HALF |
712 (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT);
714 /* enable stripping of CRC. Newer features require
715 * that the HW strips the CRC.
717 rctl |= IGC_RCTL_SECRC;
719 /* disable store bad packets and clear size bits. */
720 rctl &= ~(IGC_RCTL_SBP | IGC_RCTL_SZ_256);
722 /* enable LPE to allow for reception of jumbo frames */
723 rctl |= IGC_RCTL_LPE;
725 /* disable queue 0 to prevent tail write w/o re-config */
726 wr32(IGC_RXDCTL(0), 0);
728 /* This is useful for sniffing bad packets. */
729 if (adapter->netdev->features & NETIF_F_RXALL) {
730 /* UPE and MPE will be handled by normal PROMISC logic
733 rctl |= (IGC_RCTL_SBP | /* Receive bad packets */
734 IGC_RCTL_BAM | /* RX All Bcast Pkts */
735 IGC_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
737 rctl &= ~(IGC_RCTL_DPF | /* Allow filtered pause */
738 IGC_RCTL_CFIEN); /* Disable VLAN CFIEN Filter */
741 wr32(IGC_RCTL, rctl);
745 * igc_setup_tctl - configure the transmit control registers
746 * @adapter: Board private structure
748 static void igc_setup_tctl(struct igc_adapter *adapter)
750 struct igc_hw *hw = &adapter->hw;
753 /* disable queue 0 which icould be enabled by default */
754 wr32(IGC_TXDCTL(0), 0);
756 /* Program the Transmit Control Register */
757 tctl = rd32(IGC_TCTL);
758 tctl &= ~IGC_TCTL_CT;
759 tctl |= IGC_TCTL_PSP | IGC_TCTL_RTLC |
760 (IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT);
762 /* Enable transmits */
765 wr32(IGC_TCTL, tctl);
769 * igc_set_mac - Change the Ethernet Address of the NIC
770 * @netdev: network interface device structure
771 * @p: pointer to an address structure
773 * Returns 0 on success, negative on failure
775 static int igc_set_mac(struct net_device *netdev, void *p)
777 struct igc_adapter *adapter = netdev_priv(netdev);
778 struct igc_hw *hw = &adapter->hw;
779 struct sockaddr *addr = p;
781 if (!is_valid_ether_addr(addr->sa_data))
782 return -EADDRNOTAVAIL;
784 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
785 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
787 /* set the correct pool for the new PF MAC address in entry 0 */
788 igc_set_default_mac_filter(adapter);
793 static void igc_tx_csum(struct igc_ring *tx_ring, struct igc_tx_buffer *first)
797 static int __igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size)
799 struct net_device *netdev = tx_ring->netdev;
801 netif_stop_subqueue(netdev, tx_ring->queue_index);
803 /* memory barriier comment */
806 /* We need to check again in a case another CPU has just
807 * made room available.
809 if (igc_desc_unused(tx_ring) < size)
813 netif_wake_subqueue(netdev, tx_ring->queue_index);
815 u64_stats_update_begin(&tx_ring->tx_syncp2);
816 tx_ring->tx_stats.restart_queue2++;
817 u64_stats_update_end(&tx_ring->tx_syncp2);
822 static inline int igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size)
824 if (igc_desc_unused(tx_ring) >= size)
826 return __igc_maybe_stop_tx(tx_ring, size);
829 static u32 igc_tx_cmd_type(struct sk_buff *skb, u32 tx_flags)
831 /* set type for advanced descriptor with frame checksum insertion */
832 u32 cmd_type = IGC_ADVTXD_DTYP_DATA |
833 IGC_ADVTXD_DCMD_DEXT |
834 IGC_ADVTXD_DCMD_IFCS;
839 static void igc_tx_olinfo_status(struct igc_ring *tx_ring,
840 union igc_adv_tx_desc *tx_desc,
841 u32 tx_flags, unsigned int paylen)
843 u32 olinfo_status = paylen << IGC_ADVTXD_PAYLEN_SHIFT;
845 /* insert L4 checksum */
846 olinfo_status |= (tx_flags & IGC_TX_FLAGS_CSUM) *
847 ((IGC_TXD_POPTS_TXSM << 8) /
850 /* insert IPv4 checksum */
851 olinfo_status |= (tx_flags & IGC_TX_FLAGS_IPV4) *
852 (((IGC_TXD_POPTS_IXSM << 8)) /
855 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
858 static int igc_tx_map(struct igc_ring *tx_ring,
859 struct igc_tx_buffer *first,
862 struct sk_buff *skb = first->skb;
863 struct igc_tx_buffer *tx_buffer;
864 union igc_adv_tx_desc *tx_desc;
865 u32 tx_flags = first->tx_flags;
867 u16 i = tx_ring->next_to_use;
868 unsigned int data_len, size;
870 u32 cmd_type = igc_tx_cmd_type(skb, tx_flags);
872 tx_desc = IGC_TX_DESC(tx_ring, i);
874 igc_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len);
876 size = skb_headlen(skb);
877 data_len = skb->data_len;
879 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
883 for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
884 if (dma_mapping_error(tx_ring->dev, dma))
887 /* record length, and DMA address */
888 dma_unmap_len_set(tx_buffer, len, size);
889 dma_unmap_addr_set(tx_buffer, dma, dma);
891 tx_desc->read.buffer_addr = cpu_to_le64(dma);
893 while (unlikely(size > IGC_MAX_DATA_PER_TXD)) {
894 tx_desc->read.cmd_type_len =
895 cpu_to_le32(cmd_type ^ IGC_MAX_DATA_PER_TXD);
899 if (i == tx_ring->count) {
900 tx_desc = IGC_TX_DESC(tx_ring, 0);
903 tx_desc->read.olinfo_status = 0;
905 dma += IGC_MAX_DATA_PER_TXD;
906 size -= IGC_MAX_DATA_PER_TXD;
908 tx_desc->read.buffer_addr = cpu_to_le64(dma);
911 if (likely(!data_len))
914 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size);
918 if (i == tx_ring->count) {
919 tx_desc = IGC_TX_DESC(tx_ring, 0);
922 tx_desc->read.olinfo_status = 0;
924 size = skb_frag_size(frag);
927 dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
928 size, DMA_TO_DEVICE);
930 tx_buffer = &tx_ring->tx_buffer_info[i];
933 /* write last descriptor with RS and EOP bits */
934 cmd_type |= size | IGC_TXD_DCMD;
935 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
937 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
939 /* set the timestamp */
940 first->time_stamp = jiffies;
942 skb_tx_timestamp(skb);
944 /* Force memory writes to complete before letting h/w know there
945 * are new descriptors to fetch. (Only applicable for weak-ordered
946 * memory model archs, such as IA-64).
948 * We also need this memory barrier to make certain all of the
949 * status bits have been updated before next_to_watch is written.
953 /* set next_to_watch value indicating a packet is present */
954 first->next_to_watch = tx_desc;
957 if (i == tx_ring->count)
960 tx_ring->next_to_use = i;
962 /* Make sure there is space in the ring for the next send. */
963 igc_maybe_stop_tx(tx_ring, DESC_NEEDED);
965 if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) {
966 writel(i, tx_ring->tail);
971 dev_err(tx_ring->dev, "TX DMA map failed\n");
972 tx_buffer = &tx_ring->tx_buffer_info[i];
974 /* clear dma mappings for failed tx_buffer_info map */
975 while (tx_buffer != first) {
976 if (dma_unmap_len(tx_buffer, len))
977 dma_unmap_page(tx_ring->dev,
978 dma_unmap_addr(tx_buffer, dma),
979 dma_unmap_len(tx_buffer, len),
981 dma_unmap_len_set(tx_buffer, len, 0);
985 tx_buffer = &tx_ring->tx_buffer_info[i];
988 if (dma_unmap_len(tx_buffer, len))
989 dma_unmap_single(tx_ring->dev,
990 dma_unmap_addr(tx_buffer, dma),
991 dma_unmap_len(tx_buffer, len),
993 dma_unmap_len_set(tx_buffer, len, 0);
995 dev_kfree_skb_any(tx_buffer->skb);
996 tx_buffer->skb = NULL;
998 tx_ring->next_to_use = i;
1003 static netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb,
1004 struct igc_ring *tx_ring)
1006 u16 count = TXD_USE_COUNT(skb_headlen(skb));
1007 __be16 protocol = vlan_get_protocol(skb);
1008 struct igc_tx_buffer *first;
1013 /* need: 1 descriptor per page * PAGE_SIZE/IGC_MAX_DATA_PER_TXD,
1014 * + 1 desc for skb_headlen/IGC_MAX_DATA_PER_TXD,
1015 * + 2 desc gap to keep tail from touching head,
1016 * + 1 desc for context descriptor,
1017 * otherwise try next time
1019 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
1020 count += TXD_USE_COUNT(skb_frag_size(
1021 &skb_shinfo(skb)->frags[f]));
1023 if (igc_maybe_stop_tx(tx_ring, count + 3)) {
1024 /* this is a hard error */
1025 return NETDEV_TX_BUSY;
1028 /* record the location of the first descriptor for this packet */
1029 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
1031 first->bytecount = skb->len;
1032 first->gso_segs = 1;
1034 /* record initial flags and protocol */
1035 first->tx_flags = tx_flags;
1036 first->protocol = protocol;
1038 igc_tx_csum(tx_ring, first);
1040 igc_tx_map(tx_ring, first, hdr_len);
1042 return NETDEV_TX_OK;
1045 static inline struct igc_ring *igc_tx_queue_mapping(struct igc_adapter *adapter,
1046 struct sk_buff *skb)
1048 unsigned int r_idx = skb->queue_mapping;
1050 if (r_idx >= adapter->num_tx_queues)
1051 r_idx = r_idx % adapter->num_tx_queues;
1053 return adapter->tx_ring[r_idx];
1056 static netdev_tx_t igc_xmit_frame(struct sk_buff *skb,
1057 struct net_device *netdev)
1059 struct igc_adapter *adapter = netdev_priv(netdev);
1061 /* The minimum packet size with TCTL.PSP set is 17 so pad the skb
1062 * in order to meet this minimum size requirement.
1064 if (skb->len < 17) {
1065 if (skb_padto(skb, 17))
1066 return NETDEV_TX_OK;
1070 return igc_xmit_frame_ring(skb, igc_tx_queue_mapping(adapter, skb));
1073 static inline void igc_rx_hash(struct igc_ring *ring,
1074 union igc_adv_rx_desc *rx_desc,
1075 struct sk_buff *skb)
1077 if (ring->netdev->features & NETIF_F_RXHASH)
1079 le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),
1084 * igc_process_skb_fields - Populate skb header fields from Rx descriptor
1085 * @rx_ring: rx descriptor ring packet is being transacted on
1086 * @rx_desc: pointer to the EOP Rx descriptor
1087 * @skb: pointer to current skb being populated
1089 * This function checks the ring, descriptor, and packet information in
1090 * order to populate the hash, checksum, VLAN, timestamp, protocol, and
1091 * other fields within the skb.
1093 static void igc_process_skb_fields(struct igc_ring *rx_ring,
1094 union igc_adv_rx_desc *rx_desc,
1095 struct sk_buff *skb)
1097 igc_rx_hash(rx_ring, rx_desc, skb);
1099 skb_record_rx_queue(skb, rx_ring->queue_index);
1101 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
1104 static struct igc_rx_buffer *igc_get_rx_buffer(struct igc_ring *rx_ring,
1105 const unsigned int size)
1107 struct igc_rx_buffer *rx_buffer;
1109 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
1110 prefetchw(rx_buffer->page);
1112 /* we are reusing so sync this buffer for CPU use */
1113 dma_sync_single_range_for_cpu(rx_ring->dev,
1115 rx_buffer->page_offset,
1119 rx_buffer->pagecnt_bias--;
1125 * igc_add_rx_frag - Add contents of Rx buffer to sk_buff
1126 * @rx_ring: rx descriptor ring to transact packets on
1127 * @rx_buffer: buffer containing page to add
1128 * @skb: sk_buff to place the data into
1129 * @size: size of buffer to be added
1131 * This function will add the data contained in rx_buffer->page to the skb.
1133 static void igc_add_rx_frag(struct igc_ring *rx_ring,
1134 struct igc_rx_buffer *rx_buffer,
1135 struct sk_buff *skb,
1138 #if (PAGE_SIZE < 8192)
1139 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2;
1141 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
1142 rx_buffer->page_offset, size, truesize);
1143 rx_buffer->page_offset ^= truesize;
1145 unsigned int truesize = ring_uses_build_skb(rx_ring) ?
1146 SKB_DATA_ALIGN(IGC_SKB_PAD + size) :
1147 SKB_DATA_ALIGN(size);
1148 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
1149 rx_buffer->page_offset, size, truesize);
1150 rx_buffer->page_offset += truesize;
1154 static struct sk_buff *igc_build_skb(struct igc_ring *rx_ring,
1155 struct igc_rx_buffer *rx_buffer,
1156 union igc_adv_rx_desc *rx_desc,
1159 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset;
1160 #if (PAGE_SIZE < 8192)
1161 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2;
1163 unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
1164 SKB_DATA_ALIGN(IGC_SKB_PAD + size);
1166 struct sk_buff *skb;
1168 /* prefetch first cache line of first page */
1170 #if L1_CACHE_BYTES < 128
1171 prefetch(va + L1_CACHE_BYTES);
1174 /* build an skb around the page buffer */
1175 skb = build_skb(va - IGC_SKB_PAD, truesize);
1179 /* update pointers within the skb to store the data */
1180 skb_reserve(skb, IGC_SKB_PAD);
1181 __skb_put(skb, size);
1183 /* update buffer offset */
1184 #if (PAGE_SIZE < 8192)
1185 rx_buffer->page_offset ^= truesize;
1187 rx_buffer->page_offset += truesize;
1193 static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring,
1194 struct igc_rx_buffer *rx_buffer,
1195 union igc_adv_rx_desc *rx_desc,
1198 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset;
1199 #if (PAGE_SIZE < 8192)
1200 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2;
1202 unsigned int truesize = SKB_DATA_ALIGN(size);
1204 unsigned int headlen;
1205 struct sk_buff *skb;
1207 /* prefetch first cache line of first page */
1209 #if L1_CACHE_BYTES < 128
1210 prefetch(va + L1_CACHE_BYTES);
1213 /* allocate a skb to store the frags */
1214 skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGC_RX_HDR_LEN);
1218 /* Determine available headroom for copy */
1220 if (headlen > IGC_RX_HDR_LEN)
1221 headlen = eth_get_headlen(skb->dev, va, IGC_RX_HDR_LEN);
1223 /* align pull length to size of long to optimize memcpy performance */
1224 memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long)));
1226 /* update all of the pointers */
1229 skb_add_rx_frag(skb, 0, rx_buffer->page,
1230 (va + headlen) - page_address(rx_buffer->page),
1232 #if (PAGE_SIZE < 8192)
1233 rx_buffer->page_offset ^= truesize;
1235 rx_buffer->page_offset += truesize;
1238 rx_buffer->pagecnt_bias++;
1245 * igc_reuse_rx_page - page flip buffer and store it back on the ring
1246 * @rx_ring: rx descriptor ring to store buffers on
1247 * @old_buff: donor buffer to have page reused
1249 * Synchronizes page for reuse by the adapter
1251 static void igc_reuse_rx_page(struct igc_ring *rx_ring,
1252 struct igc_rx_buffer *old_buff)
1254 u16 nta = rx_ring->next_to_alloc;
1255 struct igc_rx_buffer *new_buff;
1257 new_buff = &rx_ring->rx_buffer_info[nta];
1259 /* update, and store next to alloc */
1261 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
1263 /* Transfer page from old buffer to new buffer.
1264 * Move each member individually to avoid possible store
1265 * forwarding stalls.
1267 new_buff->dma = old_buff->dma;
1268 new_buff->page = old_buff->page;
1269 new_buff->page_offset = old_buff->page_offset;
1270 new_buff->pagecnt_bias = old_buff->pagecnt_bias;
1273 static inline bool igc_page_is_reserved(struct page *page)
1275 return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page);
1278 static bool igc_can_reuse_rx_page(struct igc_rx_buffer *rx_buffer)
1280 unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
1281 struct page *page = rx_buffer->page;
1283 /* avoid re-using remote pages */
1284 if (unlikely(igc_page_is_reserved(page)))
1287 #if (PAGE_SIZE < 8192)
1288 /* if we are only owner of page we can reuse it */
1289 if (unlikely((page_ref_count(page) - pagecnt_bias) > 1))
1292 #define IGC_LAST_OFFSET \
1293 (SKB_WITH_OVERHEAD(PAGE_SIZE) - IGC_RXBUFFER_2048)
1295 if (rx_buffer->page_offset > IGC_LAST_OFFSET)
1299 /* If we have drained the page fragment pool we need to update
1300 * the pagecnt_bias and page count so that we fully restock the
1301 * number of references the driver holds.
1303 if (unlikely(!pagecnt_bias)) {
1304 page_ref_add(page, USHRT_MAX);
1305 rx_buffer->pagecnt_bias = USHRT_MAX;
1312 * igc_is_non_eop - process handling of non-EOP buffers
1313 * @rx_ring: Rx ring being processed
1314 * @rx_desc: Rx descriptor for current buffer
1315 * @skb: current socket buffer containing buffer in progress
1317 * This function updates next to clean. If the buffer is an EOP buffer
1318 * this function exits returning false, otherwise it will place the
1319 * sk_buff in the next buffer to be chained and return true indicating
1320 * that this is in fact a non-EOP buffer.
1322 static bool igc_is_non_eop(struct igc_ring *rx_ring,
1323 union igc_adv_rx_desc *rx_desc)
1325 u32 ntc = rx_ring->next_to_clean + 1;
1327 /* fetch, update, and store next to clean */
1328 ntc = (ntc < rx_ring->count) ? ntc : 0;
1329 rx_ring->next_to_clean = ntc;
1331 prefetch(IGC_RX_DESC(rx_ring, ntc));
1333 if (likely(igc_test_staterr(rx_desc, IGC_RXD_STAT_EOP)))
1340 * igc_cleanup_headers - Correct corrupted or empty headers
1341 * @rx_ring: rx descriptor ring packet is being transacted on
1342 * @rx_desc: pointer to the EOP Rx descriptor
1343 * @skb: pointer to current skb being fixed
1345 * Address the case where we are pulling data in on pages only
1346 * and as such no data is present in the skb header.
1348 * In addition if skb is not at least 60 bytes we need to pad it so that
1349 * it is large enough to qualify as a valid Ethernet frame.
1351 * Returns true if an error was encountered and skb was freed.
1353 static bool igc_cleanup_headers(struct igc_ring *rx_ring,
1354 union igc_adv_rx_desc *rx_desc,
1355 struct sk_buff *skb)
1357 if (unlikely((igc_test_staterr(rx_desc,
1358 IGC_RXDEXT_ERR_FRAME_ERR_MASK)))) {
1359 struct net_device *netdev = rx_ring->netdev;
1361 if (!(netdev->features & NETIF_F_RXALL)) {
1362 dev_kfree_skb_any(skb);
1367 /* if eth_skb_pad returns an error the skb was freed */
1368 if (eth_skb_pad(skb))
1374 static void igc_put_rx_buffer(struct igc_ring *rx_ring,
1375 struct igc_rx_buffer *rx_buffer)
1377 if (igc_can_reuse_rx_page(rx_buffer)) {
1378 /* hand second half of page back to the ring */
1379 igc_reuse_rx_page(rx_ring, rx_buffer);
1381 /* We are not reusing the buffer so unmap it and free
1382 * any references we are holding to it
1384 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
1385 igc_rx_pg_size(rx_ring), DMA_FROM_DEVICE,
1387 __page_frag_cache_drain(rx_buffer->page,
1388 rx_buffer->pagecnt_bias);
1391 /* clear contents of rx_buffer */
1392 rx_buffer->page = NULL;
1396 * igc_alloc_rx_buffers - Replace used receive buffers; packet split
1397 * @adapter: address of board private structure
1399 static void igc_alloc_rx_buffers(struct igc_ring *rx_ring, u16 cleaned_count)
1401 union igc_adv_rx_desc *rx_desc;
1402 u16 i = rx_ring->next_to_use;
1403 struct igc_rx_buffer *bi;
1410 rx_desc = IGC_RX_DESC(rx_ring, i);
1411 bi = &rx_ring->rx_buffer_info[i];
1412 i -= rx_ring->count;
1414 bufsz = igc_rx_bufsz(rx_ring);
1417 if (!igc_alloc_mapped_page(rx_ring, bi))
1420 /* sync the buffer for use by the device */
1421 dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
1422 bi->page_offset, bufsz,
1425 /* Refresh the desc even if buffer_addrs didn't change
1426 * because each write-back erases this info.
1428 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
1434 rx_desc = IGC_RX_DESC(rx_ring, 0);
1435 bi = rx_ring->rx_buffer_info;
1436 i -= rx_ring->count;
1439 /* clear the length for the next_to_use descriptor */
1440 rx_desc->wb.upper.length = 0;
1443 } while (cleaned_count);
1445 i += rx_ring->count;
1447 if (rx_ring->next_to_use != i) {
1448 /* record the next descriptor to use */
1449 rx_ring->next_to_use = i;
1451 /* update next to alloc since we have filled the ring */
1452 rx_ring->next_to_alloc = i;
1454 /* Force memory writes to complete before letting h/w
1455 * know there are new descriptors to fetch. (Only
1456 * applicable for weak-ordered memory model archs,
1460 writel(i, rx_ring->tail);
1464 static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget)
1466 unsigned int total_bytes = 0, total_packets = 0;
1467 struct igc_ring *rx_ring = q_vector->rx.ring;
1468 struct sk_buff *skb = rx_ring->skb;
1469 u16 cleaned_count = igc_desc_unused(rx_ring);
1471 while (likely(total_packets < budget)) {
1472 union igc_adv_rx_desc *rx_desc;
1473 struct igc_rx_buffer *rx_buffer;
1476 /* return some buffers to hardware, one at a time is too slow */
1477 if (cleaned_count >= IGC_RX_BUFFER_WRITE) {
1478 igc_alloc_rx_buffers(rx_ring, cleaned_count);
1482 rx_desc = IGC_RX_DESC(rx_ring, rx_ring->next_to_clean);
1483 size = le16_to_cpu(rx_desc->wb.upper.length);
1487 /* This memory barrier is needed to keep us from reading
1488 * any other fields out of the rx_desc until we know the
1489 * descriptor has been written back
1493 rx_buffer = igc_get_rx_buffer(rx_ring, size);
1495 /* retrieve a buffer from the ring */
1497 igc_add_rx_frag(rx_ring, rx_buffer, skb, size);
1498 else if (ring_uses_build_skb(rx_ring))
1499 skb = igc_build_skb(rx_ring, rx_buffer, rx_desc, size);
1501 skb = igc_construct_skb(rx_ring, rx_buffer,
1504 /* exit if we failed to retrieve a buffer */
1506 rx_ring->rx_stats.alloc_failed++;
1507 rx_buffer->pagecnt_bias++;
1511 igc_put_rx_buffer(rx_ring, rx_buffer);
1514 /* fetch next buffer in frame if non-eop */
1515 if (igc_is_non_eop(rx_ring, rx_desc))
1518 /* verify the packet layout is correct */
1519 if (igc_cleanup_headers(rx_ring, rx_desc, skb)) {
1524 /* probably a little skewed due to removing CRC */
1525 total_bytes += skb->len;
1527 /* populate checksum, timestamp, VLAN, and protocol */
1528 igc_process_skb_fields(rx_ring, rx_desc, skb);
1530 napi_gro_receive(&q_vector->napi, skb);
1532 /* reset skb pointer */
1535 /* update budget accounting */
1539 /* place incomplete frames back on ring for completion */
1542 u64_stats_update_begin(&rx_ring->rx_syncp);
1543 rx_ring->rx_stats.packets += total_packets;
1544 rx_ring->rx_stats.bytes += total_bytes;
1545 u64_stats_update_end(&rx_ring->rx_syncp);
1546 q_vector->rx.total_packets += total_packets;
1547 q_vector->rx.total_bytes += total_bytes;
1550 igc_alloc_rx_buffers(rx_ring, cleaned_count);
1552 return total_packets;
1555 static inline unsigned int igc_rx_offset(struct igc_ring *rx_ring)
1557 return ring_uses_build_skb(rx_ring) ? IGC_SKB_PAD : 0;
1560 static bool igc_alloc_mapped_page(struct igc_ring *rx_ring,
1561 struct igc_rx_buffer *bi)
1563 struct page *page = bi->page;
1566 /* since we are recycling buffers we should seldom need to alloc */
1570 /* alloc new page for storage */
1571 page = dev_alloc_pages(igc_rx_pg_order(rx_ring));
1572 if (unlikely(!page)) {
1573 rx_ring->rx_stats.alloc_failed++;
1577 /* map page for use */
1578 dma = dma_map_page_attrs(rx_ring->dev, page, 0,
1579 igc_rx_pg_size(rx_ring),
1583 /* if mapping failed free memory back to system since
1584 * there isn't much point in holding memory we can't use
1586 if (dma_mapping_error(rx_ring->dev, dma)) {
1589 rx_ring->rx_stats.alloc_failed++;
1595 bi->page_offset = igc_rx_offset(rx_ring);
1596 bi->pagecnt_bias = 1;
1602 * igc_clean_tx_irq - Reclaim resources after transmit completes
1603 * @q_vector: pointer to q_vector containing needed info
1604 * @napi_budget: Used to determine if we are in netpoll
1606 * returns true if ring is completely cleaned
1608 static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget)
1610 struct igc_adapter *adapter = q_vector->adapter;
1611 unsigned int total_bytes = 0, total_packets = 0;
1612 unsigned int budget = q_vector->tx.work_limit;
1613 struct igc_ring *tx_ring = q_vector->tx.ring;
1614 unsigned int i = tx_ring->next_to_clean;
1615 struct igc_tx_buffer *tx_buffer;
1616 union igc_adv_tx_desc *tx_desc;
1618 if (test_bit(__IGC_DOWN, &adapter->state))
1621 tx_buffer = &tx_ring->tx_buffer_info[i];
1622 tx_desc = IGC_TX_DESC(tx_ring, i);
1623 i -= tx_ring->count;
1626 union igc_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
1628 /* if next_to_watch is not set then there is no work pending */
1632 /* prevent any other reads prior to eop_desc */
1635 /* if DD is not set pending work has not been completed */
1636 if (!(eop_desc->wb.status & cpu_to_le32(IGC_TXD_STAT_DD)))
1639 /* clear next_to_watch to prevent false hangs */
1640 tx_buffer->next_to_watch = NULL;
1642 /* update the statistics for this packet */
1643 total_bytes += tx_buffer->bytecount;
1644 total_packets += tx_buffer->gso_segs;
1647 napi_consume_skb(tx_buffer->skb, napi_budget);
1649 /* unmap skb header data */
1650 dma_unmap_single(tx_ring->dev,
1651 dma_unmap_addr(tx_buffer, dma),
1652 dma_unmap_len(tx_buffer, len),
1655 /* clear tx_buffer data */
1656 dma_unmap_len_set(tx_buffer, len, 0);
1658 /* clear last DMA location and unmap remaining buffers */
1659 while (tx_desc != eop_desc) {
1664 i -= tx_ring->count;
1665 tx_buffer = tx_ring->tx_buffer_info;
1666 tx_desc = IGC_TX_DESC(tx_ring, 0);
1669 /* unmap any remaining paged data */
1670 if (dma_unmap_len(tx_buffer, len)) {
1671 dma_unmap_page(tx_ring->dev,
1672 dma_unmap_addr(tx_buffer, dma),
1673 dma_unmap_len(tx_buffer, len),
1675 dma_unmap_len_set(tx_buffer, len, 0);
1679 /* move us one more past the eop_desc for start of next pkt */
1684 i -= tx_ring->count;
1685 tx_buffer = tx_ring->tx_buffer_info;
1686 tx_desc = IGC_TX_DESC(tx_ring, 0);
1689 /* issue prefetch for next Tx descriptor */
1692 /* update budget accounting */
1694 } while (likely(budget));
1696 netdev_tx_completed_queue(txring_txq(tx_ring),
1697 total_packets, total_bytes);
1699 i += tx_ring->count;
1700 tx_ring->next_to_clean = i;
1701 u64_stats_update_begin(&tx_ring->tx_syncp);
1702 tx_ring->tx_stats.bytes += total_bytes;
1703 tx_ring->tx_stats.packets += total_packets;
1704 u64_stats_update_end(&tx_ring->tx_syncp);
1705 q_vector->tx.total_bytes += total_bytes;
1706 q_vector->tx.total_packets += total_packets;
1708 if (test_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
1709 struct igc_hw *hw = &adapter->hw;
1711 /* Detect a transmit hang in hardware, this serializes the
1712 * check with the clearing of time_stamp and movement of i
1714 clear_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
1715 if (tx_buffer->next_to_watch &&
1716 time_after(jiffies, tx_buffer->time_stamp +
1717 (adapter->tx_timeout_factor * HZ)) &&
1718 !(rd32(IGC_STATUS) & IGC_STATUS_TXOFF)) {
1719 /* detected Tx unit hang */
1720 dev_err(tx_ring->dev,
1721 "Detected Tx Unit Hang\n"
1725 " next_to_use <%x>\n"
1726 " next_to_clean <%x>\n"
1727 "buffer_info[next_to_clean]\n"
1728 " time_stamp <%lx>\n"
1729 " next_to_watch <%p>\n"
1731 " desc.status <%x>\n",
1732 tx_ring->queue_index,
1733 rd32(IGC_TDH(tx_ring->reg_idx)),
1734 readl(tx_ring->tail),
1735 tx_ring->next_to_use,
1736 tx_ring->next_to_clean,
1737 tx_buffer->time_stamp,
1738 tx_buffer->next_to_watch,
1740 tx_buffer->next_to_watch->wb.status);
1741 netif_stop_subqueue(tx_ring->netdev,
1742 tx_ring->queue_index);
1744 /* we are about to reset, no point in enabling stuff */
1749 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
1750 if (unlikely(total_packets &&
1751 netif_carrier_ok(tx_ring->netdev) &&
1752 igc_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) {
1753 /* Make sure that anybody stopping the queue after this
1754 * sees the new next_to_clean.
1757 if (__netif_subqueue_stopped(tx_ring->netdev,
1758 tx_ring->queue_index) &&
1759 !(test_bit(__IGC_DOWN, &adapter->state))) {
1760 netif_wake_subqueue(tx_ring->netdev,
1761 tx_ring->queue_index);
1763 u64_stats_update_begin(&tx_ring->tx_syncp);
1764 tx_ring->tx_stats.restart_queue++;
1765 u64_stats_update_end(&tx_ring->tx_syncp);
1773 * igc_up - Open the interface and prepare it to handle traffic
1774 * @adapter: board private structure
1776 void igc_up(struct igc_adapter *adapter)
1778 struct igc_hw *hw = &adapter->hw;
1781 /* hardware has been reset, we need to reload some things */
1782 igc_configure(adapter);
1784 clear_bit(__IGC_DOWN, &adapter->state);
1786 for (i = 0; i < adapter->num_q_vectors; i++)
1787 napi_enable(&adapter->q_vector[i]->napi);
1789 if (adapter->msix_entries)
1790 igc_configure_msix(adapter);
1792 igc_assign_vector(adapter->q_vector[0], 0);
1794 /* Clear any pending interrupts. */
1796 igc_irq_enable(adapter);
1798 netif_tx_start_all_queues(adapter->netdev);
1800 /* start the watchdog. */
1801 hw->mac.get_link_status = 1;
1802 schedule_work(&adapter->watchdog_task);
1806 * igc_update_stats - Update the board statistics counters
1807 * @adapter: board private structure
1809 void igc_update_stats(struct igc_adapter *adapter)
1811 struct rtnl_link_stats64 *net_stats = &adapter->stats64;
1812 struct pci_dev *pdev = adapter->pdev;
1813 struct igc_hw *hw = &adapter->hw;
1814 u64 _bytes, _packets;
1820 /* Prevent stats update while adapter is being reset, or if the pci
1821 * connection is down.
1823 if (adapter->link_speed == 0)
1825 if (pci_channel_offline(pdev))
1832 for (i = 0; i < adapter->num_rx_queues; i++) {
1833 struct igc_ring *ring = adapter->rx_ring[i];
1834 u32 rqdpc = rd32(IGC_RQDPC(i));
1836 if (hw->mac.type >= igc_i225)
1837 wr32(IGC_RQDPC(i), 0);
1840 ring->rx_stats.drops += rqdpc;
1841 net_stats->rx_fifo_errors += rqdpc;
1845 start = u64_stats_fetch_begin_irq(&ring->rx_syncp);
1846 _bytes = ring->rx_stats.bytes;
1847 _packets = ring->rx_stats.packets;
1848 } while (u64_stats_fetch_retry_irq(&ring->rx_syncp, start));
1850 packets += _packets;
1853 net_stats->rx_bytes = bytes;
1854 net_stats->rx_packets = packets;
1858 for (i = 0; i < adapter->num_tx_queues; i++) {
1859 struct igc_ring *ring = adapter->tx_ring[i];
1862 start = u64_stats_fetch_begin_irq(&ring->tx_syncp);
1863 _bytes = ring->tx_stats.bytes;
1864 _packets = ring->tx_stats.packets;
1865 } while (u64_stats_fetch_retry_irq(&ring->tx_syncp, start));
1867 packets += _packets;
1869 net_stats->tx_bytes = bytes;
1870 net_stats->tx_packets = packets;
1873 /* read stats registers */
1874 adapter->stats.crcerrs += rd32(IGC_CRCERRS);
1875 adapter->stats.gprc += rd32(IGC_GPRC);
1876 adapter->stats.gorc += rd32(IGC_GORCL);
1877 rd32(IGC_GORCH); /* clear GORCL */
1878 adapter->stats.bprc += rd32(IGC_BPRC);
1879 adapter->stats.mprc += rd32(IGC_MPRC);
1880 adapter->stats.roc += rd32(IGC_ROC);
1882 adapter->stats.prc64 += rd32(IGC_PRC64);
1883 adapter->stats.prc127 += rd32(IGC_PRC127);
1884 adapter->stats.prc255 += rd32(IGC_PRC255);
1885 adapter->stats.prc511 += rd32(IGC_PRC511);
1886 adapter->stats.prc1023 += rd32(IGC_PRC1023);
1887 adapter->stats.prc1522 += rd32(IGC_PRC1522);
1888 adapter->stats.symerrs += rd32(IGC_SYMERRS);
1889 adapter->stats.sec += rd32(IGC_SEC);
1891 mpc = rd32(IGC_MPC);
1892 adapter->stats.mpc += mpc;
1893 net_stats->rx_fifo_errors += mpc;
1894 adapter->stats.scc += rd32(IGC_SCC);
1895 adapter->stats.ecol += rd32(IGC_ECOL);
1896 adapter->stats.mcc += rd32(IGC_MCC);
1897 adapter->stats.latecol += rd32(IGC_LATECOL);
1898 adapter->stats.dc += rd32(IGC_DC);
1899 adapter->stats.rlec += rd32(IGC_RLEC);
1900 adapter->stats.xonrxc += rd32(IGC_XONRXC);
1901 adapter->stats.xontxc += rd32(IGC_XONTXC);
1902 adapter->stats.xoffrxc += rd32(IGC_XOFFRXC);
1903 adapter->stats.xofftxc += rd32(IGC_XOFFTXC);
1904 adapter->stats.fcruc += rd32(IGC_FCRUC);
1905 adapter->stats.gptc += rd32(IGC_GPTC);
1906 adapter->stats.gotc += rd32(IGC_GOTCL);
1907 rd32(IGC_GOTCH); /* clear GOTCL */
1908 adapter->stats.rnbc += rd32(IGC_RNBC);
1909 adapter->stats.ruc += rd32(IGC_RUC);
1910 adapter->stats.rfc += rd32(IGC_RFC);
1911 adapter->stats.rjc += rd32(IGC_RJC);
1912 adapter->stats.tor += rd32(IGC_TORH);
1913 adapter->stats.tot += rd32(IGC_TOTH);
1914 adapter->stats.tpr += rd32(IGC_TPR);
1916 adapter->stats.ptc64 += rd32(IGC_PTC64);
1917 adapter->stats.ptc127 += rd32(IGC_PTC127);
1918 adapter->stats.ptc255 += rd32(IGC_PTC255);
1919 adapter->stats.ptc511 += rd32(IGC_PTC511);
1920 adapter->stats.ptc1023 += rd32(IGC_PTC1023);
1921 adapter->stats.ptc1522 += rd32(IGC_PTC1522);
1923 adapter->stats.mptc += rd32(IGC_MPTC);
1924 adapter->stats.bptc += rd32(IGC_BPTC);
1926 adapter->stats.tpt += rd32(IGC_TPT);
1927 adapter->stats.colc += rd32(IGC_COLC);
1929 adapter->stats.algnerrc += rd32(IGC_ALGNERRC);
1931 adapter->stats.tsctc += rd32(IGC_TSCTC);
1932 adapter->stats.tsctfc += rd32(IGC_TSCTFC);
1934 adapter->stats.iac += rd32(IGC_IAC);
1935 adapter->stats.icrxoc += rd32(IGC_ICRXOC);
1936 adapter->stats.icrxptc += rd32(IGC_ICRXPTC);
1937 adapter->stats.icrxatc += rd32(IGC_ICRXATC);
1938 adapter->stats.ictxptc += rd32(IGC_ICTXPTC);
1939 adapter->stats.ictxatc += rd32(IGC_ICTXATC);
1940 adapter->stats.ictxqec += rd32(IGC_ICTXQEC);
1941 adapter->stats.ictxqmtc += rd32(IGC_ICTXQMTC);
1942 adapter->stats.icrxdmtc += rd32(IGC_ICRXDMTC);
1944 /* Fill out the OS statistics structure */
1945 net_stats->multicast = adapter->stats.mprc;
1946 net_stats->collisions = adapter->stats.colc;
1950 /* RLEC on some newer hardware can be incorrect so build
1951 * our own version based on RUC and ROC
1953 net_stats->rx_errors = adapter->stats.rxerrc +
1954 adapter->stats.crcerrs + adapter->stats.algnerrc +
1955 adapter->stats.ruc + adapter->stats.roc +
1956 adapter->stats.cexterr;
1957 net_stats->rx_length_errors = adapter->stats.ruc +
1959 net_stats->rx_crc_errors = adapter->stats.crcerrs;
1960 net_stats->rx_frame_errors = adapter->stats.algnerrc;
1961 net_stats->rx_missed_errors = adapter->stats.mpc;
1964 net_stats->tx_errors = adapter->stats.ecol +
1965 adapter->stats.latecol;
1966 net_stats->tx_aborted_errors = adapter->stats.ecol;
1967 net_stats->tx_window_errors = adapter->stats.latecol;
1968 net_stats->tx_carrier_errors = adapter->stats.tncrs;
1970 /* Tx Dropped needs to be maintained elsewhere */
1972 /* Management Stats */
1973 adapter->stats.mgptc += rd32(IGC_MGTPTC);
1974 adapter->stats.mgprc += rd32(IGC_MGTPRC);
1975 adapter->stats.mgpdc += rd32(IGC_MGTPDC);
1978 static void igc_nfc_filter_exit(struct igc_adapter *adapter)
1980 struct igc_nfc_filter *rule;
1982 spin_lock(&adapter->nfc_lock);
1984 hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node)
1985 igc_erase_filter(adapter, rule);
1987 hlist_for_each_entry(rule, &adapter->cls_flower_list, nfc_node)
1988 igc_erase_filter(adapter, rule);
1990 spin_unlock(&adapter->nfc_lock);
1993 static void igc_nfc_filter_restore(struct igc_adapter *adapter)
1995 struct igc_nfc_filter *rule;
1997 spin_lock(&adapter->nfc_lock);
1999 hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node)
2000 igc_add_filter(adapter, rule);
2002 spin_unlock(&adapter->nfc_lock);
2006 * igc_down - Close the interface
2007 * @adapter: board private structure
2009 void igc_down(struct igc_adapter *adapter)
2011 struct net_device *netdev = adapter->netdev;
2012 struct igc_hw *hw = &adapter->hw;
2016 set_bit(__IGC_DOWN, &adapter->state);
2018 /* disable receives in the hardware */
2019 rctl = rd32(IGC_RCTL);
2020 wr32(IGC_RCTL, rctl & ~IGC_RCTL_EN);
2021 /* flush and sleep below */
2023 igc_nfc_filter_exit(adapter);
2025 /* set trans_start so we don't get spurious watchdogs during reset */
2026 netif_trans_update(netdev);
2028 netif_carrier_off(netdev);
2029 netif_tx_stop_all_queues(netdev);
2031 /* disable transmits in the hardware */
2032 tctl = rd32(IGC_TCTL);
2033 tctl &= ~IGC_TCTL_EN;
2034 wr32(IGC_TCTL, tctl);
2035 /* flush both disables and wait for them to finish */
2037 usleep_range(10000, 20000);
2039 igc_irq_disable(adapter);
2041 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
2043 for (i = 0; i < adapter->num_q_vectors; i++) {
2044 if (adapter->q_vector[i]) {
2045 napi_synchronize(&adapter->q_vector[i]->napi);
2046 napi_disable(&adapter->q_vector[i]->napi);
2050 del_timer_sync(&adapter->watchdog_timer);
2051 del_timer_sync(&adapter->phy_info_timer);
2053 /* record the stats before reset*/
2054 spin_lock(&adapter->stats64_lock);
2055 igc_update_stats(adapter);
2056 spin_unlock(&adapter->stats64_lock);
2058 adapter->link_speed = 0;
2059 adapter->link_duplex = 0;
2061 if (!pci_channel_offline(adapter->pdev))
2064 /* clear VLAN promisc flag so VFTA will be updated if necessary */
2065 adapter->flags &= ~IGC_FLAG_VLAN_PROMISC;
2067 igc_clean_all_tx_rings(adapter);
2068 igc_clean_all_rx_rings(adapter);
2071 void igc_reinit_locked(struct igc_adapter *adapter)
2073 WARN_ON(in_interrupt());
2074 while (test_and_set_bit(__IGC_RESETTING, &adapter->state))
2075 usleep_range(1000, 2000);
2078 clear_bit(__IGC_RESETTING, &adapter->state);
2081 static void igc_reset_task(struct work_struct *work)
2083 struct igc_adapter *adapter;
2085 adapter = container_of(work, struct igc_adapter, reset_task);
2087 netdev_err(adapter->netdev, "Reset adapter\n");
2088 igc_reinit_locked(adapter);
2092 * igc_change_mtu - Change the Maximum Transfer Unit
2093 * @netdev: network interface device structure
2094 * @new_mtu: new value for maximum frame size
2096 * Returns 0 on success, negative on failure
2098 static int igc_change_mtu(struct net_device *netdev, int new_mtu)
2100 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
2101 struct igc_adapter *adapter = netdev_priv(netdev);
2102 struct pci_dev *pdev = adapter->pdev;
2104 /* adjust max frame to be at least the size of a standard frame */
2105 if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN))
2106 max_frame = ETH_FRAME_LEN + ETH_FCS_LEN;
2108 while (test_and_set_bit(__IGC_RESETTING, &adapter->state))
2109 usleep_range(1000, 2000);
2111 /* igc_down has a dependency on max_frame_size */
2112 adapter->max_frame_size = max_frame;
2114 if (netif_running(netdev))
2117 dev_info(&pdev->dev, "changing MTU from %d to %d\n",
2118 netdev->mtu, new_mtu);
2119 netdev->mtu = new_mtu;
2121 if (netif_running(netdev))
2126 clear_bit(__IGC_RESETTING, &adapter->state);
2132 * igc_get_stats - Get System Network Statistics
2133 * @netdev: network interface device structure
2135 * Returns the address of the device statistics structure.
2136 * The statistics are updated here and also from the timer callback.
2138 static struct net_device_stats *igc_get_stats(struct net_device *netdev)
2140 struct igc_adapter *adapter = netdev_priv(netdev);
2142 if (!test_bit(__IGC_RESETTING, &adapter->state))
2143 igc_update_stats(adapter);
2145 /* only return the current stats */
2146 return &netdev->stats;
2149 static netdev_features_t igc_fix_features(struct net_device *netdev,
2150 netdev_features_t features)
2152 /* Since there is no support for separate Rx/Tx vlan accel
2153 * enable/disable make sure Tx flag is always in same state as Rx.
2155 if (features & NETIF_F_HW_VLAN_CTAG_RX)
2156 features |= NETIF_F_HW_VLAN_CTAG_TX;
2158 features &= ~NETIF_F_HW_VLAN_CTAG_TX;
2163 static int igc_set_features(struct net_device *netdev,
2164 netdev_features_t features)
2166 netdev_features_t changed = netdev->features ^ features;
2167 struct igc_adapter *adapter = netdev_priv(netdev);
2169 /* Add VLAN support */
2170 if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE)))
2173 if (!(features & NETIF_F_NTUPLE)) {
2174 struct hlist_node *node2;
2175 struct igc_nfc_filter *rule;
2177 spin_lock(&adapter->nfc_lock);
2178 hlist_for_each_entry_safe(rule, node2,
2179 &adapter->nfc_filter_list, nfc_node) {
2180 igc_erase_filter(adapter, rule);
2181 hlist_del(&rule->nfc_node);
2184 spin_unlock(&adapter->nfc_lock);
2185 adapter->nfc_filter_count = 0;
2188 netdev->features = features;
2190 if (netif_running(netdev))
2191 igc_reinit_locked(adapter);
2198 static netdev_features_t
2199 igc_features_check(struct sk_buff *skb, struct net_device *dev,
2200 netdev_features_t features)
2202 unsigned int network_hdr_len, mac_hdr_len;
2204 /* Make certain the headers can be described by a context descriptor */
2205 mac_hdr_len = skb_network_header(skb) - skb->data;
2206 if (unlikely(mac_hdr_len > IGC_MAX_MAC_HDR_LEN))
2207 return features & ~(NETIF_F_HW_CSUM |
2209 NETIF_F_HW_VLAN_CTAG_TX |
2213 network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb);
2214 if (unlikely(network_hdr_len > IGC_MAX_NETWORK_HDR_LEN))
2215 return features & ~(NETIF_F_HW_CSUM |
2220 /* We can only support IPv4 TSO in tunnels if we can mangle the
2221 * inner IP ID field, so strip TSO if MANGLEID is not supported.
2223 if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID))
2224 features &= ~NETIF_F_TSO;
2230 * igc_configure - configure the hardware for RX and TX
2231 * @adapter: private board structure
2233 static void igc_configure(struct igc_adapter *adapter)
2235 struct net_device *netdev = adapter->netdev;
2238 igc_get_hw_control(adapter);
2239 igc_set_rx_mode(netdev);
2241 igc_setup_tctl(adapter);
2242 igc_setup_mrqc(adapter);
2243 igc_setup_rctl(adapter);
2245 igc_nfc_filter_restore(adapter);
2246 igc_configure_tx(adapter);
2247 igc_configure_rx(adapter);
2249 igc_rx_fifo_flush_base(&adapter->hw);
2251 /* call igc_desc_unused which always leaves
2252 * at least 1 descriptor unused to make sure
2253 * next_to_use != next_to_clean
2255 for (i = 0; i < adapter->num_rx_queues; i++) {
2256 struct igc_ring *ring = adapter->rx_ring[i];
2258 igc_alloc_rx_buffers(ring, igc_desc_unused(ring));
2263 * igc_rar_set_index - Sync RAL[index] and RAH[index] registers with MAC table
2264 * @adapter: address of board private structure
2265 * @index: Index of the RAR entry which need to be synced with MAC table
2267 static void igc_rar_set_index(struct igc_adapter *adapter, u32 index)
2269 u8 *addr = adapter->mac_table[index].addr;
2270 struct igc_hw *hw = &adapter->hw;
2271 u32 rar_low, rar_high;
2273 /* HW expects these to be in network order when they are plugged
2274 * into the registers which are little endian. In order to guarantee
2275 * that ordering we need to do an leXX_to_cpup here in order to be
2276 * ready for the byteswap that occurs with writel
2278 rar_low = le32_to_cpup((__le32 *)(addr));
2279 rar_high = le16_to_cpup((__le16 *)(addr + 4));
2281 /* Indicate to hardware the Address is Valid. */
2282 if (adapter->mac_table[index].state & IGC_MAC_STATE_IN_USE) {
2283 if (is_valid_ether_addr(addr))
2284 rar_high |= IGC_RAH_AV;
2286 rar_high |= IGC_RAH_POOL_1 <<
2287 adapter->mac_table[index].queue;
2290 wr32(IGC_RAL(index), rar_low);
2292 wr32(IGC_RAH(index), rar_high);
2296 /* Set default MAC address for the PF in the first RAR entry */
2297 static void igc_set_default_mac_filter(struct igc_adapter *adapter)
2299 struct igc_mac_addr *mac_table = &adapter->mac_table[0];
2301 ether_addr_copy(mac_table->addr, adapter->hw.mac.addr);
2302 mac_table->state = IGC_MAC_STATE_DEFAULT | IGC_MAC_STATE_IN_USE;
2304 igc_rar_set_index(adapter, 0);
2307 /* If the filter to be added and an already existing filter express
2308 * the same address and address type, it should be possible to only
2309 * override the other configurations, for example the queue to steer
2312 static bool igc_mac_entry_can_be_used(const struct igc_mac_addr *entry,
2313 const u8 *addr, const u8 flags)
2315 if (!(entry->state & IGC_MAC_STATE_IN_USE))
2318 if ((entry->state & IGC_MAC_STATE_SRC_ADDR) !=
2319 (flags & IGC_MAC_STATE_SRC_ADDR))
2322 if (!ether_addr_equal(addr, entry->addr))
2328 /* Add a MAC filter for 'addr' directing matching traffic to 'queue',
2329 * 'flags' is used to indicate what kind of match is made, match is by
2330 * default for the destination address, if matching by source address
2331 * is desired the flag IGC_MAC_STATE_SRC_ADDR can be used.
2333 static int igc_add_mac_filter_flags(struct igc_adapter *adapter,
2334 const u8 *addr, const u8 queue,
2337 struct igc_hw *hw = &adapter->hw;
2338 int rar_entries = hw->mac.rar_entry_count;
2341 if (is_zero_ether_addr(addr))
2344 /* Search for the first empty entry in the MAC table.
2345 * Do not touch entries at the end of the table reserved for the VF MAC
2348 for (i = 0; i < rar_entries; i++) {
2349 if (!igc_mac_entry_can_be_used(&adapter->mac_table[i],
2353 ether_addr_copy(adapter->mac_table[i].addr, addr);
2354 adapter->mac_table[i].queue = queue;
2355 adapter->mac_table[i].state |= IGC_MAC_STATE_IN_USE | flags;
2357 igc_rar_set_index(adapter, i);
2364 int igc_add_mac_steering_filter(struct igc_adapter *adapter,
2365 const u8 *addr, u8 queue, u8 flags)
2367 return igc_add_mac_filter_flags(adapter, addr, queue,
2368 IGC_MAC_STATE_QUEUE_STEERING | flags);
2371 /* Remove a MAC filter for 'addr' directing matching traffic to
2372 * 'queue', 'flags' is used to indicate what kind of match need to be
2373 * removed, match is by default for the destination address, if
2374 * matching by source address is to be removed the flag
2375 * IGC_MAC_STATE_SRC_ADDR can be used.
2377 static int igc_del_mac_filter_flags(struct igc_adapter *adapter,
2378 const u8 *addr, const u8 queue,
2381 struct igc_hw *hw = &adapter->hw;
2382 int rar_entries = hw->mac.rar_entry_count;
2385 if (is_zero_ether_addr(addr))
2388 /* Search for matching entry in the MAC table based on given address
2389 * and queue. Do not touch entries at the end of the table reserved
2390 * for the VF MAC addresses.
2392 for (i = 0; i < rar_entries; i++) {
2393 if (!(adapter->mac_table[i].state & IGC_MAC_STATE_IN_USE))
2395 if ((adapter->mac_table[i].state & flags) != flags)
2397 if (adapter->mac_table[i].queue != queue)
2399 if (!ether_addr_equal(adapter->mac_table[i].addr, addr))
2402 /* When a filter for the default address is "deleted",
2403 * we return it to its initial configuration
2405 if (adapter->mac_table[i].state & IGC_MAC_STATE_DEFAULT) {
2406 adapter->mac_table[i].state =
2407 IGC_MAC_STATE_DEFAULT | IGC_MAC_STATE_IN_USE;
2409 adapter->mac_table[i].state = 0;
2410 adapter->mac_table[i].queue = 0;
2411 memset(adapter->mac_table[i].addr, 0, ETH_ALEN);
2414 igc_rar_set_index(adapter, i);
2421 int igc_del_mac_steering_filter(struct igc_adapter *adapter,
2422 const u8 *addr, u8 queue, u8 flags)
2424 return igc_del_mac_filter_flags(adapter, addr, queue,
2425 IGC_MAC_STATE_QUEUE_STEERING | flags);
2429 * igc_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
2430 * @netdev: network interface device structure
2432 * The set_rx_mode entry point is called whenever the unicast or multicast
2433 * address lists or the network interface flags are updated. This routine is
2434 * responsible for configuring the hardware for proper unicast, multicast,
2435 * promiscuous mode, and all-multi behavior.
2437 static void igc_set_rx_mode(struct net_device *netdev)
2442 * igc_msix_other - msix other interrupt handler
2443 * @irq: interrupt number
2444 * @data: pointer to a q_vector
2446 static irqreturn_t igc_msix_other(int irq, void *data)
2448 struct igc_adapter *adapter = data;
2449 struct igc_hw *hw = &adapter->hw;
2450 u32 icr = rd32(IGC_ICR);
2452 /* reading ICR causes bit 31 of EICR to be cleared */
2453 if (icr & IGC_ICR_DRSTA)
2454 schedule_work(&adapter->reset_task);
2456 if (icr & IGC_ICR_DOUTSYNC) {
2457 /* HW is reporting DMA is out of sync */
2458 adapter->stats.doosync++;
2461 if (icr & IGC_ICR_LSC) {
2462 hw->mac.get_link_status = 1;
2463 /* guard against interrupt when we're going down */
2464 if (!test_bit(__IGC_DOWN, &adapter->state))
2465 mod_timer(&adapter->watchdog_timer, jiffies + 1);
2468 wr32(IGC_EIMS, adapter->eims_other);
2474 * igc_write_ivar - configure ivar for given MSI-X vector
2475 * @hw: pointer to the HW structure
2476 * @msix_vector: vector number we are allocating to a given ring
2477 * @index: row index of IVAR register to write within IVAR table
2478 * @offset: column offset of in IVAR, should be multiple of 8
2480 * The IVAR table consists of 2 columns,
2481 * each containing an cause allocation for an Rx and Tx ring, and a
2482 * variable number of rows depending on the number of queues supported.
2484 static void igc_write_ivar(struct igc_hw *hw, int msix_vector,
2485 int index, int offset)
2487 u32 ivar = array_rd32(IGC_IVAR0, index);
2489 /* clear any bits that are currently set */
2490 ivar &= ~((u32)0xFF << offset);
2492 /* write vector and valid bit */
2493 ivar |= (msix_vector | IGC_IVAR_VALID) << offset;
2495 array_wr32(IGC_IVAR0, index, ivar);
2498 static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector)
2500 struct igc_adapter *adapter = q_vector->adapter;
2501 struct igc_hw *hw = &adapter->hw;
2502 int rx_queue = IGC_N0_QUEUE;
2503 int tx_queue = IGC_N0_QUEUE;
2505 if (q_vector->rx.ring)
2506 rx_queue = q_vector->rx.ring->reg_idx;
2507 if (q_vector->tx.ring)
2508 tx_queue = q_vector->tx.ring->reg_idx;
2510 switch (hw->mac.type) {
2512 if (rx_queue > IGC_N0_QUEUE)
2513 igc_write_ivar(hw, msix_vector,
2515 (rx_queue & 0x1) << 4);
2516 if (tx_queue > IGC_N0_QUEUE)
2517 igc_write_ivar(hw, msix_vector,
2519 ((tx_queue & 0x1) << 4) + 8);
2520 q_vector->eims_value = BIT(msix_vector);
2523 WARN_ONCE(hw->mac.type != igc_i225, "Wrong MAC type\n");
2527 /* add q_vector eims value to global eims_enable_mask */
2528 adapter->eims_enable_mask |= q_vector->eims_value;
2530 /* configure q_vector to set itr on first interrupt */
2531 q_vector->set_itr = 1;
2535 * igc_configure_msix - Configure MSI-X hardware
2536 * @adapter: Pointer to adapter structure
2538 * igc_configure_msix sets up the hardware to properly
2539 * generate MSI-X interrupts.
2541 static void igc_configure_msix(struct igc_adapter *adapter)
2543 struct igc_hw *hw = &adapter->hw;
2547 adapter->eims_enable_mask = 0;
2549 /* set vector for other causes, i.e. link changes */
2550 switch (hw->mac.type) {
2552 /* Turn on MSI-X capability first, or our settings
2553 * won't stick. And it will take days to debug.
2555 wr32(IGC_GPIE, IGC_GPIE_MSIX_MODE |
2556 IGC_GPIE_PBA | IGC_GPIE_EIAME |
2559 /* enable msix_other interrupt */
2560 adapter->eims_other = BIT(vector);
2561 tmp = (vector++ | IGC_IVAR_VALID) << 8;
2563 wr32(IGC_IVAR_MISC, tmp);
2566 /* do nothing, since nothing else supports MSI-X */
2568 } /* switch (hw->mac.type) */
2570 adapter->eims_enable_mask |= adapter->eims_other;
2572 for (i = 0; i < adapter->num_q_vectors; i++)
2573 igc_assign_vector(adapter->q_vector[i], vector++);
2578 static irqreturn_t igc_msix_ring(int irq, void *data)
2580 struct igc_q_vector *q_vector = data;
2582 /* Write the ITR value calculated from the previous interrupt. */
2583 igc_write_itr(q_vector);
2585 napi_schedule(&q_vector->napi);
2591 * igc_request_msix - Initialize MSI-X interrupts
2592 * @adapter: Pointer to adapter structure
2594 * igc_request_msix allocates MSI-X vectors and requests interrupts from the
2597 static int igc_request_msix(struct igc_adapter *adapter)
2599 int i = 0, err = 0, vector = 0, free_vector = 0;
2600 struct net_device *netdev = adapter->netdev;
2602 err = request_irq(adapter->msix_entries[vector].vector,
2603 &igc_msix_other, 0, netdev->name, adapter);
2607 for (i = 0; i < adapter->num_q_vectors; i++) {
2608 struct igc_q_vector *q_vector = adapter->q_vector[i];
2612 q_vector->itr_register = adapter->io_addr + IGC_EITR(vector);
2614 if (q_vector->rx.ring && q_vector->tx.ring)
2615 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
2616 q_vector->rx.ring->queue_index);
2617 else if (q_vector->tx.ring)
2618 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
2619 q_vector->tx.ring->queue_index);
2620 else if (q_vector->rx.ring)
2621 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
2622 q_vector->rx.ring->queue_index);
2624 sprintf(q_vector->name, "%s-unused", netdev->name);
2626 err = request_irq(adapter->msix_entries[vector].vector,
2627 igc_msix_ring, 0, q_vector->name,
2633 igc_configure_msix(adapter);
2637 /* free already assigned IRQs */
2638 free_irq(adapter->msix_entries[free_vector++].vector, adapter);
2641 for (i = 0; i < vector; i++) {
2642 free_irq(adapter->msix_entries[free_vector++].vector,
2643 adapter->q_vector[i]);
2650 * igc_reset_q_vector - Reset config for interrupt vector
2651 * @adapter: board private structure to initialize
2652 * @v_idx: Index of vector to be reset
2654 * If NAPI is enabled it will delete any references to the
2655 * NAPI struct. This is preparation for igc_free_q_vector.
2657 static void igc_reset_q_vector(struct igc_adapter *adapter, int v_idx)
2659 struct igc_q_vector *q_vector = adapter->q_vector[v_idx];
2661 /* if we're coming from igc_set_interrupt_capability, the vectors are
2667 if (q_vector->tx.ring)
2668 adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL;
2670 if (q_vector->rx.ring)
2671 adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL;
2673 netif_napi_del(&q_vector->napi);
2676 static void igc_reset_interrupt_capability(struct igc_adapter *adapter)
2678 int v_idx = adapter->num_q_vectors;
2680 if (adapter->msix_entries) {
2681 pci_disable_msix(adapter->pdev);
2682 kfree(adapter->msix_entries);
2683 adapter->msix_entries = NULL;
2684 } else if (adapter->flags & IGC_FLAG_HAS_MSI) {
2685 pci_disable_msi(adapter->pdev);
2689 igc_reset_q_vector(adapter, v_idx);
2693 * igc_clear_interrupt_scheme - reset the device to a state of no interrupts
2694 * @adapter: Pointer to adapter structure
2696 * This function resets the device so that it has 0 rx queues, tx queues, and
2697 * MSI-X interrupts allocated.
2699 static void igc_clear_interrupt_scheme(struct igc_adapter *adapter)
2701 igc_free_q_vectors(adapter);
2702 igc_reset_interrupt_capability(adapter);
2706 * igc_free_q_vectors - Free memory allocated for interrupt vectors
2707 * @adapter: board private structure to initialize
2709 * This function frees the memory allocated to the q_vectors. In addition if
2710 * NAPI is enabled it will delete any references to the NAPI struct prior
2711 * to freeing the q_vector.
2713 static void igc_free_q_vectors(struct igc_adapter *adapter)
2715 int v_idx = adapter->num_q_vectors;
2717 adapter->num_tx_queues = 0;
2718 adapter->num_rx_queues = 0;
2719 adapter->num_q_vectors = 0;
2722 igc_reset_q_vector(adapter, v_idx);
2723 igc_free_q_vector(adapter, v_idx);
2728 * igc_free_q_vector - Free memory allocated for specific interrupt vector
2729 * @adapter: board private structure to initialize
2730 * @v_idx: Index of vector to be freed
2732 * This function frees the memory allocated to the q_vector.
2734 static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx)
2736 struct igc_q_vector *q_vector = adapter->q_vector[v_idx];
2738 adapter->q_vector[v_idx] = NULL;
2740 /* igc_get_stats64() might access the rings on this vector,
2741 * we must wait a grace period before freeing it.
2744 kfree_rcu(q_vector, rcu);
2747 /* Need to wait a few seconds after link up to get diagnostic information from
2750 static void igc_update_phy_info(struct timer_list *t)
2752 struct igc_adapter *adapter = from_timer(adapter, t, phy_info_timer);
2754 igc_get_phy_info(&adapter->hw);
2758 * igc_has_link - check shared code for link and determine up/down
2759 * @adapter: pointer to driver private info
2761 bool igc_has_link(struct igc_adapter *adapter)
2763 struct igc_hw *hw = &adapter->hw;
2764 bool link_active = false;
2766 /* get_link_status is set on LSC (link status) interrupt or
2767 * rx sequence error interrupt. get_link_status will stay
2768 * false until the igc_check_for_link establishes link
2769 * for copper adapters ONLY
2771 switch (hw->phy.media_type) {
2772 case igc_media_type_copper:
2773 if (!hw->mac.get_link_status)
2775 hw->mac.ops.check_for_link(hw);
2776 link_active = !hw->mac.get_link_status;
2779 case igc_media_type_unknown:
2783 if (hw->mac.type == igc_i225 &&
2784 hw->phy.id == I225_I_PHY_ID) {
2785 if (!netif_carrier_ok(adapter->netdev)) {
2786 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
2787 } else if (!(adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)) {
2788 adapter->flags |= IGC_FLAG_NEED_LINK_UPDATE;
2789 adapter->link_check_timeout = jiffies;
2797 * igc_watchdog - Timer Call-back
2798 * @data: pointer to adapter cast into an unsigned long
2800 static void igc_watchdog(struct timer_list *t)
2802 struct igc_adapter *adapter = from_timer(adapter, t, watchdog_timer);
2803 /* Do the rest outside of interrupt context */
2804 schedule_work(&adapter->watchdog_task);
2807 static void igc_watchdog_task(struct work_struct *work)
2809 struct igc_adapter *adapter = container_of(work,
2812 struct net_device *netdev = adapter->netdev;
2813 struct igc_hw *hw = &adapter->hw;
2814 struct igc_phy_info *phy = &hw->phy;
2815 u16 phy_data, retry_count = 20;
2820 link = igc_has_link(adapter);
2822 if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) {
2823 if (time_after(jiffies, (adapter->link_check_timeout + HZ)))
2824 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
2829 /* Force link down if we have fiber to swap to */
2830 if (adapter->flags & IGC_FLAG_MAS_ENABLE) {
2831 if (hw->phy.media_type == igc_media_type_copper) {
2832 connsw = rd32(IGC_CONNSW);
2833 if (!(connsw & IGC_CONNSW_AUTOSENSE_EN))
2838 if (!netif_carrier_ok(netdev)) {
2841 hw->mac.ops.get_speed_and_duplex(hw,
2842 &adapter->link_speed,
2843 &adapter->link_duplex);
2845 ctrl = rd32(IGC_CTRL);
2846 /* Link status message must follow this format */
2848 "igc: %s NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n",
2850 adapter->link_speed,
2851 adapter->link_duplex == FULL_DUPLEX ?
2853 (ctrl & IGC_CTRL_TFCE) &&
2854 (ctrl & IGC_CTRL_RFCE) ? "RX/TX" :
2855 (ctrl & IGC_CTRL_RFCE) ? "RX" :
2856 (ctrl & IGC_CTRL_TFCE) ? "TX" : "None");
2858 /* check if SmartSpeed worked */
2859 igc_check_downshift(hw);
2860 if (phy->speed_downgraded)
2861 netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n");
2863 /* adjust timeout factor according to speed/duplex */
2864 adapter->tx_timeout_factor = 1;
2865 switch (adapter->link_speed) {
2867 adapter->tx_timeout_factor = 14;
2870 /* maybe add some timeout factor ? */
2874 if (adapter->link_speed != SPEED_1000)
2877 /* wait for Remote receiver status OK */
2879 if (!igc_read_phy_reg(hw, PHY_1000T_STATUS,
2881 if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) &&
2885 goto retry_read_status;
2886 } else if (!retry_count) {
2887 dev_err(&adapter->pdev->dev, "exceed max 2 second\n");
2890 dev_err(&adapter->pdev->dev, "read 1000Base-T Status Reg\n");
2893 netif_carrier_on(netdev);
2895 /* link state has changed, schedule phy info update */
2896 if (!test_bit(__IGC_DOWN, &adapter->state))
2897 mod_timer(&adapter->phy_info_timer,
2898 round_jiffies(jiffies + 2 * HZ));
2901 if (netif_carrier_ok(netdev)) {
2902 adapter->link_speed = 0;
2903 adapter->link_duplex = 0;
2905 /* Links status message must follow this format */
2906 netdev_info(netdev, "igc: %s NIC Link is Down\n",
2908 netif_carrier_off(netdev);
2910 /* link state has changed, schedule phy info update */
2911 if (!test_bit(__IGC_DOWN, &adapter->state))
2912 mod_timer(&adapter->phy_info_timer,
2913 round_jiffies(jiffies + 2 * HZ));
2915 /* link is down, time to check for alternate media */
2916 if (adapter->flags & IGC_FLAG_MAS_ENABLE) {
2917 if (adapter->flags & IGC_FLAG_MEDIA_RESET) {
2918 schedule_work(&adapter->reset_task);
2919 /* return immediately */
2924 /* also check for alternate media here */
2925 } else if (!netif_carrier_ok(netdev) &&
2926 (adapter->flags & IGC_FLAG_MAS_ENABLE)) {
2927 if (adapter->flags & IGC_FLAG_MEDIA_RESET) {
2928 schedule_work(&adapter->reset_task);
2929 /* return immediately */
2935 spin_lock(&adapter->stats64_lock);
2936 igc_update_stats(adapter);
2937 spin_unlock(&adapter->stats64_lock);
2939 for (i = 0; i < adapter->num_tx_queues; i++) {
2940 struct igc_ring *tx_ring = adapter->tx_ring[i];
2942 if (!netif_carrier_ok(netdev)) {
2943 /* We've lost link, so the controller stops DMA,
2944 * but we've got queued Tx work that's never going
2945 * to get done, so reset controller to flush Tx.
2946 * (Do the reset outside of interrupt context).
2948 if (igc_desc_unused(tx_ring) + 1 < tx_ring->count) {
2949 adapter->tx_timeout_count++;
2950 schedule_work(&adapter->reset_task);
2951 /* return immediately since reset is imminent */
2956 /* Force detection of hung controller every watchdog period */
2957 set_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
2960 /* Cause software interrupt to ensure Rx ring is cleaned */
2961 if (adapter->flags & IGC_FLAG_HAS_MSIX) {
2964 for (i = 0; i < adapter->num_q_vectors; i++)
2965 eics |= adapter->q_vector[i]->eims_value;
2966 wr32(IGC_EICS, eics);
2968 wr32(IGC_ICS, IGC_ICS_RXDMT0);
2971 /* Reset the timer */
2972 if (!test_bit(__IGC_DOWN, &adapter->state)) {
2973 if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)
2974 mod_timer(&adapter->watchdog_timer,
2975 round_jiffies(jiffies + HZ));
2977 mod_timer(&adapter->watchdog_timer,
2978 round_jiffies(jiffies + 2 * HZ));
2983 * igc_update_ring_itr - update the dynamic ITR value based on packet size
2984 * @q_vector: pointer to q_vector
2986 * Stores a new ITR value based on strictly on packet size. This
2987 * algorithm is less sophisticated than that used in igc_update_itr,
2988 * due to the difficulty of synchronizing statistics across multiple
2989 * receive rings. The divisors and thresholds used by this function
2990 * were determined based on theoretical maximum wire speed and testing
2991 * data, in order to minimize response time while increasing bulk
2993 * NOTE: This function is called only when operating in a multiqueue
2994 * receive environment.
2996 static void igc_update_ring_itr(struct igc_q_vector *q_vector)
2998 struct igc_adapter *adapter = q_vector->adapter;
2999 int new_val = q_vector->itr_val;
3000 int avg_wire_size = 0;
3001 unsigned int packets;
3003 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3004 * ints/sec - ITR timer value of 120 ticks.
3006 switch (adapter->link_speed) {
3009 new_val = IGC_4K_ITR;
3015 packets = q_vector->rx.total_packets;
3017 avg_wire_size = q_vector->rx.total_bytes / packets;
3019 packets = q_vector->tx.total_packets;
3021 avg_wire_size = max_t(u32, avg_wire_size,
3022 q_vector->tx.total_bytes / packets);
3024 /* if avg_wire_size isn't set no work was done */
3028 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3029 avg_wire_size += 24;
3031 /* Don't starve jumbo frames */
3032 avg_wire_size = min(avg_wire_size, 3000);
3034 /* Give a little boost to mid-size frames */
3035 if (avg_wire_size > 300 && avg_wire_size < 1200)
3036 new_val = avg_wire_size / 3;
3038 new_val = avg_wire_size / 2;
3040 /* conservative mode (itr 3) eliminates the lowest_latency setting */
3041 if (new_val < IGC_20K_ITR &&
3042 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
3043 (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
3044 new_val = IGC_20K_ITR;
3047 if (new_val != q_vector->itr_val) {
3048 q_vector->itr_val = new_val;
3049 q_vector->set_itr = 1;
3052 q_vector->rx.total_bytes = 0;
3053 q_vector->rx.total_packets = 0;
3054 q_vector->tx.total_bytes = 0;
3055 q_vector->tx.total_packets = 0;
3059 * igc_update_itr - update the dynamic ITR value based on statistics
3060 * @q_vector: pointer to q_vector
3061 * @ring_container: ring info to update the itr for
3063 * Stores a new ITR value based on packets and byte
3064 * counts during the last interrupt. The advantage of per interrupt
3065 * computation is faster updates and more accurate ITR for the current
3066 * traffic pattern. Constants in this function were computed
3067 * based on theoretical maximum wire speed and thresholds were set based
3068 * on testing data as well as attempting to minimize response time
3069 * while increasing bulk throughput.
3070 * NOTE: These calculations are only valid when operating in a single-
3071 * queue environment.
3073 static void igc_update_itr(struct igc_q_vector *q_vector,
3074 struct igc_ring_container *ring_container)
3076 unsigned int packets = ring_container->total_packets;
3077 unsigned int bytes = ring_container->total_bytes;
3078 u8 itrval = ring_container->itr;
3080 /* no packets, exit with status unchanged */
3085 case lowest_latency:
3086 /* handle TSO and jumbo frames */
3087 if (bytes / packets > 8000)
3088 itrval = bulk_latency;
3089 else if ((packets < 5) && (bytes > 512))
3090 itrval = low_latency;
3092 case low_latency: /* 50 usec aka 20000 ints/s */
3093 if (bytes > 10000) {
3094 /* this if handles the TSO accounting */
3095 if (bytes / packets > 8000)
3096 itrval = bulk_latency;
3097 else if ((packets < 10) || ((bytes / packets) > 1200))
3098 itrval = bulk_latency;
3099 else if ((packets > 35))
3100 itrval = lowest_latency;
3101 } else if (bytes / packets > 2000) {
3102 itrval = bulk_latency;
3103 } else if (packets <= 2 && bytes < 512) {
3104 itrval = lowest_latency;
3107 case bulk_latency: /* 250 usec aka 4000 ints/s */
3108 if (bytes > 25000) {
3110 itrval = low_latency;
3111 } else if (bytes < 1500) {
3112 itrval = low_latency;
3117 /* clear work counters since we have the values we need */
3118 ring_container->total_bytes = 0;
3119 ring_container->total_packets = 0;
3121 /* write updated itr to ring container */
3122 ring_container->itr = itrval;
3126 * igc_intr_msi - Interrupt Handler
3127 * @irq: interrupt number
3128 * @data: pointer to a network interface device structure
3130 static irqreturn_t igc_intr_msi(int irq, void *data)
3132 struct igc_adapter *adapter = data;
3133 struct igc_q_vector *q_vector = adapter->q_vector[0];
3134 struct igc_hw *hw = &adapter->hw;
3135 /* read ICR disables interrupts using IAM */
3136 u32 icr = rd32(IGC_ICR);
3138 igc_write_itr(q_vector);
3140 if (icr & IGC_ICR_DRSTA)
3141 schedule_work(&adapter->reset_task);
3143 if (icr & IGC_ICR_DOUTSYNC) {
3144 /* HW is reporting DMA is out of sync */
3145 adapter->stats.doosync++;
3148 if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) {
3149 hw->mac.get_link_status = 1;
3150 if (!test_bit(__IGC_DOWN, &adapter->state))
3151 mod_timer(&adapter->watchdog_timer, jiffies + 1);
3154 napi_schedule(&q_vector->napi);
3160 * igc_intr - Legacy Interrupt Handler
3161 * @irq: interrupt number
3162 * @data: pointer to a network interface device structure
3164 static irqreturn_t igc_intr(int irq, void *data)
3166 struct igc_adapter *adapter = data;
3167 struct igc_q_vector *q_vector = adapter->q_vector[0];
3168 struct igc_hw *hw = &adapter->hw;
3169 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
3170 * need for the IMC write
3172 u32 icr = rd32(IGC_ICR);
3174 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
3175 * not set, then the adapter didn't send an interrupt
3177 if (!(icr & IGC_ICR_INT_ASSERTED))
3180 igc_write_itr(q_vector);
3182 if (icr & IGC_ICR_DRSTA)
3183 schedule_work(&adapter->reset_task);
3185 if (icr & IGC_ICR_DOUTSYNC) {
3186 /* HW is reporting DMA is out of sync */
3187 adapter->stats.doosync++;
3190 if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) {
3191 hw->mac.get_link_status = 1;
3192 /* guard against interrupt when we're going down */
3193 if (!test_bit(__IGC_DOWN, &adapter->state))
3194 mod_timer(&adapter->watchdog_timer, jiffies + 1);
3197 napi_schedule(&q_vector->napi);
3202 static void igc_set_itr(struct igc_q_vector *q_vector)
3204 struct igc_adapter *adapter = q_vector->adapter;
3205 u32 new_itr = q_vector->itr_val;
3208 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3209 switch (adapter->link_speed) {
3213 new_itr = IGC_4K_ITR;
3219 igc_update_itr(q_vector, &q_vector->tx);
3220 igc_update_itr(q_vector, &q_vector->rx);
3222 current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
3224 /* conservative mode (itr 3) eliminates the lowest_latency setting */
3225 if (current_itr == lowest_latency &&
3226 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
3227 (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
3228 current_itr = low_latency;
3230 switch (current_itr) {
3231 /* counts and packets in update_itr are dependent on these numbers */
3232 case lowest_latency:
3233 new_itr = IGC_70K_ITR; /* 70,000 ints/sec */
3236 new_itr = IGC_20K_ITR; /* 20,000 ints/sec */
3239 new_itr = IGC_4K_ITR; /* 4,000 ints/sec */
3246 if (new_itr != q_vector->itr_val) {
3247 /* this attempts to bias the interrupt rate towards Bulk
3248 * by adding intermediate steps when interrupt rate is
3251 new_itr = new_itr > q_vector->itr_val ?
3252 max((new_itr * q_vector->itr_val) /
3253 (new_itr + (q_vector->itr_val >> 2)),
3255 /* Don't write the value here; it resets the adapter's
3256 * internal timer, and causes us to delay far longer than
3257 * we should between interrupts. Instead, we write the ITR
3258 * value at the beginning of the next interrupt so the timing
3259 * ends up being correct.
3261 q_vector->itr_val = new_itr;
3262 q_vector->set_itr = 1;
3266 static void igc_ring_irq_enable(struct igc_q_vector *q_vector)
3268 struct igc_adapter *adapter = q_vector->adapter;
3269 struct igc_hw *hw = &adapter->hw;
3271 if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
3272 (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
3273 if (adapter->num_q_vectors == 1)
3274 igc_set_itr(q_vector);
3276 igc_update_ring_itr(q_vector);
3279 if (!test_bit(__IGC_DOWN, &adapter->state)) {
3280 if (adapter->msix_entries)
3281 wr32(IGC_EIMS, q_vector->eims_value);
3283 igc_irq_enable(adapter);
3288 * igc_poll - NAPI Rx polling callback
3289 * @napi: napi polling structure
3290 * @budget: count of how many packets we should handle
3292 static int igc_poll(struct napi_struct *napi, int budget)
3294 struct igc_q_vector *q_vector = container_of(napi,
3295 struct igc_q_vector,
3297 bool clean_complete = true;
3300 if (q_vector->tx.ring)
3301 clean_complete = igc_clean_tx_irq(q_vector, budget);
3303 if (q_vector->rx.ring) {
3304 int cleaned = igc_clean_rx_irq(q_vector, budget);
3306 work_done += cleaned;
3307 if (cleaned >= budget)
3308 clean_complete = false;
3311 /* If all work not completed, return budget and keep polling */
3312 if (!clean_complete)
3315 /* Exit the polling mode, but don't re-enable interrupts if stack might
3316 * poll us due to busy-polling
3318 if (likely(napi_complete_done(napi, work_done)))
3319 igc_ring_irq_enable(q_vector);
3321 return min(work_done, budget - 1);
3325 * igc_set_interrupt_capability - set MSI or MSI-X if supported
3326 * @adapter: Pointer to adapter structure
3328 * Attempt to configure interrupts using the best available
3329 * capabilities of the hardware and kernel.
3331 static void igc_set_interrupt_capability(struct igc_adapter *adapter,
3339 adapter->flags |= IGC_FLAG_HAS_MSIX;
3341 /* Number of supported queues. */
3342 adapter->num_rx_queues = adapter->rss_queues;
3344 adapter->num_tx_queues = adapter->rss_queues;
3346 /* start with one vector for every Rx queue */
3347 numvecs = adapter->num_rx_queues;
3349 /* if Tx handler is separate add 1 for every Tx queue */
3350 if (!(adapter->flags & IGC_FLAG_QUEUE_PAIRS))
3351 numvecs += adapter->num_tx_queues;
3353 /* store the number of vectors reserved for queues */
3354 adapter->num_q_vectors = numvecs;
3356 /* add 1 vector for link status interrupts */
3359 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
3362 if (!adapter->msix_entries)
3365 /* populate entry values */
3366 for (i = 0; i < numvecs; i++)
3367 adapter->msix_entries[i].entry = i;
3369 err = pci_enable_msix_range(adapter->pdev,
3370 adapter->msix_entries,
3376 kfree(adapter->msix_entries);
3377 adapter->msix_entries = NULL;
3379 igc_reset_interrupt_capability(adapter);
3382 adapter->flags &= ~IGC_FLAG_HAS_MSIX;
3384 adapter->rss_queues = 1;
3385 adapter->flags |= IGC_FLAG_QUEUE_PAIRS;
3386 adapter->num_rx_queues = 1;
3387 adapter->num_tx_queues = 1;
3388 adapter->num_q_vectors = 1;
3389 if (!pci_enable_msi(adapter->pdev))
3390 adapter->flags |= IGC_FLAG_HAS_MSI;
3393 static void igc_add_ring(struct igc_ring *ring,
3394 struct igc_ring_container *head)
3401 * igc_alloc_q_vector - Allocate memory for a single interrupt vector
3402 * @adapter: board private structure to initialize
3403 * @v_count: q_vectors allocated on adapter, used for ring interleaving
3404 * @v_idx: index of vector in adapter struct
3405 * @txr_count: total number of Tx rings to allocate
3406 * @txr_idx: index of first Tx ring to allocate
3407 * @rxr_count: total number of Rx rings to allocate
3408 * @rxr_idx: index of first Rx ring to allocate
3410 * We allocate one q_vector. If allocation fails we return -ENOMEM.
3412 static int igc_alloc_q_vector(struct igc_adapter *adapter,
3413 unsigned int v_count, unsigned int v_idx,
3414 unsigned int txr_count, unsigned int txr_idx,
3415 unsigned int rxr_count, unsigned int rxr_idx)
3417 struct igc_q_vector *q_vector;
3418 struct igc_ring *ring;
3421 /* igc only supports 1 Tx and/or 1 Rx queue per vector */
3422 if (txr_count > 1 || rxr_count > 1)
3425 ring_count = txr_count + rxr_count;
3427 /* allocate q_vector and rings */
3428 q_vector = adapter->q_vector[v_idx];
3430 q_vector = kzalloc(struct_size(q_vector, ring, ring_count),
3433 memset(q_vector, 0, struct_size(q_vector, ring, ring_count));
3437 /* initialize NAPI */
3438 netif_napi_add(adapter->netdev, &q_vector->napi,
3441 /* tie q_vector and adapter together */
3442 adapter->q_vector[v_idx] = q_vector;
3443 q_vector->adapter = adapter;
3445 /* initialize work limits */
3446 q_vector->tx.work_limit = adapter->tx_work_limit;
3448 /* initialize ITR configuration */
3449 q_vector->itr_register = adapter->io_addr + IGC_EITR(0);
3450 q_vector->itr_val = IGC_START_ITR;
3452 /* initialize pointer to rings */
3453 ring = q_vector->ring;
3455 /* initialize ITR */
3457 /* rx or rx/tx vector */
3458 if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3)
3459 q_vector->itr_val = adapter->rx_itr_setting;
3461 /* tx only vector */
3462 if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3)
3463 q_vector->itr_val = adapter->tx_itr_setting;
3467 /* assign generic ring traits */
3468 ring->dev = &adapter->pdev->dev;
3469 ring->netdev = adapter->netdev;
3471 /* configure backlink on ring */
3472 ring->q_vector = q_vector;
3474 /* update q_vector Tx values */
3475 igc_add_ring(ring, &q_vector->tx);
3477 /* apply Tx specific ring traits */
3478 ring->count = adapter->tx_ring_count;
3479 ring->queue_index = txr_idx;
3481 /* assign ring to adapter */
3482 adapter->tx_ring[txr_idx] = ring;
3484 /* push pointer to next ring */
3489 /* assign generic ring traits */
3490 ring->dev = &adapter->pdev->dev;
3491 ring->netdev = adapter->netdev;
3493 /* configure backlink on ring */
3494 ring->q_vector = q_vector;
3496 /* update q_vector Rx values */
3497 igc_add_ring(ring, &q_vector->rx);
3499 /* apply Rx specific ring traits */
3500 ring->count = adapter->rx_ring_count;
3501 ring->queue_index = rxr_idx;
3503 /* assign ring to adapter */
3504 adapter->rx_ring[rxr_idx] = ring;
3511 * igc_alloc_q_vectors - Allocate memory for interrupt vectors
3512 * @adapter: board private structure to initialize
3514 * We allocate one q_vector per queue interrupt. If allocation fails we
3517 static int igc_alloc_q_vectors(struct igc_adapter *adapter)
3519 int rxr_remaining = adapter->num_rx_queues;
3520 int txr_remaining = adapter->num_tx_queues;
3521 int rxr_idx = 0, txr_idx = 0, v_idx = 0;
3522 int q_vectors = adapter->num_q_vectors;
3525 if (q_vectors >= (rxr_remaining + txr_remaining)) {
3526 for (; rxr_remaining; v_idx++) {
3527 err = igc_alloc_q_vector(adapter, q_vectors, v_idx,
3533 /* update counts and index */
3539 for (; v_idx < q_vectors; v_idx++) {
3540 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx);
3541 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx);
3543 err = igc_alloc_q_vector(adapter, q_vectors, v_idx,
3544 tqpv, txr_idx, rqpv, rxr_idx);
3549 /* update counts and index */
3550 rxr_remaining -= rqpv;
3551 txr_remaining -= tqpv;
3559 adapter->num_tx_queues = 0;
3560 adapter->num_rx_queues = 0;
3561 adapter->num_q_vectors = 0;
3564 igc_free_q_vector(adapter, v_idx);
3570 * igc_cache_ring_register - Descriptor ring to register mapping
3571 * @adapter: board private structure to initialize
3573 * Once we know the feature-set enabled for the device, we'll cache
3574 * the register offset the descriptor ring is assigned to.
3576 static void igc_cache_ring_register(struct igc_adapter *adapter)
3580 switch (adapter->hw.mac.type) {
3584 for (; i < adapter->num_rx_queues; i++)
3585 adapter->rx_ring[i]->reg_idx = i;
3586 for (; j < adapter->num_tx_queues; j++)
3587 adapter->tx_ring[j]->reg_idx = j;
3593 * igc_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
3594 * @adapter: Pointer to adapter structure
3596 * This function initializes the interrupts and allocates all of the queues.
3598 static int igc_init_interrupt_scheme(struct igc_adapter *adapter, bool msix)
3600 struct pci_dev *pdev = adapter->pdev;
3603 igc_set_interrupt_capability(adapter, msix);
3605 err = igc_alloc_q_vectors(adapter);
3607 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
3608 goto err_alloc_q_vectors;
3611 igc_cache_ring_register(adapter);
3615 err_alloc_q_vectors:
3616 igc_reset_interrupt_capability(adapter);
3620 static void igc_free_irq(struct igc_adapter *adapter)
3622 if (adapter->msix_entries) {
3625 free_irq(adapter->msix_entries[vector++].vector, adapter);
3627 for (i = 0; i < adapter->num_q_vectors; i++)
3628 free_irq(adapter->msix_entries[vector++].vector,
3629 adapter->q_vector[i]);
3631 free_irq(adapter->pdev->irq, adapter);
3636 * igc_irq_disable - Mask off interrupt generation on the NIC
3637 * @adapter: board private structure
3639 static void igc_irq_disable(struct igc_adapter *adapter)
3641 struct igc_hw *hw = &adapter->hw;
3643 if (adapter->msix_entries) {
3644 u32 regval = rd32(IGC_EIAM);
3646 wr32(IGC_EIAM, regval & ~adapter->eims_enable_mask);
3647 wr32(IGC_EIMC, adapter->eims_enable_mask);
3648 regval = rd32(IGC_EIAC);
3649 wr32(IGC_EIAC, regval & ~adapter->eims_enable_mask);
3656 if (adapter->msix_entries) {
3659 synchronize_irq(adapter->msix_entries[vector++].vector);
3661 for (i = 0; i < adapter->num_q_vectors; i++)
3662 synchronize_irq(adapter->msix_entries[vector++].vector);
3664 synchronize_irq(adapter->pdev->irq);
3669 * igc_irq_enable - Enable default interrupt generation settings
3670 * @adapter: board private structure
3672 static void igc_irq_enable(struct igc_adapter *adapter)
3674 struct igc_hw *hw = &adapter->hw;
3676 if (adapter->msix_entries) {
3677 u32 ims = IGC_IMS_LSC | IGC_IMS_DOUTSYNC | IGC_IMS_DRSTA;
3678 u32 regval = rd32(IGC_EIAC);
3680 wr32(IGC_EIAC, regval | adapter->eims_enable_mask);
3681 regval = rd32(IGC_EIAM);
3682 wr32(IGC_EIAM, regval | adapter->eims_enable_mask);
3683 wr32(IGC_EIMS, adapter->eims_enable_mask);
3686 wr32(IGC_IMS, IMS_ENABLE_MASK | IGC_IMS_DRSTA);
3687 wr32(IGC_IAM, IMS_ENABLE_MASK | IGC_IMS_DRSTA);
3692 * igc_request_irq - initialize interrupts
3693 * @adapter: Pointer to adapter structure
3695 * Attempts to configure interrupts using the best available
3696 * capabilities of the hardware and kernel.
3698 static int igc_request_irq(struct igc_adapter *adapter)
3700 struct net_device *netdev = adapter->netdev;
3701 struct pci_dev *pdev = adapter->pdev;
3704 if (adapter->flags & IGC_FLAG_HAS_MSIX) {
3705 err = igc_request_msix(adapter);
3708 /* fall back to MSI */
3709 igc_free_all_tx_resources(adapter);
3710 igc_free_all_rx_resources(adapter);
3712 igc_clear_interrupt_scheme(adapter);
3713 err = igc_init_interrupt_scheme(adapter, false);
3716 igc_setup_all_tx_resources(adapter);
3717 igc_setup_all_rx_resources(adapter);
3718 igc_configure(adapter);
3721 igc_assign_vector(adapter->q_vector[0], 0);
3723 if (adapter->flags & IGC_FLAG_HAS_MSI) {
3724 err = request_irq(pdev->irq, &igc_intr_msi, 0,
3725 netdev->name, adapter);
3729 /* fall back to legacy interrupts */
3730 igc_reset_interrupt_capability(adapter);
3731 adapter->flags &= ~IGC_FLAG_HAS_MSI;
3734 err = request_irq(pdev->irq, &igc_intr, IRQF_SHARED,
3735 netdev->name, adapter);
3738 dev_err(&pdev->dev, "Error %d getting interrupt\n",
3745 static void igc_write_itr(struct igc_q_vector *q_vector)
3747 u32 itr_val = q_vector->itr_val & IGC_QVECTOR_MASK;
3749 if (!q_vector->set_itr)
3753 itr_val = IGC_ITR_VAL_MASK;
3755 itr_val |= IGC_EITR_CNT_IGNR;
3757 writel(itr_val, q_vector->itr_register);
3758 q_vector->set_itr = 0;
3762 * igc_open - Called when a network interface is made active
3763 * @netdev: network interface device structure
3765 * Returns 0 on success, negative value on failure
3767 * The open entry point is called when a network interface is made
3768 * active by the system (IFF_UP). At this point all resources needed
3769 * for transmit and receive operations are allocated, the interrupt
3770 * handler is registered with the OS, the watchdog timer is started,
3771 * and the stack is notified that the interface is ready.
3773 static int __igc_open(struct net_device *netdev, bool resuming)
3775 struct igc_adapter *adapter = netdev_priv(netdev);
3776 struct igc_hw *hw = &adapter->hw;
3780 /* disallow open during test */
3782 if (test_bit(__IGC_TESTING, &adapter->state)) {
3787 netif_carrier_off(netdev);
3789 /* allocate transmit descriptors */
3790 err = igc_setup_all_tx_resources(adapter);
3794 /* allocate receive descriptors */
3795 err = igc_setup_all_rx_resources(adapter);
3799 igc_power_up_link(adapter);
3801 igc_configure(adapter);
3803 err = igc_request_irq(adapter);
3807 /* Notify the stack of the actual queue counts. */
3808 err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues);
3810 goto err_set_queues;
3812 err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues);
3814 goto err_set_queues;
3816 clear_bit(__IGC_DOWN, &adapter->state);
3818 for (i = 0; i < adapter->num_q_vectors; i++)
3819 napi_enable(&adapter->q_vector[i]->napi);
3821 /* Clear any pending interrupts. */
3823 igc_irq_enable(adapter);
3825 netif_tx_start_all_queues(netdev);
3827 /* start the watchdog. */
3828 hw->mac.get_link_status = 1;
3829 schedule_work(&adapter->watchdog_task);
3834 igc_free_irq(adapter);
3836 igc_release_hw_control(adapter);
3837 igc_power_down_link(adapter);
3838 igc_free_all_rx_resources(adapter);
3840 igc_free_all_tx_resources(adapter);
3847 static int igc_open(struct net_device *netdev)
3849 return __igc_open(netdev, false);
3853 * igc_close - Disables a network interface
3854 * @netdev: network interface device structure
3856 * Returns 0, this is not allowed to fail
3858 * The close entry point is called when an interface is de-activated
3859 * by the OS. The hardware is still under the driver's control, but
3860 * needs to be disabled. A global MAC reset is issued to stop the
3861 * hardware, and all transmit and receive resources are freed.
3863 static int __igc_close(struct net_device *netdev, bool suspending)
3865 struct igc_adapter *adapter = netdev_priv(netdev);
3867 WARN_ON(test_bit(__IGC_RESETTING, &adapter->state));
3871 igc_release_hw_control(adapter);
3873 igc_free_irq(adapter);
3875 igc_free_all_tx_resources(adapter);
3876 igc_free_all_rx_resources(adapter);
3881 static int igc_close(struct net_device *netdev)
3883 if (netif_device_present(netdev) || netdev->dismantle)
3884 return __igc_close(netdev, false);
3888 static const struct net_device_ops igc_netdev_ops = {
3889 .ndo_open = igc_open,
3890 .ndo_stop = igc_close,
3891 .ndo_start_xmit = igc_xmit_frame,
3892 .ndo_set_mac_address = igc_set_mac,
3893 .ndo_change_mtu = igc_change_mtu,
3894 .ndo_get_stats = igc_get_stats,
3895 .ndo_fix_features = igc_fix_features,
3896 .ndo_set_features = igc_set_features,
3897 .ndo_features_check = igc_features_check,
3900 /* PCIe configuration access */
3901 void igc_read_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value)
3903 struct igc_adapter *adapter = hw->back;
3905 pci_read_config_word(adapter->pdev, reg, value);
3908 void igc_write_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value)
3910 struct igc_adapter *adapter = hw->back;
3912 pci_write_config_word(adapter->pdev, reg, *value);
3915 s32 igc_read_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value)
3917 struct igc_adapter *adapter = hw->back;
3919 if (!pci_is_pcie(adapter->pdev))
3920 return -IGC_ERR_CONFIG;
3922 pcie_capability_read_word(adapter->pdev, reg, value);
3927 s32 igc_write_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value)
3929 struct igc_adapter *adapter = hw->back;
3931 if (!pci_is_pcie(adapter->pdev))
3932 return -IGC_ERR_CONFIG;
3934 pcie_capability_write_word(adapter->pdev, reg, *value);
3939 u32 igc_rd32(struct igc_hw *hw, u32 reg)
3941 struct igc_adapter *igc = container_of(hw, struct igc_adapter, hw);
3942 u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr);
3945 if (IGC_REMOVED(hw_addr))
3948 value = readl(&hw_addr[reg]);
3950 /* reads should not return all F's */
3951 if (!(~value) && (!reg || !(~readl(hw_addr)))) {
3952 struct net_device *netdev = igc->netdev;
3955 netif_device_detach(netdev);
3956 netdev_err(netdev, "PCIe link lost, device now detached\n");
3957 WARN(1, "igc: Failed to read reg 0x%x!\n", reg);
3963 int igc_set_spd_dplx(struct igc_adapter *adapter, u32 spd, u8 dplx)
3965 struct pci_dev *pdev = adapter->pdev;
3966 struct igc_mac_info *mac = &adapter->hw.mac;
3970 /* Make sure dplx is at most 1 bit and lsb of speed is not set
3971 * for the switch() below to work
3973 if ((spd & 1) || (dplx & ~1))
3976 switch (spd + dplx) {
3977 case SPEED_10 + DUPLEX_HALF:
3978 mac->forced_speed_duplex = ADVERTISE_10_HALF;
3980 case SPEED_10 + DUPLEX_FULL:
3981 mac->forced_speed_duplex = ADVERTISE_10_FULL;
3983 case SPEED_100 + DUPLEX_HALF:
3984 mac->forced_speed_duplex = ADVERTISE_100_HALF;
3986 case SPEED_100 + DUPLEX_FULL:
3987 mac->forced_speed_duplex = ADVERTISE_100_FULL;
3989 case SPEED_1000 + DUPLEX_FULL:
3991 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
3993 case SPEED_1000 + DUPLEX_HALF: /* not supported */
3995 case SPEED_2500 + DUPLEX_FULL:
3997 adapter->hw.phy.autoneg_advertised = ADVERTISE_2500_FULL;
3999 case SPEED_2500 + DUPLEX_HALF: /* not supported */
4004 /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */
4005 adapter->hw.phy.mdix = AUTO_ALL_MODES;
4010 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
4015 * igc_probe - Device Initialization Routine
4016 * @pdev: PCI device information struct
4017 * @ent: entry in igc_pci_tbl
4019 * Returns 0 on success, negative on failure
4021 * igc_probe initializes an adapter identified by a pci_dev structure.
4022 * The OS initialization, configuring the adapter private structure,
4023 * and a hardware reset occur.
4025 static int igc_probe(struct pci_dev *pdev,
4026 const struct pci_device_id *ent)
4028 struct igc_adapter *adapter;
4029 struct net_device *netdev;
4031 const struct igc_info *ei = igc_info_tbl[ent->driver_data];
4034 err = pci_enable_device_mem(pdev);
4038 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
4040 err = dma_set_coherent_mask(&pdev->dev,
4043 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
4045 err = dma_set_coherent_mask(&pdev->dev,
4048 dev_err(&pdev->dev, "igc: Wrong DMA config\n");
4054 err = pci_request_selected_regions(pdev,
4055 pci_select_bars(pdev,
4061 pci_enable_pcie_error_reporting(pdev);
4063 pci_set_master(pdev);
4066 netdev = alloc_etherdev_mq(sizeof(struct igc_adapter),
4070 goto err_alloc_etherdev;
4072 SET_NETDEV_DEV(netdev, &pdev->dev);
4074 pci_set_drvdata(pdev, netdev);
4075 adapter = netdev_priv(netdev);
4076 adapter->netdev = netdev;
4077 adapter->pdev = pdev;
4080 adapter->port_num = hw->bus.func;
4081 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
4083 err = pci_save_state(pdev);
4088 adapter->io_addr = ioremap(pci_resource_start(pdev, 0),
4089 pci_resource_len(pdev, 0));
4090 if (!adapter->io_addr)
4093 /* hw->hw_addr can be zeroed, so use adapter->io_addr for unmap */
4094 hw->hw_addr = adapter->io_addr;
4096 netdev->netdev_ops = &igc_netdev_ops;
4097 igc_set_ethtool_ops(netdev);
4098 netdev->watchdog_timeo = 5 * HZ;
4100 netdev->mem_start = pci_resource_start(pdev, 0);
4101 netdev->mem_end = pci_resource_end(pdev, 0);
4103 /* PCI config space info */
4104 hw->vendor_id = pdev->vendor;
4105 hw->device_id = pdev->device;
4106 hw->revision_id = pdev->revision;
4107 hw->subsystem_vendor_id = pdev->subsystem_vendor;
4108 hw->subsystem_device_id = pdev->subsystem_device;
4110 /* Copy the default MAC and PHY function pointers */
4111 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
4112 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
4114 /* Initialize skew-specific constants */
4115 err = ei->get_invariants(hw);
4119 /* setup the private structure */
4120 err = igc_sw_init(adapter);
4124 /* copy netdev features into list of user selectable features */
4125 netdev->hw_features |= NETIF_F_NTUPLE;
4127 /* MTU range: 68 - 9216 */
4128 netdev->min_mtu = ETH_MIN_MTU;
4129 netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE;
4131 /* before reading the NVM, reset the controller to put the device in a
4132 * known good starting state
4134 hw->mac.ops.reset_hw(hw);
4136 if (eth_platform_get_mac_address(&pdev->dev, hw->mac.addr)) {
4137 /* copy the MAC address out of the NVM */
4138 if (hw->mac.ops.read_mac_addr(hw))
4139 dev_err(&pdev->dev, "NVM Read Error\n");
4142 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
4144 if (!is_valid_ether_addr(netdev->dev_addr)) {
4145 dev_err(&pdev->dev, "Invalid MAC Address\n");
4150 /* configure RXPBSIZE and TXPBSIZE */
4151 wr32(IGC_RXPBS, I225_RXPBSIZE_DEFAULT);
4152 wr32(IGC_TXPBS, I225_TXPBSIZE_DEFAULT);
4154 timer_setup(&adapter->watchdog_timer, igc_watchdog, 0);
4155 timer_setup(&adapter->phy_info_timer, igc_update_phy_info, 0);
4157 INIT_WORK(&adapter->reset_task, igc_reset_task);
4158 INIT_WORK(&adapter->watchdog_task, igc_watchdog_task);
4160 /* Initialize link properties that are user-changeable */
4161 adapter->fc_autoneg = true;
4162 hw->mac.autoneg = true;
4163 hw->phy.autoneg_advertised = 0xaf;
4165 hw->fc.requested_mode = igc_fc_default;
4166 hw->fc.current_mode = igc_fc_default;
4168 /* reset the hardware with the new settings */
4171 /* let the f/w know that the h/w is now under the control of the
4174 igc_get_hw_control(adapter);
4176 strncpy(netdev->name, "eth%d", IFNAMSIZ);
4177 err = register_netdev(netdev);
4181 /* carrier off reporting is important to ethtool even BEFORE open */
4182 netif_carrier_off(netdev);
4184 /* Check if Media Autosense is enabled */
4187 /* print pcie link status and MAC address */
4188 pcie_print_link_status(pdev);
4189 netdev_info(netdev, "MAC: %pM\n", netdev->dev_addr);
4194 igc_release_hw_control(adapter);
4196 if (!igc_check_reset_block(hw))
4199 igc_clear_interrupt_scheme(adapter);
4200 iounmap(adapter->io_addr);
4202 free_netdev(netdev);
4204 pci_release_selected_regions(pdev,
4205 pci_select_bars(pdev, IORESOURCE_MEM));
4208 pci_disable_device(pdev);
4213 * igc_remove - Device Removal Routine
4214 * @pdev: PCI device information struct
4216 * igc_remove is called by the PCI subsystem to alert the driver
4217 * that it should release a PCI device. This could be caused by a
4218 * Hot-Plug event, or because the driver is going to be removed from
4221 static void igc_remove(struct pci_dev *pdev)
4223 struct net_device *netdev = pci_get_drvdata(pdev);
4224 struct igc_adapter *adapter = netdev_priv(netdev);
4226 set_bit(__IGC_DOWN, &adapter->state);
4228 del_timer_sync(&adapter->watchdog_timer);
4229 del_timer_sync(&adapter->phy_info_timer);
4231 cancel_work_sync(&adapter->reset_task);
4232 cancel_work_sync(&adapter->watchdog_task);
4234 /* Release control of h/w to f/w. If f/w is AMT enabled, this
4235 * would have already happened in close and is redundant.
4237 igc_release_hw_control(adapter);
4238 unregister_netdev(netdev);
4240 igc_clear_interrupt_scheme(adapter);
4241 pci_iounmap(pdev, adapter->io_addr);
4242 pci_release_mem_regions(pdev);
4244 kfree(adapter->mac_table);
4245 kfree(adapter->shadow_vfta);
4246 free_netdev(netdev);
4248 pci_disable_pcie_error_reporting(pdev);
4250 pci_disable_device(pdev);
4253 static struct pci_driver igc_driver = {
4254 .name = igc_driver_name,
4255 .id_table = igc_pci_tbl,
4257 .remove = igc_remove,
4260 void igc_set_flag_queue_pairs(struct igc_adapter *adapter,
4261 const u32 max_rss_queues)
4263 /* Determine if we need to pair queues. */
4264 /* If rss_queues > half of max_rss_queues, pair the queues in
4265 * order to conserve interrupts due to limited supply.
4267 if (adapter->rss_queues > (max_rss_queues / 2))
4268 adapter->flags |= IGC_FLAG_QUEUE_PAIRS;
4270 adapter->flags &= ~IGC_FLAG_QUEUE_PAIRS;
4273 unsigned int igc_get_max_rss_queues(struct igc_adapter *adapter)
4275 unsigned int max_rss_queues;
4277 /* Determine the maximum number of RSS queues supported. */
4278 max_rss_queues = IGC_MAX_RX_QUEUES;
4280 return max_rss_queues;
4283 static void igc_init_queue_configuration(struct igc_adapter *adapter)
4287 max_rss_queues = igc_get_max_rss_queues(adapter);
4288 adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus());
4290 igc_set_flag_queue_pairs(adapter, max_rss_queues);
4294 * igc_sw_init - Initialize general software structures (struct igc_adapter)
4295 * @adapter: board private structure to initialize
4297 * igc_sw_init initializes the Adapter private data structure.
4298 * Fields are initialized based on PCI device information and
4299 * OS network device settings (MTU size).
4301 static int igc_sw_init(struct igc_adapter *adapter)
4303 struct net_device *netdev = adapter->netdev;
4304 struct pci_dev *pdev = adapter->pdev;
4305 struct igc_hw *hw = &adapter->hw;
4307 int size = sizeof(struct igc_mac_addr) * hw->mac.rar_entry_count;
4309 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
4311 /* set default ring sizes */
4312 adapter->tx_ring_count = IGC_DEFAULT_TXD;
4313 adapter->rx_ring_count = IGC_DEFAULT_RXD;
4315 /* set default ITR values */
4316 adapter->rx_itr_setting = IGC_DEFAULT_ITR;
4317 adapter->tx_itr_setting = IGC_DEFAULT_ITR;
4319 /* set default work limits */
4320 adapter->tx_work_limit = IGC_DEFAULT_TX_WORK;
4322 /* adjust max frame to be at least the size of a standard frame */
4323 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
4325 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
4327 spin_lock_init(&adapter->nfc_lock);
4328 spin_lock_init(&adapter->stats64_lock);
4329 /* Assume MSI-X interrupts, will be checked during IRQ allocation */
4330 adapter->flags |= IGC_FLAG_HAS_MSIX;
4332 adapter->mac_table = kzalloc(size, GFP_ATOMIC);
4333 if (!adapter->mac_table)
4336 igc_init_queue_configuration(adapter);
4338 /* This call may decrease the number of queues */
4339 if (igc_init_interrupt_scheme(adapter, true)) {
4340 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
4344 /* Explicitly disable IRQ since the NIC can be in any state. */
4345 igc_irq_disable(adapter);
4347 set_bit(__IGC_DOWN, &adapter->state);
4353 * igc_reinit_queues - return error
4354 * @adapter: pointer to adapter structure
4356 int igc_reinit_queues(struct igc_adapter *adapter)
4358 struct net_device *netdev = adapter->netdev;
4359 struct pci_dev *pdev = adapter->pdev;
4362 if (netif_running(netdev))
4365 igc_reset_interrupt_capability(adapter);
4367 if (igc_init_interrupt_scheme(adapter, true)) {
4368 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
4372 if (netif_running(netdev))
4373 err = igc_open(netdev);
4379 * igc_get_hw_dev - return device
4380 * @hw: pointer to hardware structure
4382 * used by hardware layer to print debugging information
4384 struct net_device *igc_get_hw_dev(struct igc_hw *hw)
4386 struct igc_adapter *adapter = hw->back;
4388 return adapter->netdev;
4392 * igc_init_module - Driver Registration Routine
4394 * igc_init_module is the first routine called when the driver is
4395 * loaded. All it does is register with the PCI subsystem.
4397 static int __init igc_init_module(void)
4401 pr_info("%s - version %s\n",
4402 igc_driver_string, igc_driver_version);
4404 pr_info("%s\n", igc_copyright);
4406 ret = pci_register_driver(&igc_driver);
4410 module_init(igc_init_module);
4413 * igc_exit_module - Driver Exit Cleanup Routine
4415 * igc_exit_module is called just before the driver is removed
4418 static void __exit igc_exit_module(void)
4420 pci_unregister_driver(&igc_driver);
4423 module_exit(igc_exit_module);