drivers/net/ethernet/intel/iavf/iavf_main.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2013 - 2018 Intel Corporation. */
   3
   4 #include "iavf.h"
   5 #include "i40e_prototype.h"
   6 #include "iavf_client.h"
   7 /* All iavf tracepoints are defined by the include below, which must
   8  * be included exactly once across the whole kernel with
   9  * CREATE_TRACE_POINTS defined
  10  */
  11 #define CREATE_TRACE_POINTS
  12 #include "i40e_trace.h"
  13
  14 static int iavf_setup_all_tx_resources(struct iavf_adapter *adapter);
  15 static int iavf_setup_all_rx_resources(struct iavf_adapter *adapter);
  16 static int iavf_close(struct net_device *netdev);
  17
  18 char iavf_driver_name[] = "iavf";
  19 static const char iavf_driver_string[] =
  20         "Intel(R) Ethernet Adaptive Virtual Function Network Driver";
  21
  22 #define DRV_KERN "-k"
  23
  24 #define DRV_VERSION_MAJOR 3
  25 #define DRV_VERSION_MINOR 2
  26 #define DRV_VERSION_BUILD 3
  27 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \
  28              __stringify(DRV_VERSION_MINOR) "." \
  29              __stringify(DRV_VERSION_BUILD) \
  30              DRV_KERN
  31 const char iavf_driver_version[] = DRV_VERSION;
  32 static const char iavf_copyright[] =
  33         "Copyright (c) 2013 - 2018 Intel Corporation.";
  34
  35 /* iavf_pci_tbl - PCI Device ID Table
  36  *
  37  * Wildcard entries (PCI_ANY_ID) should come last
  38  * Last entry must be all 0s
  39  *
  40  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  41  *   Class, Class Mask, private data (not used) }
  42  */
  43 static const struct pci_device_id iavf_pci_tbl[] = {
  44         {PCI_VDEVICE(INTEL, I40E_DEV_ID_VF), 0},
  45         {PCI_VDEVICE(INTEL, I40E_DEV_ID_VF_HV), 0},
  46         {PCI_VDEVICE(INTEL, I40E_DEV_ID_X722_VF), 0},
  47         {PCI_VDEVICE(INTEL, I40E_DEV_ID_ADAPTIVE_VF), 0},
  48         /* required last entry */
  49         {0, }
  50 };
  51
  52 MODULE_DEVICE_TABLE(pci, iavf_pci_tbl);
  53
  54 MODULE_ALIAS("i40evf");
  55 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
  56 MODULE_DESCRIPTION("Intel(R) XL710 X710 Virtual Function Network Driver");
  57 MODULE_LICENSE("GPL");
  58 MODULE_VERSION(DRV_VERSION);
  59
  60 static struct workqueue_struct *iavf_wq;
  61
  62 /**
  63  * iavf_allocate_dma_mem_d - OS specific memory alloc for shared code
  64  * @hw:   pointer to the HW structure
  65  * @mem:  ptr to mem struct to fill out
  66  * @size: size of memory requested
  67  * @alignment: what to align the allocation to
  68  **/
  69 iavf_status iavf_allocate_dma_mem_d(struct i40e_hw *hw,
  70                                     struct i40e_dma_mem *mem,
  71                                     u64 size, u32 alignment)
  72 {
  73         struct iavf_adapter *adapter = (struct iavf_adapter *)hw->back;
  74
  75         if (!mem)
  76                 return I40E_ERR_PARAM;
  77
  78         mem->size = ALIGN(size, alignment);
  79         mem->va = dma_alloc_coherent(&adapter->pdev->dev, mem->size,
  80                                      (dma_addr_t *)&mem->pa, GFP_KERNEL);
  81         if (mem->va)
  82                 return 0;
  83         else
  84                 return I40E_ERR_NO_MEMORY;
  85 }
  86
  87 /**
  88  * iavf_free_dma_mem_d - OS specific memory free for shared code
  89  * @hw:   pointer to the HW structure
  90  * @mem:  ptr to mem struct to free
  91  **/
  92 iavf_status iavf_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem)
  93 {
  94         struct iavf_adapter *adapter = (struct iavf_adapter *)hw->back;
  95
  96         if (!mem || !mem->va)
  97                 return I40E_ERR_PARAM;
  98         dma_free_coherent(&adapter->pdev->dev, mem->size,
  99                           mem->va, (dma_addr_t)mem->pa);
 100         return 0;
 101 }
 102
 103 /**
 104  * iavf_allocate_virt_mem_d - OS specific memory alloc for shared code
 105  * @hw:   pointer to the HW structure
 106  * @mem:  ptr to mem struct to fill out
 107  * @size: size of memory requested
 108  **/
 109 iavf_status iavf_allocate_virt_mem_d(struct i40e_hw *hw,
 110                                      struct i40e_virt_mem *mem, u32 size)
 111 {
 112         if (!mem)
 113                 return I40E_ERR_PARAM;
 114
 115         mem->size = size;
 116         mem->va = kzalloc(size, GFP_KERNEL);
 117
 118         if (mem->va)
 119                 return 0;
 120         else
 121                 return I40E_ERR_NO_MEMORY;
 122 }
 123
 124 /**
 125  * iavf_free_virt_mem_d - OS specific memory free for shared code
 126  * @hw:   pointer to the HW structure
 127  * @mem:  ptr to mem struct to free
 128  **/
 129 iavf_status iavf_free_virt_mem_d(struct i40e_hw *hw,
 130                                  struct i40e_virt_mem *mem)
 131 {
 132         if (!mem)
 133                 return I40E_ERR_PARAM;
 134
 135         /* it's ok to kfree a NULL pointer */
 136         kfree(mem->va);
 137
 138         return 0;
 139 }
 140
 141 /**
 142  * iavf_debug_d - OS dependent version of debug printing
 143  * @hw:  pointer to the HW structure
 144  * @mask: debug level mask
 145  * @fmt_str: printf-type format description
 146  **/
 147 void iavf_debug_d(void *hw, u32 mask, char *fmt_str, ...)
 148 {
 149         char buf[512];
 150         va_list argptr;
 151
 152         if (!(mask & ((struct i40e_hw *)hw)->debug_mask))
 153                 return;
 154
 155         va_start(argptr, fmt_str);
 156         vsnprintf(buf, sizeof(buf), fmt_str, argptr);
 157         va_end(argptr);
 158
 159         /* the debug string is already formatted with a newline */
 160         pr_info("%s", buf);
 161 }
 162
 163 /**
 164  * iavf_schedule_reset - Set the flags and schedule a reset event
 165  * @adapter: board private structure
 166  **/
 167 void iavf_schedule_reset(struct iavf_adapter *adapter)
 168 {
 169         if (!(adapter->flags &
 170               (IAVF_FLAG_RESET_PENDING | IAVF_FLAG_RESET_NEEDED))) {
 171                 adapter->flags |= IAVF_FLAG_RESET_NEEDED;
 172                 schedule_work(&adapter->reset_task);
 173         }
 174 }
 175
 176 /**
 177  * iavf_tx_timeout - Respond to a Tx Hang
 178  * @netdev: network interface device structure
 179  **/
 180 static void iavf_tx_timeout(struct net_device *netdev)
 181 {
 182         struct iavf_adapter *adapter = netdev_priv(netdev);
 183
 184         adapter->tx_timeout_count++;
 185         iavf_schedule_reset(adapter);
 186 }
 187
 188 /**
 189  * iavf_misc_irq_disable - Mask off interrupt generation on the NIC
 190  * @adapter: board private structure
 191  **/
 192 static void iavf_misc_irq_disable(struct iavf_adapter *adapter)
 193 {
 194         struct i40e_hw *hw = &adapter->hw;
 195
 196         if (!adapter->msix_entries)
 197                 return;
 198
 199         wr32(hw, I40E_VFINT_DYN_CTL01, 0);
 200
 201         /* read flush */
 202         rd32(hw, I40E_VFGEN_RSTAT);
 203
 204         synchronize_irq(adapter->msix_entries[0].vector);
 205 }
 206
 207 /**
 208  * iavf_misc_irq_enable - Enable default interrupt generation settings
 209  * @adapter: board private structure
 210  **/
 211 static void iavf_misc_irq_enable(struct iavf_adapter *adapter)
 212 {
 213         struct i40e_hw *hw = &adapter->hw;
 214
 215         wr32(hw, I40E_VFINT_DYN_CTL01, I40E_VFINT_DYN_CTL01_INTENA_MASK |
 216                                        I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
 217         wr32(hw, I40E_VFINT_ICR0_ENA1, I40E_VFINT_ICR0_ENA1_ADMINQ_MASK);
 218
 219         /* read flush */
 220         rd32(hw, I40E_VFGEN_RSTAT);
 221 }
 222
 223 /**
 224  * iavf_irq_disable - Mask off interrupt generation on the NIC
 225  * @adapter: board private structure
 226  **/
 227 static void iavf_irq_disable(struct iavf_adapter *adapter)
 228 {
 229         int i;
 230         struct i40e_hw *hw = &adapter->hw;
 231
 232         if (!adapter->msix_entries)
 233                 return;
 234
 235         for (i = 1; i < adapter->num_msix_vectors; i++) {
 236                 wr32(hw, I40E_VFINT_DYN_CTLN1(i - 1), 0);
 237                 synchronize_irq(adapter->msix_entries[i].vector);
 238         }
 239         /* read flush */
 240         rd32(hw, I40E_VFGEN_RSTAT);
 241 }
 242
 243 /**
 244  * iavf_irq_enable_queues - Enable interrupt for specified queues
 245  * @adapter: board private structure
 246  * @mask: bitmap of queues to enable
 247  **/
 248 void iavf_irq_enable_queues(struct iavf_adapter *adapter, u32 mask)
 249 {
 250         struct i40e_hw *hw = &adapter->hw;
 251         int i;
 252
 253         for (i = 1; i < adapter->num_msix_vectors; i++) {
 254                 if (mask & BIT(i - 1)) {
 255                         wr32(hw, I40E_VFINT_DYN_CTLN1(i - 1),
 256                              I40E_VFINT_DYN_CTLN1_INTENA_MASK |
 257                              I40E_VFINT_DYN_CTLN1_ITR_INDX_MASK);
 258                 }
 259         }
 260 }
 261
 262 /**
 263  * iavf_irq_enable - Enable default interrupt generation settings
 264  * @adapter: board private structure
 265  * @flush: boolean value whether to run rd32()
 266  **/
 267 void iavf_irq_enable(struct iavf_adapter *adapter, bool flush)
 268 {
 269         struct i40e_hw *hw = &adapter->hw;
 270
 271         iavf_misc_irq_enable(adapter);
 272         iavf_irq_enable_queues(adapter, ~0);
 273
 274         if (flush)
 275                 rd32(hw, I40E_VFGEN_RSTAT);
 276 }
 277
 278 /**
 279  * iavf_msix_aq - Interrupt handler for vector 0
 280  * @irq: interrupt number
 281  * @data: pointer to netdev
 282  **/
 283 static irqreturn_t iavf_msix_aq(int irq, void *data)
 284 {
 285         struct net_device *netdev = data;
 286         struct iavf_adapter *adapter = netdev_priv(netdev);
 287         struct i40e_hw *hw = &adapter->hw;
 288
 289         /* handle non-queue interrupts, these reads clear the registers */
 290         rd32(hw, I40E_VFINT_ICR01);
 291         rd32(hw, I40E_VFINT_ICR0_ENA1);
 292
 293         /* schedule work on the private workqueue */
 294         schedule_work(&adapter->adminq_task);
 295
 296         return IRQ_HANDLED;
 297 }
 298
 299 /**
 300  * iavf_msix_clean_rings - MSIX mode Interrupt Handler
 301  * @irq: interrupt number
 302  * @data: pointer to a q_vector
 303  **/
 304 static irqreturn_t iavf_msix_clean_rings(int irq, void *data)
 305 {
 306         struct i40e_q_vector *q_vector = data;
 307
 308         if (!q_vector->tx.ring && !q_vector->rx.ring)
 309                 return IRQ_HANDLED;
 310
 311         napi_schedule_irqoff(&q_vector->napi);
 312
 313         return IRQ_HANDLED;
 314 }
 315
 316 /**
 317  * iavf_map_vector_to_rxq - associate irqs with rx queues
 318  * @adapter: board private structure
 319  * @v_idx: interrupt number
 320  * @r_idx: queue number
 321  **/
 322 static void
 323 iavf_map_vector_to_rxq(struct iavf_adapter *adapter, int v_idx, int r_idx)
 324 {
 325         struct i40e_q_vector *q_vector = &adapter->q_vectors[v_idx];
 326         struct i40e_ring *rx_ring = &adapter->rx_rings[r_idx];
 327         struct i40e_hw *hw = &adapter->hw;
 328
 329         rx_ring->q_vector = q_vector;
 330         rx_ring->next = q_vector->rx.ring;
 331         rx_ring->vsi = &adapter->vsi;
 332         q_vector->rx.ring = rx_ring;
 333         q_vector->rx.count++;
 334         q_vector->rx.next_update = jiffies + 1;
 335         q_vector->rx.target_itr = ITR_TO_REG(rx_ring->itr_setting);
 336         q_vector->ring_mask |= BIT(r_idx);
 337         wr32(hw, I40E_VFINT_ITRN1(I40E_RX_ITR, q_vector->reg_idx),
 338              q_vector->rx.current_itr);
 339         q_vector->rx.current_itr = q_vector->rx.target_itr;
 340 }
 341
 342 /**
 343  * iavf_map_vector_to_txq - associate irqs with tx queues
 344  * @adapter: board private structure
 345  * @v_idx: interrupt number
 346  * @t_idx: queue number
 347  **/
 348 static void
 349 iavf_map_vector_to_txq(struct iavf_adapter *adapter, int v_idx, int t_idx)
 350 {
 351         struct i40e_q_vector *q_vector = &adapter->q_vectors[v_idx];
 352         struct i40e_ring *tx_ring = &adapter->tx_rings[t_idx];
 353         struct i40e_hw *hw = &adapter->hw;
 354
 355         tx_ring->q_vector = q_vector;
 356         tx_ring->next = q_vector->tx.ring;
 357         tx_ring->vsi = &adapter->vsi;
 358         q_vector->tx.ring = tx_ring;
 359         q_vector->tx.count++;
 360         q_vector->tx.next_update = jiffies + 1;
 361         q_vector->tx.target_itr = ITR_TO_REG(tx_ring->itr_setting);
 362         q_vector->num_ringpairs++;
 363         wr32(hw, I40E_VFINT_ITRN1(I40E_TX_ITR, q_vector->reg_idx),
 364              q_vector->tx.target_itr);
 365         q_vector->tx.current_itr = q_vector->tx.target_itr;
 366 }
 367
 368 /**
 369  * iavf_map_rings_to_vectors - Maps descriptor rings to vectors
 370  * @adapter: board private structure to initialize
 371  *
 372  * This function maps descriptor rings to the queue-specific vectors
 373  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
 374  * one vector per ring/queue, but on a constrained vector budget, we
 375  * group the rings as "efficiently" as possible.  You would add new
 376  * mapping configurations in here.
 377  **/
 378 static void iavf_map_rings_to_vectors(struct iavf_adapter *adapter)
 379 {
 380         int rings_remaining = adapter->num_active_queues;
 381         int ridx = 0, vidx = 0;
 382         int q_vectors;
 383
 384         q_vectors = adapter->num_msix_vectors - NONQ_VECS;
 385
 386         for (; ridx < rings_remaining; ridx++) {
 387                 iavf_map_vector_to_rxq(adapter, vidx, ridx);
 388                 iavf_map_vector_to_txq(adapter, vidx, ridx);
 389
 390                 /* In the case where we have more queues than vectors, continue
 391                  * round-robin on vectors until all queues are mapped.
 392                  */
 393                 if (++vidx >= q_vectors)
 394                         vidx = 0;
 395         }
 396
 397         adapter->aq_required |= IAVF_FLAG_AQ_MAP_VECTORS;
 398 }
 399
 400 #ifdef CONFIG_NET_POLL_CONTROLLER
 401 /**
 402  * iavf_netpoll - A Polling 'interrupt' handler
 403  * @netdev: network interface device structure
 404  *
 405  * This is used by netconsole to send skbs without having to re-enable
 406  * interrupts.  It's not called while the normal interrupt routine is executing.
 407  **/
 408 static void iavf_netpoll(struct net_device *netdev)
 409 {
 410         struct iavf_adapter *adapter = netdev_priv(netdev);
 411         int q_vectors = adapter->num_msix_vectors - NONQ_VECS;
 412         int i;
 413
 414         /* if interface is down do nothing */
 415         if (test_bit(__I40E_VSI_DOWN, adapter->vsi.state))
 416                 return;
 417
 418         for (i = 0; i < q_vectors; i++)
 419                 iavf_msix_clean_rings(0, &adapter->q_vectors[i]);
 420 }
 421
 422 #endif
 423 /**
 424  * iavf_irq_affinity_notify - Callback for affinity changes
 425  * @notify: context as to what irq was changed
 426  * @mask: the new affinity mask
 427  *
 428  * This is a callback function used by the irq_set_affinity_notifier function
 429  * so that we may register to receive changes to the irq affinity masks.
 430  **/
 431 static void iavf_irq_affinity_notify(struct irq_affinity_notify *notify,
 432                                      const cpumask_t *mask)
 433 {
 434         struct i40e_q_vector *q_vector =
 435                 container_of(notify, struct i40e_q_vector, affinity_notify);
 436
 437         cpumask_copy(&q_vector->affinity_mask, mask);
 438 }
 439
 440 /**
 441  * iavf_irq_affinity_release - Callback for affinity notifier release
 442  * @ref: internal core kernel usage
 443  *
 444  * This is a callback function used by the irq_set_affinity_notifier function
 445  * to inform the current notification subscriber that they will no longer
 446  * receive notifications.
 447  **/
 448 static void iavf_irq_affinity_release(struct kref *ref) {}
 449
 450 /**
 451  * iavf_request_traffic_irqs - Initialize MSI-X interrupts
 452  * @adapter: board private structure
 453  * @basename: device basename
 454  *
 455  * Allocates MSI-X vectors for tx and rx handling, and requests
 456  * interrupts from the kernel.
 457  **/
 458 static int
 459 iavf_request_traffic_irqs(struct iavf_adapter *adapter, char *basename)
 460 {
 461         unsigned int vector, q_vectors;
 462         unsigned int rx_int_idx = 0, tx_int_idx = 0;
 463         int irq_num, err;
 464         int cpu;
 465
 466         iavf_irq_disable(adapter);
 467         /* Decrement for Other and TCP Timer vectors */
 468         q_vectors = adapter->num_msix_vectors - NONQ_VECS;
 469
 470         for (vector = 0; vector < q_vectors; vector++) {
 471                 struct i40e_q_vector *q_vector = &adapter->q_vectors[vector];
 472
 473                 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;
 474
 475                 if (q_vector->tx.ring && q_vector->rx.ring) {
 476                         snprintf(q_vector->name, sizeof(q_vector->name),
 477                                  "iavf-%s-TxRx-%d", basename, rx_int_idx++);
 478                         tx_int_idx++;
 479                 } else if (q_vector->rx.ring) {
 480                         snprintf(q_vector->name, sizeof(q_vector->name),
 481                                  "iavf-%s-rx-%d", basename, rx_int_idx++);
 482                 } else if (q_vector->tx.ring) {
 483                         snprintf(q_vector->name, sizeof(q_vector->name),
 484                                  "iavf-%s-tx-%d", basename, tx_int_idx++);
 485                 } else {
 486                         /* skip this unused q_vector */
 487                         continue;
 488                 }
 489                 err = request_irq(irq_num,
 490                                   iavf_msix_clean_rings,
 491                                   0,
 492                                   q_vector->name,
 493                                   q_vector);
 494                 if (err) {
 495                         dev_info(&adapter->pdev->dev,
 496                                  "Request_irq failed, error: %d\n", err);
 497                         goto free_queue_irqs;
 498                 }
 499                 /* register for affinity change notifications */
 500                 q_vector->affinity_notify.notify = iavf_irq_affinity_notify;
 501                 q_vector->affinity_notify.release =
 502                                                    iavf_irq_affinity_release;
 503                 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
 504                 /* Spread the IRQ affinity hints across online CPUs. Note that
 505                  * get_cpu_mask returns a mask with a permanent lifetime so
 506                  * it's safe to use as a hint for irq_set_affinity_hint.
 507                  */
 508                 cpu = cpumask_local_spread(q_vector->v_idx, -1);
 509                 irq_set_affinity_hint(irq_num, get_cpu_mask(cpu));
 510         }
 511
 512         return 0;
 513
 514 free_queue_irqs:
 515         while (vector) {
 516                 vector--;
 517                 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;
 518                 irq_set_affinity_notifier(irq_num, NULL);
 519                 irq_set_affinity_hint(irq_num, NULL);
 520                 free_irq(irq_num, &adapter->q_vectors[vector]);
 521         }
 522         return err;
 523 }
 524
 525 /**
 526  * iavf_request_misc_irq - Initialize MSI-X interrupts
 527  * @adapter: board private structure
 528  *
 529  * Allocates MSI-X vector 0 and requests interrupts from the kernel. This
 530  * vector is only for the admin queue, and stays active even when the netdev
 531  * is closed.
 532  **/
 533 static int iavf_request_misc_irq(struct iavf_adapter *adapter)
 534 {
 535         struct net_device *netdev = adapter->netdev;
 536         int err;
 537
 538         snprintf(adapter->misc_vector_name,
 539                  sizeof(adapter->misc_vector_name) - 1, "iavf-%s:mbx",
 540                  dev_name(&adapter->pdev->dev));
 541         err = request_irq(adapter->msix_entries[0].vector,
 542                           &iavf_msix_aq, 0,
 543                           adapter->misc_vector_name, netdev);
 544         if (err) {
 545                 dev_err(&adapter->pdev->dev,
 546                         "request_irq for %s failed: %d\n",
 547                         adapter->misc_vector_name, err);
 548                 free_irq(adapter->msix_entries[0].vector, netdev);
 549         }
 550         return err;
 551 }
 552
 553 /**
 554  * iavf_free_traffic_irqs - Free MSI-X interrupts
 555  * @adapter: board private structure
 556  *
 557  * Frees all MSI-X vectors other than 0.
 558  **/
 559 static void iavf_free_traffic_irqs(struct iavf_adapter *adapter)
 560 {
 561         int vector, irq_num, q_vectors;
 562
 563         if (!adapter->msix_entries)
 564                 return;
 565
 566         q_vectors = adapter->num_msix_vectors - NONQ_VECS;
 567
 568         for (vector = 0; vector < q_vectors; vector++) {
 569                 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;
 570                 irq_set_affinity_notifier(irq_num, NULL);
 571                 irq_set_affinity_hint(irq_num, NULL);
 572                 free_irq(irq_num, &adapter->q_vectors[vector]);
 573         }
 574 }
 575
 576 /**
 577  * iavf_free_misc_irq - Free MSI-X miscellaneous vector
 578  * @adapter: board private structure
 579  *
 580  * Frees MSI-X vector 0.
 581  **/
 582 static void iavf_free_misc_irq(struct iavf_adapter *adapter)
 583 {
 584         struct net_device *netdev = adapter->netdev;
 585
 586         if (!adapter->msix_entries)
 587                 return;
 588
 589         free_irq(adapter->msix_entries[0].vector, netdev);
 590 }
 591
 592 /**
 593  * iavf_configure_tx - Configure Transmit Unit after Reset
 594  * @adapter: board private structure
 595  *
 596  * Configure the Tx unit of the MAC after a reset.
 597  **/
 598 static void iavf_configure_tx(struct iavf_adapter *adapter)
 599 {
 600         struct i40e_hw *hw = &adapter->hw;
 601         int i;
 602
 603         for (i = 0; i < adapter->num_active_queues; i++)
 604                 adapter->tx_rings[i].tail = hw->hw_addr + I40E_QTX_TAIL1(i);
 605 }
 606
 607 /**
 608  * iavf_configure_rx - Configure Receive Unit after Reset
 609  * @adapter: board private structure
 610  *
 611  * Configure the Rx unit of the MAC after a reset.
 612  **/
 613 static void iavf_configure_rx(struct iavf_adapter *adapter)
 614 {
 615         unsigned int rx_buf_len = I40E_RXBUFFER_2048;
 616         struct i40e_hw *hw = &adapter->hw;
 617         int i;
 618
 619         /* Legacy Rx will always default to a 2048 buffer size. */
 620 #if (PAGE_SIZE < 8192)
 621         if (!(adapter->flags & IAVF_FLAG_LEGACY_RX)) {
 622                 struct net_device *netdev = adapter->netdev;
 623
 624                 /* For jumbo frames on systems with 4K pages we have to use
 625                  * an order 1 page, so we might as well increase the size
 626                  * of our Rx buffer to make better use of the available space
 627                  */
 628                 rx_buf_len = I40E_RXBUFFER_3072;
 629
 630                 /* We use a 1536 buffer size for configurations with
 631                  * standard Ethernet mtu.  On x86 this gives us enough room
 632                  * for shared info and 192 bytes of padding.
 633                  */
 634                 if (!I40E_2K_TOO_SMALL_WITH_PADDING &&
 635                     (netdev->mtu <= ETH_DATA_LEN))
 636                         rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
 637         }
 638 #endif
 639
 640         for (i = 0; i < adapter->num_active_queues; i++) {
 641                 adapter->rx_rings[i].tail = hw->hw_addr + I40E_QRX_TAIL1(i);
 642                 adapter->rx_rings[i].rx_buf_len = rx_buf_len;
 643
 644                 if (adapter->flags & IAVF_FLAG_LEGACY_RX)
 645                         clear_ring_build_skb_enabled(&adapter->rx_rings[i]);
 646                 else
 647                         set_ring_build_skb_enabled(&adapter->rx_rings[i]);
 648         }
 649 }
 650
 651 /**
 652  * iavf_find_vlan - Search filter list for specific vlan filter
 653  * @adapter: board private structure
 654  * @vlan: vlan tag
 655  *
 656  * Returns ptr to the filter object or NULL. Must be called while holding the
 657  * mac_vlan_list_lock.
 658  **/
 659 static struct
 660 iavf_vlan_filter *iavf_find_vlan(struct iavf_adapter *adapter, u16 vlan)
 661 {
 662         struct iavf_vlan_filter *f;
 663
 664         list_for_each_entry(f, &adapter->vlan_filter_list, list) {
 665                 if (vlan == f->vlan)
 666                         return f;
 667         }
 668         return NULL;
 669 }
 670
 671 /**
 672  * iavf_add_vlan - Add a vlan filter to the list
 673  * @adapter: board private structure
 674  * @vlan: VLAN tag
 675  *
 676  * Returns ptr to the filter object or NULL when no memory available.
 677  **/
 678 static struct
 679 iavf_vlan_filter *iavf_add_vlan(struct iavf_adapter *adapter, u16 vlan)
 680 {
 681         struct iavf_vlan_filter *f = NULL;
 682
 683         spin_lock_bh(&adapter->mac_vlan_list_lock);
 684
 685         f = iavf_find_vlan(adapter, vlan);
 686         if (!f) {
 687                 f = kzalloc(sizeof(*f), GFP_KERNEL);
 688                 if (!f)
 689                         goto clearout;
 690
 691                 f->vlan = vlan;
 692
 693                 INIT_LIST_HEAD(&f->list);
 694                 list_add(&f->list, &adapter->vlan_filter_list);
 695                 f->add = true;
 696                 adapter->aq_required |= IAVF_FLAG_AQ_ADD_VLAN_FILTER;
 697         }
 698
 699 clearout:
 700         spin_unlock_bh(&adapter->mac_vlan_list_lock);
 701         return f;
 702 }
 703
 704 /**
 705  * iavf_del_vlan - Remove a vlan filter from the list
 706  * @adapter: board private structure
 707  * @vlan: VLAN tag
 708  **/
 709 static void iavf_del_vlan(struct iavf_adapter *adapter, u16 vlan)
 710 {
 711         struct iavf_vlan_filter *f;
 712
 713         spin_lock_bh(&adapter->mac_vlan_list_lock);
 714
 715         f = iavf_find_vlan(adapter, vlan);
 716         if (f) {
 717                 f->remove = true;
 718                 adapter->aq_required |= IAVF_FLAG_AQ_DEL_VLAN_FILTER;
 719         }
 720
 721         spin_unlock_bh(&adapter->mac_vlan_list_lock);
 722 }
 723
 724 /**
 725  * iavf_vlan_rx_add_vid - Add a VLAN filter to a device
 726  * @netdev: network device struct
 727  * @proto: unused protocol data
 728  * @vid: VLAN tag
 729  **/
 730 static int iavf_vlan_rx_add_vid(struct net_device *netdev,
 731                                 __always_unused __be16 proto, u16 vid)
 732 {
 733         struct iavf_adapter *adapter = netdev_priv(netdev);
 734
 735         if (!VLAN_ALLOWED(adapter))
 736                 return -EIO;
 737         if (iavf_add_vlan(adapter, vid) == NULL)
 738                 return -ENOMEM;
 739         return 0;
 740 }
 741
 742 /**
 743  * iavf_vlan_rx_kill_vid - Remove a VLAN filter from a device
 744  * @netdev: network device struct
 745  * @proto: unused protocol data
 746  * @vid: VLAN tag
 747  **/
 748 static int iavf_vlan_rx_kill_vid(struct net_device *netdev,
 749                                  __always_unused __be16 proto, u16 vid)
 750 {
 751         struct iavf_adapter *adapter = netdev_priv(netdev);
 752
 753         if (VLAN_ALLOWED(adapter)) {
 754                 iavf_del_vlan(adapter, vid);
 755                 return 0;
 756         }
 757         return -EIO;
 758 }
 759
 760 /**
 761  * iavf_find_filter - Search filter list for specific mac filter
 762  * @adapter: board private structure
 763  * @macaddr: the MAC address
 764  *
 765  * Returns ptr to the filter object or NULL. Must be called while holding the
 766  * mac_vlan_list_lock.
 767  **/
 768 static struct
 769 iavf_mac_filter *iavf_find_filter(struct iavf_adapter *adapter,
 770                                   const u8 *macaddr)
 771 {
 772         struct iavf_mac_filter *f;
 773
 774         if (!macaddr)
 775                 return NULL;
 776
 777         list_for_each_entry(f, &adapter->mac_filter_list, list) {
 778                 if (ether_addr_equal(macaddr, f->macaddr))
 779                         return f;
 780         }
 781         return NULL;
 782 }
 783
 784 /**
 785  * i40e_add_filter - Add a mac filter to the filter list
 786  * @adapter: board private structure
 787  * @macaddr: the MAC address
 788  *
 789  * Returns ptr to the filter object or NULL when no memory available.
 790  **/
 791 static struct
 792 iavf_mac_filter *iavf_add_filter(struct iavf_adapter *adapter,
 793                                  const u8 *macaddr)
 794 {
 795         struct iavf_mac_filter *f;
 796
 797         if (!macaddr)
 798                 return NULL;
 799
 800         f = iavf_find_filter(adapter, macaddr);
 801         if (!f) {
 802                 f = kzalloc(sizeof(*f), GFP_ATOMIC);
 803                 if (!f)
 804                         return f;
 805
 806                 ether_addr_copy(f->macaddr, macaddr);
 807
 808                 list_add_tail(&f->list, &adapter->mac_filter_list);
 809                 f->add = true;
 810                 adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER;
 811         } else {
 812                 f->remove = false;
 813         }
 814
 815         return f;
 816 }
 817
 818 /**
 819  * iavf_set_mac - NDO callback to set port mac address
 820  * @netdev: network interface device structure
 821  * @p: pointer to an address structure
 822  *
 823  * Returns 0 on success, negative on failure
 824  **/
 825 static int iavf_set_mac(struct net_device *netdev, void *p)
 826 {
 827         struct iavf_adapter *adapter = netdev_priv(netdev);
 828         struct i40e_hw *hw = &adapter->hw;
 829         struct iavf_mac_filter *f;
 830         struct sockaddr *addr = p;
 831
 832         if (!is_valid_ether_addr(addr->sa_data))
 833                 return -EADDRNOTAVAIL;
 834
 835         if (ether_addr_equal(netdev->dev_addr, addr->sa_data))
 836                 return 0;
 837
 838         if (adapter->flags & IAVF_FLAG_ADDR_SET_BY_PF)
 839                 return -EPERM;
 840
 841         spin_lock_bh(&adapter->mac_vlan_list_lock);
 842
 843         f = iavf_find_filter(adapter, hw->mac.addr);
 844         if (f) {
 845                 f->remove = true;
 846                 adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER;
 847         }
 848
 849         f = iavf_add_filter(adapter, addr->sa_data);
 850
 851         spin_unlock_bh(&adapter->mac_vlan_list_lock);
 852
 853         if (f) {
 854                 ether_addr_copy(hw->mac.addr, addr->sa_data);
 855                 ether_addr_copy(netdev->dev_addr, adapter->hw.mac.addr);
 856         }
 857
 858         return (f == NULL) ? -ENOMEM : 0;
 859 }
 860
 861 /**
 862  * iavf_addr_sync - Callback for dev_(mc|uc)_sync to add address
 863  * @netdev: the netdevice
 864  * @addr: address to add
 865  *
 866  * Called by __dev_(mc|uc)_sync when an address needs to be added. We call
 867  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
 868  */
 869 static int iavf_addr_sync(struct net_device *netdev, const u8 *addr)
 870 {
 871         struct iavf_adapter *adapter = netdev_priv(netdev);
 872
 873         if (iavf_add_filter(adapter, addr))
 874                 return 0;
 875         else
 876                 return -ENOMEM;
 877 }
 878
 879 /**
 880  * iavf_addr_unsync - Callback for dev_(mc|uc)_sync to remove address
 881  * @netdev: the netdevice
 882  * @addr: address to add
 883  *
 884  * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call
 885  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
 886  */
 887 static int iavf_addr_unsync(struct net_device *netdev, const u8 *addr)
 888 {
 889         struct iavf_adapter *adapter = netdev_priv(netdev);
 890         struct iavf_mac_filter *f;
 891
 892         /* Under some circumstances, we might receive a request to delete
 893          * our own device address from our uc list. Because we store the
 894          * device address in the VSI's MAC/VLAN filter list, we need to ignore
 895          * such requests and not delete our device address from this list.
 896          */
 897         if (ether_addr_equal(addr, netdev->dev_addr))
 898                 return 0;
 899
 900         f = iavf_find_filter(adapter, addr);
 901         if (f) {
 902                 f->remove = true;
 903                 adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER;
 904         }
 905         return 0;
 906 }
 907
 908 /**
 909  * iavf_set_rx_mode - NDO callback to set the netdev filters
 910  * @netdev: network interface device structure
 911  **/
 912 static void iavf_set_rx_mode(struct net_device *netdev)
 913 {
 914         struct iavf_adapter *adapter = netdev_priv(netdev);
 915
 916         spin_lock_bh(&adapter->mac_vlan_list_lock);
 917         __dev_uc_sync(netdev, iavf_addr_sync, iavf_addr_unsync);
 918         __dev_mc_sync(netdev, iavf_addr_sync, iavf_addr_unsync);
 919         spin_unlock_bh(&adapter->mac_vlan_list_lock);
 920
 921         if (netdev->flags & IFF_PROMISC &&
 922             !(adapter->flags & IAVF_FLAG_PROMISC_ON))
 923                 adapter->aq_required |= IAVF_FLAG_AQ_REQUEST_PROMISC;
 924         else if (!(netdev->flags & IFF_PROMISC) &&
 925                  adapter->flags & IAVF_FLAG_PROMISC_ON)
 926                 adapter->aq_required |= IAVF_FLAG_AQ_RELEASE_PROMISC;
 927
 928         if (netdev->flags & IFF_ALLMULTI &&
 929             !(adapter->flags & IAVF_FLAG_ALLMULTI_ON))
 930                 adapter->aq_required |= IAVF_FLAG_AQ_REQUEST_ALLMULTI;
 931         else if (!(netdev->flags & IFF_ALLMULTI) &&
 932                  adapter->flags & IAVF_FLAG_ALLMULTI_ON)
 933                 adapter->aq_required |= IAVF_FLAG_AQ_RELEASE_ALLMULTI;
 934 }
 935
 936 /**
 937  * iavf_napi_enable_all - enable NAPI on all queue vectors
 938  * @adapter: board private structure
 939  **/
 940 static void iavf_napi_enable_all(struct iavf_adapter *adapter)
 941 {
 942         int q_idx;
 943         struct i40e_q_vector *q_vector;
 944         int q_vectors = adapter->num_msix_vectors - NONQ_VECS;
 945
 946         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
 947                 struct napi_struct *napi;
 948
 949                 q_vector = &adapter->q_vectors[q_idx];
 950                 napi = &q_vector->napi;
 951                 napi_enable(napi);
 952         }
 953 }
 954
 955 /**
 956  * iavf_napi_disable_all - disable NAPI on all queue vectors
 957  * @adapter: board private structure
 958  **/
 959 static void iavf_napi_disable_all(struct iavf_adapter *adapter)
 960 {
 961         int q_idx;
 962         struct i40e_q_vector *q_vector;
 963         int q_vectors = adapter->num_msix_vectors - NONQ_VECS;
 964
 965         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
 966                 q_vector = &adapter->q_vectors[q_idx];
 967                 napi_disable(&q_vector->napi);
 968         }
 969 }
 970
 971 /**
 972  * iavf_configure - set up transmit and receive data structures
 973  * @adapter: board private structure
 974  **/
 975 static void iavf_configure(struct iavf_adapter *adapter)
 976 {
 977         struct net_device *netdev = adapter->netdev;
 978         int i;
 979
 980         iavf_set_rx_mode(netdev);
 981
 982         iavf_configure_tx(adapter);
 983         iavf_configure_rx(adapter);
 984         adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_QUEUES;
 985
 986         for (i = 0; i < adapter->num_active_queues; i++) {
 987                 struct i40e_ring *ring = &adapter->rx_rings[i];
 988
 989                 iavf_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring));
 990         }
 991 }
 992
 993 /**
 994  * iavf_up_complete - Finish the last steps of bringing up a connection
 995  * @adapter: board private structure
 996  *
 997  * Expects to be called while holding the __IAVF_IN_CRITICAL_TASK bit lock.
 998  **/
 999 static void iavf_up_complete(struct iavf_adapter *adapter)
1000 {
1001         adapter->state = __IAVF_RUNNING;
1002         clear_bit(__I40E_VSI_DOWN, adapter->vsi.state);
1003
1004         iavf_napi_enable_all(adapter);
1005
1006         adapter->aq_required |= IAVF_FLAG_AQ_ENABLE_QUEUES;
1007         if (CLIENT_ENABLED(adapter))
1008                 adapter->flags |= IAVF_FLAG_CLIENT_NEEDS_OPEN;
1009         mod_timer_pending(&adapter->watchdog_timer, jiffies + 1);
1010 }
1011
1012 /**
1013  * i40e_down - Shutdown the connection processing
1014  * @adapter: board private structure
1015  *
1016  * Expects to be called while holding the __IAVF_IN_CRITICAL_TASK bit lock.
1017  **/
1018 void iavf_down(struct iavf_adapter *adapter)
1019 {
1020         struct net_device *netdev = adapter->netdev;
1021         struct iavf_vlan_filter *vlf;
1022         struct iavf_mac_filter *f;
1023         struct iavf_cloud_filter *cf;
1024
1025         if (adapter->state <= __IAVF_DOWN_PENDING)
1026                 return;
1027
1028         netif_carrier_off(netdev);
1029         netif_tx_disable(netdev);
1030         adapter->link_up = false;
1031         iavf_napi_disable_all(adapter);
1032         iavf_irq_disable(adapter);
1033
1034         spin_lock_bh(&adapter->mac_vlan_list_lock);
1035
1036         /* clear the sync flag on all filters */
1037         __dev_uc_unsync(adapter->netdev, NULL);
1038         __dev_mc_unsync(adapter->netdev, NULL);
1039
1040         /* remove all MAC filters */
1041         list_for_each_entry(f, &adapter->mac_filter_list, list) {
1042                 f->remove = true;
1043         }
1044
1045         /* remove all VLAN filters */
1046         list_for_each_entry(vlf, &adapter->vlan_filter_list, list) {
1047                 vlf->remove = true;
1048         }
1049
1050         spin_unlock_bh(&adapter->mac_vlan_list_lock);
1051
1052         /* remove all cloud filters */
1053         spin_lock_bh(&adapter->cloud_filter_list_lock);
1054         list_for_each_entry(cf, &adapter->cloud_filter_list, list) {
1055                 cf->del = true;
1056         }
1057         spin_unlock_bh(&adapter->cloud_filter_list_lock);
1058
1059         if (!(adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) &&
1060             adapter->state != __IAVF_RESETTING) {
1061                 /* cancel any current operation */
1062                 adapter->current_op = VIRTCHNL_OP_UNKNOWN;
1063                 /* Schedule operations to close down the HW. Don't wait
1064                  * here for this to complete. The watchdog is still running
1065                  * and it will take care of this.
1066                  */
1067                 adapter->aq_required = IAVF_FLAG_AQ_DEL_MAC_FILTER;
1068                 adapter->aq_required |= IAVF_FLAG_AQ_DEL_VLAN_FILTER;
1069                 adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER;
1070                 adapter->aq_required |= IAVF_FLAG_AQ_DISABLE_QUEUES;
1071         }
1072
1073         mod_timer_pending(&adapter->watchdog_timer, jiffies + 1);
1074 }
1075
1076 /**
1077  * iavf_acquire_msix_vectors - Setup the MSIX capability
1078  * @adapter: board private structure
1079  * @vectors: number of vectors to request
1080  *
1081  * Work with the OS to set up the MSIX vectors needed.
1082  *
1083  * Returns 0 on success, negative on failure
1084  **/
1085 static int
1086 iavf_acquire_msix_vectors(struct iavf_adapter *adapter, int vectors)
1087 {
1088         int err, vector_threshold;
1089
1090         /* We'll want at least 3 (vector_threshold):
1091          * 0) Other (Admin Queue and link, mostly)
1092          * 1) TxQ[0] Cleanup
1093          * 2) RxQ[0] Cleanup
1094          */
1095         vector_threshold = MIN_MSIX_COUNT;
1096
1097         /* The more we get, the more we will assign to Tx/Rx Cleanup
1098          * for the separate queues...where Rx Cleanup >= Tx Cleanup.
1099          * Right now, we simply care about how many we'll get; we'll
1100          * set them up later while requesting irq's.
1101          */
1102         err = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
1103                                     vector_threshold, vectors);
1104         if (err < 0) {
1105                 dev_err(&adapter->pdev->dev, "Unable to allocate MSI-X interrupts\n");
1106                 kfree(adapter->msix_entries);
1107                 adapter->msix_entries = NULL;
1108                 return err;
1109         }
1110
1111         /* Adjust for only the vectors we'll use, which is minimum
1112          * of max_msix_q_vectors + NONQ_VECS, or the number of
1113          * vectors we were allocated.
1114          */
1115         adapter->num_msix_vectors = err;
1116         return 0;
1117 }
1118
1119 /**
1120  * iavf_free_queues - Free memory for all rings
1121  * @adapter: board private structure to initialize
1122  *
1123  * Free all of the memory associated with queue pairs.
1124  **/
1125 static void iavf_free_queues(struct iavf_adapter *adapter)
1126 {
1127         if (!adapter->vsi_res)
1128                 return;
1129         adapter->num_active_queues = 0;
1130         kfree(adapter->tx_rings);
1131         adapter->tx_rings = NULL;
1132         kfree(adapter->rx_rings);
1133         adapter->rx_rings = NULL;
1134 }
1135
1136 /**
1137  * iavf_alloc_queues - Allocate memory for all rings
1138  * @adapter: board private structure to initialize
1139  *
1140  * We allocate one ring per queue at run-time since we don't know the
1141  * number of queues at compile-time.  The polling_netdev array is
1142  * intended for Multiqueue, but should work fine with a single queue.
1143  **/
1144 static int iavf_alloc_queues(struct iavf_adapter *adapter)
1145 {
1146         int i, num_active_queues;
1147
1148         /* If we're in reset reallocating queues we don't actually know yet for
1149          * certain the PF gave us the number of queues we asked for but we'll
1150          * assume it did.  Once basic reset is finished we'll confirm once we
1151          * start negotiating config with PF.
1152          */
1153         if (adapter->num_req_queues)
1154                 num_active_queues = adapter->num_req_queues;
1155         else if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&
1156                  adapter->num_tc)
1157                 num_active_queues = adapter->ch_config.total_qps;
1158         else
1159                 num_active_queues = min_t(int,
1160                                           adapter->vsi_res->num_queue_pairs,
1161                                           (int)(num_online_cpus()));
1162
1163
1164         adapter->tx_rings = kcalloc(num_active_queues,
1165                                     sizeof(struct i40e_ring), GFP_KERNEL);
1166         if (!adapter->tx_rings)
1167                 goto err_out;
1168         adapter->rx_rings = kcalloc(num_active_queues,
1169                                     sizeof(struct i40e_ring), GFP_KERNEL);
1170         if (!adapter->rx_rings)
1171                 goto err_out;
1172
1173         for (i = 0; i < num_active_queues; i++) {
1174                 struct i40e_ring *tx_ring;
1175                 struct i40e_ring *rx_ring;
1176
1177                 tx_ring = &adapter->tx_rings[i];
1178
1179                 tx_ring->queue_index = i;
1180                 tx_ring->netdev = adapter->netdev;
1181                 tx_ring->dev = &adapter->pdev->dev;
1182                 tx_ring->count = adapter->tx_desc_count;
1183                 tx_ring->itr_setting = I40E_ITR_TX_DEF;
1184                 if (adapter->flags & IAVF_FLAG_WB_ON_ITR_CAPABLE)
1185                         tx_ring->flags |= I40E_TXR_FLAGS_WB_ON_ITR;
1186
1187                 rx_ring = &adapter->rx_rings[i];
1188                 rx_ring->queue_index = i;
1189                 rx_ring->netdev = adapter->netdev;
1190                 rx_ring->dev = &adapter->pdev->dev;
1191                 rx_ring->count = adapter->rx_desc_count;
1192                 rx_ring->itr_setting = I40E_ITR_RX_DEF;
1193         }
1194
1195         adapter->num_active_queues = num_active_queues;
1196
1197         return 0;
1198
1199 err_out:
1200         iavf_free_queues(adapter);
1201         return -ENOMEM;
1202 }
1203
1204 /**
1205  * iavf_set_interrupt_capability - set MSI-X or FAIL if not supported
1206  * @adapter: board private structure to initialize
1207  *
1208  * Attempt to configure the interrupts using the best available
1209  * capabilities of the hardware and the kernel.
1210  **/
1211 static int iavf_set_interrupt_capability(struct iavf_adapter *adapter)
1212 {
1213         int vector, v_budget;
1214         int pairs = 0;
1215         int err = 0;
1216
1217         if (!adapter->vsi_res) {
1218                 err = -EIO;
1219                 goto out;
1220         }
1221         pairs = adapter->num_active_queues;
1222
1223         /* It's easy to be greedy for MSI-X vectors, but it really doesn't do
1224          * us much good if we have more vectors than CPUs. However, we already
1225          * limit the total number of queues by the number of CPUs so we do not
1226          * need any further limiting here.
1227          */
1228         v_budget = min_t(int, pairs + NONQ_VECS,
1229                          (int)adapter->vf_res->max_vectors);
1230
1231         adapter->msix_entries = kcalloc(v_budget,
1232                                         sizeof(struct msix_entry), GFP_KERNEL);
1233         if (!adapter->msix_entries) {
1234                 err = -ENOMEM;
1235                 goto out;
1236         }
1237
1238         for (vector = 0; vector < v_budget; vector++)
1239                 adapter->msix_entries[vector].entry = vector;
1240
1241         err = iavf_acquire_msix_vectors(adapter, v_budget);
1242
1243 out:
1244         netif_set_real_num_rx_queues(adapter->netdev, pairs);
1245         netif_set_real_num_tx_queues(adapter->netdev, pairs);
1246         return err;
1247 }
1248
1249 /**
1250  * i40e_config_rss_aq - Configure RSS keys and lut by using AQ commands
1251  * @adapter: board private structure
1252  *
1253  * Return 0 on success, negative on failure
1254  **/
1255 static int iavf_config_rss_aq(struct iavf_adapter *adapter)
1256 {
1257         struct i40e_aqc_get_set_rss_key_data *rss_key =
1258                 (struct i40e_aqc_get_set_rss_key_data *)adapter->rss_key;
1259         struct i40e_hw *hw = &adapter->hw;
1260         int ret = 0;
1261
1262         if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
1263                 /* bail because we already have a command pending */
1264                 dev_err(&adapter->pdev->dev, "Cannot configure RSS, command %d pending\n",
1265                         adapter->current_op);
1266                 return -EBUSY;
1267         }
1268
1269         ret = iavf_aq_set_rss_key(hw, adapter->vsi.id, rss_key);
1270         if (ret) {
1271                 dev_err(&adapter->pdev->dev, "Cannot set RSS key, err %s aq_err %s\n",
1272                         iavf_stat_str(hw, ret),
1273                         iavf_aq_str(hw, hw->aq.asq_last_status));
1274                 return ret;
1275
1276         }
1277
1278         ret = iavf_aq_set_rss_lut(hw, adapter->vsi.id, false,
1279                                   adapter->rss_lut, adapter->rss_lut_size);
1280         if (ret) {
1281                 dev_err(&adapter->pdev->dev, "Cannot set RSS lut, err %s aq_err %s\n",
1282                         iavf_stat_str(hw, ret),
1283                         iavf_aq_str(hw, hw->aq.asq_last_status));
1284         }
1285
1286         return ret;
1287
1288 }
1289
1290 /**
1291  * iavf_config_rss_reg - Configure RSS keys and lut by writing registers
1292  * @adapter: board private structure
1293  *
1294  * Returns 0 on success, negative on failure
1295  **/
1296 static int iavf_config_rss_reg(struct iavf_adapter *adapter)
1297 {
1298         struct i40e_hw *hw = &adapter->hw;
1299         u32 *dw;
1300         u16 i;
1301
1302         dw = (u32 *)adapter->rss_key;
1303         for (i = 0; i <= adapter->rss_key_size / 4; i++)
1304                 wr32(hw, I40E_VFQF_HKEY(i), dw[i]);
1305
1306         dw = (u32 *)adapter->rss_lut;
1307         for (i = 0; i <= adapter->rss_lut_size / 4; i++)
1308                 wr32(hw, I40E_VFQF_HLUT(i), dw[i]);
1309
1310         i40e_flush(hw);
1311
1312         return 0;
1313 }
1314
1315 /**
1316  * iavf_config_rss - Configure RSS keys and lut
1317  * @adapter: board private structure
1318  *
1319  * Returns 0 on success, negative on failure
1320  **/
1321 int iavf_config_rss(struct iavf_adapter *adapter)
1322 {
1323
1324         if (RSS_PF(adapter)) {
1325                 adapter->aq_required |= IAVF_FLAG_AQ_SET_RSS_LUT |
1326                                         IAVF_FLAG_AQ_SET_RSS_KEY;
1327                 return 0;
1328         } else if (RSS_AQ(adapter)) {
1329                 return iavf_config_rss_aq(adapter);
1330         } else {
1331                 return iavf_config_rss_reg(adapter);
1332         }
1333 }
1334
1335 /**
1336  * iavf_fill_rss_lut - Fill the lut with default values
1337  * @adapter: board private structure
1338  **/
1339 static void iavf_fill_rss_lut(struct iavf_adapter *adapter)
1340 {
1341         u16 i;
1342
1343         for (i = 0; i < adapter->rss_lut_size; i++)
1344                 adapter->rss_lut[i] = i % adapter->num_active_queues;
1345 }
1346
1347 /**
1348  * iavf_init_rss - Prepare for RSS
1349  * @adapter: board private structure
1350  *
1351  * Return 0 on success, negative on failure
1352  **/
1353 static int iavf_init_rss(struct iavf_adapter *adapter)
1354 {
1355         struct i40e_hw *hw = &adapter->hw;
1356         int ret;
1357
1358         if (!RSS_PF(adapter)) {
1359                 /* Enable PCTYPES for RSS, TCP/UDP with IPv4/IPv6 */
1360                 if (adapter->vf_res->vf_cap_flags &
1361                     VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2)
1362                         adapter->hena = I40E_DEFAULT_RSS_HENA_EXPANDED;
1363                 else
1364                         adapter->hena = I40E_DEFAULT_RSS_HENA;
1365
1366                 wr32(hw, I40E_VFQF_HENA(0), (u32)adapter->hena);
1367                 wr32(hw, I40E_VFQF_HENA(1), (u32)(adapter->hena >> 32));
1368         }
1369
1370         iavf_fill_rss_lut(adapter);
1371
1372         netdev_rss_key_fill((void *)adapter->rss_key, adapter->rss_key_size);
1373         ret = iavf_config_rss(adapter);
1374
1375         return ret;
1376 }
1377
1378 /**
1379  * iavf_alloc_q_vectors - Allocate memory for interrupt vectors
1380  * @adapter: board private structure to initialize
1381  *
1382  * We allocate one q_vector per queue interrupt.  If allocation fails we
1383  * return -ENOMEM.
1384  **/
1385 static int iavf_alloc_q_vectors(struct iavf_adapter *adapter)
1386 {
1387         int q_idx = 0, num_q_vectors;
1388         struct i40e_q_vector *q_vector;
1389
1390         num_q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1391         adapter->q_vectors = kcalloc(num_q_vectors, sizeof(*q_vector),
1392                                      GFP_KERNEL);
1393         if (!adapter->q_vectors)
1394                 return -ENOMEM;
1395
1396         for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1397                 q_vector = &adapter->q_vectors[q_idx];
1398                 q_vector->adapter = adapter;
1399                 q_vector->vsi = &adapter->vsi;
1400                 q_vector->v_idx = q_idx;
1401                 q_vector->reg_idx = q_idx;
1402                 cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);
1403                 netif_napi_add(adapter->netdev, &q_vector->napi,
1404                                iavf_napi_poll, NAPI_POLL_WEIGHT);
1405         }
1406
1407         return 0;
1408 }
1409
1410 /**
1411  * iavf_free_q_vectors - Free memory allocated for interrupt vectors
1412  * @adapter: board private structure to initialize
1413  *
1414  * This function frees the memory allocated to the q_vectors.  In addition if
1415  * NAPI is enabled it will delete any references to the NAPI struct prior
1416  * to freeing the q_vector.
1417  **/
1418 static void iavf_free_q_vectors(struct iavf_adapter *adapter)
1419 {
1420         int q_idx, num_q_vectors;
1421         int napi_vectors;
1422
1423         if (!adapter->q_vectors)
1424                 return;
1425
1426         num_q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1427         napi_vectors = adapter->num_active_queues;
1428
1429         for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1430                 struct i40e_q_vector *q_vector = &adapter->q_vectors[q_idx];
1431
1432                 if (q_idx < napi_vectors)
1433                         netif_napi_del(&q_vector->napi);
1434         }
1435         kfree(adapter->q_vectors);
1436         adapter->q_vectors = NULL;
1437 }
1438
1439 /**
1440  * iavf_reset_interrupt_capability - Reset MSIX setup
1441  * @adapter: board private structure
1442  *
1443  **/
1444 void iavf_reset_interrupt_capability(struct iavf_adapter *adapter)
1445 {
1446         if (!adapter->msix_entries)
1447                 return;
1448
1449         pci_disable_msix(adapter->pdev);
1450         kfree(adapter->msix_entries);
1451         adapter->msix_entries = NULL;
1452 }
1453
1454 /**
1455  * iavf_init_interrupt_scheme - Determine if MSIX is supported and init
1456  * @adapter: board private structure to initialize
1457  *
1458  **/
1459 int iavf_init_interrupt_scheme(struct iavf_adapter *adapter)
1460 {
1461         int err;
1462
1463         err = iavf_alloc_queues(adapter);
1464         if (err) {
1465                 dev_err(&adapter->pdev->dev,
1466                         "Unable to allocate memory for queues\n");
1467                 goto err_alloc_queues;
1468         }
1469
1470         rtnl_lock();
1471         err = iavf_set_interrupt_capability(adapter);
1472         rtnl_unlock();
1473         if (err) {
1474                 dev_err(&adapter->pdev->dev,
1475                         "Unable to setup interrupt capabilities\n");
1476                 goto err_set_interrupt;
1477         }
1478
1479         err = iavf_alloc_q_vectors(adapter);
1480         if (err) {
1481                 dev_err(&adapter->pdev->dev,
1482                         "Unable to allocate memory for queue vectors\n");
1483                 goto err_alloc_q_vectors;
1484         }
1485
1486         /* If we've made it so far while ADq flag being ON, then we haven't
1487          * bailed out anywhere in middle. And ADq isn't just enabled but actual
1488          * resources have been allocated in the reset path.
1489          * Now we can truly claim that ADq is enabled.
1490          */
1491         if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&
1492             adapter->num_tc)
1493                 dev_info(&adapter->pdev->dev, "ADq Enabled, %u TCs created",
1494                          adapter->num_tc);
1495
1496         dev_info(&adapter->pdev->dev, "Multiqueue %s: Queue pair count = %u",
1497                  (adapter->num_active_queues > 1) ? "Enabled" : "Disabled",
1498                  adapter->num_active_queues);
1499
1500         return 0;
1501 err_alloc_q_vectors:
1502         iavf_reset_interrupt_capability(adapter);
1503 err_set_interrupt:
1504         iavf_free_queues(adapter);
1505 err_alloc_queues:
1506         return err;
1507 }
1508
1509 /**
1510  * iavf_free_rss - Free memory used by RSS structs
1511  * @adapter: board private structure
1512  **/
1513 static void iavf_free_rss(struct iavf_adapter *adapter)
1514 {
1515         kfree(adapter->rss_key);
1516         adapter->rss_key = NULL;
1517
1518         kfree(adapter->rss_lut);
1519         adapter->rss_lut = NULL;
1520 }
1521
1522 /**
1523  * iavf_reinit_interrupt_scheme - Reallocate queues and vectors
1524  * @adapter: board private structure
1525  *
1526  * Returns 0 on success, negative on failure
1527  **/
1528 static int iavf_reinit_interrupt_scheme(struct iavf_adapter *adapter)
1529 {
1530         struct net_device *netdev = adapter->netdev;
1531         int err;
1532
1533         if (netif_running(netdev))
1534                 iavf_free_traffic_irqs(adapter);
1535         iavf_free_misc_irq(adapter);
1536         iavf_reset_interrupt_capability(adapter);
1537         iavf_free_q_vectors(adapter);
1538         iavf_free_queues(adapter);
1539
1540         err =  iavf_init_interrupt_scheme(adapter);
1541         if (err)
1542                 goto err;
1543
1544         netif_tx_stop_all_queues(netdev);
1545
1546         err = iavf_request_misc_irq(adapter);
1547         if (err)
1548                 goto err;
1549
1550         set_bit(__I40E_VSI_DOWN, adapter->vsi.state);
1551
1552         iavf_map_rings_to_vectors(adapter);
1553
1554         if (RSS_AQ(adapter))
1555                 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_RSS;
1556         else
1557                 err = iavf_init_rss(adapter);
1558 err:
1559         return err;
1560 }
1561
1562 /**
1563  * iavf_watchdog_timer - Periodic call-back timer
1564  * @data: pointer to adapter disguised as unsigned long
1565  **/
1566 static void iavf_watchdog_timer(struct timer_list *t)
1567 {
1568         struct iavf_adapter *adapter = from_timer(adapter, t,
1569                                                     watchdog_timer);
1570
1571         schedule_work(&adapter->watchdog_task);
1572         /* timer will be rescheduled in watchdog task */
1573 }
1574
1575 /**
1576  * iavf_watchdog_task - Periodic call-back task
1577  * @work: pointer to work_struct
1578  **/
1579 static void iavf_watchdog_task(struct work_struct *work)
1580 {
1581         struct iavf_adapter *adapter = container_of(work,
1582                                                       struct iavf_adapter,
1583                                                       watchdog_task);
1584         struct i40e_hw *hw = &adapter->hw;
1585         u32 reg_val;
1586
1587         if (test_and_set_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section))
1588                 goto restart_watchdog;
1589
1590         if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) {
1591                 reg_val = rd32(hw, I40E_VFGEN_RSTAT) &
1592                           I40E_VFGEN_RSTAT_VFR_STATE_MASK;
1593                 if ((reg_val == VIRTCHNL_VFR_VFACTIVE) ||
1594                     (reg_val == VIRTCHNL_VFR_COMPLETED)) {
1595                         /* A chance for redemption! */
1596                         dev_err(&adapter->pdev->dev, "Hardware came out of reset. Attempting reinit.\n");
1597                         adapter->state = __IAVF_STARTUP;
1598                         adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED;
1599                         schedule_delayed_work(&adapter->init_task, 10);
1600                         clear_bit(__IAVF_IN_CRITICAL_TASK,
1601                                   &adapter->crit_section);
1602                         /* Don't reschedule the watchdog, since we've restarted
1603                          * the init task. When init_task contacts the PF and
1604                          * gets everything set up again, it'll restart the
1605                          * watchdog for us. Down, boy. Sit. Stay. Woof.
1606                          */
1607                         return;
1608                 }
1609                 adapter->aq_required = 0;
1610                 adapter->current_op = VIRTCHNL_OP_UNKNOWN;
1611                 goto watchdog_done;
1612         }
1613
1614         if ((adapter->state < __IAVF_DOWN) ||
1615             (adapter->flags & IAVF_FLAG_RESET_PENDING))
1616                 goto watchdog_done;
1617
1618         /* check for reset */
1619         reg_val = rd32(hw, I40E_VF_ARQLEN1) & I40E_VF_ARQLEN1_ARQENABLE_MASK;
1620         if (!(adapter->flags & IAVF_FLAG_RESET_PENDING) && !reg_val) {
1621                 adapter->state = __IAVF_RESETTING;
1622                 adapter->flags |= IAVF_FLAG_RESET_PENDING;
1623                 dev_err(&adapter->pdev->dev, "Hardware reset detected\n");
1624                 schedule_work(&adapter->reset_task);
1625                 adapter->aq_required = 0;
1626                 adapter->current_op = VIRTCHNL_OP_UNKNOWN;
1627                 goto watchdog_done;
1628         }
1629
1630         /* Process admin queue tasks. After init, everything gets done
1631          * here so we don't race on the admin queue.
1632          */
1633         if (adapter->current_op) {
1634                 if (!iavf_asq_done(hw)) {
1635                         dev_dbg(&adapter->pdev->dev, "Admin queue timeout\n");
1636                         iavf_send_api_ver(adapter);
1637                 }
1638                 goto watchdog_done;
1639         }
1640         if (adapter->aq_required & IAVF_FLAG_AQ_GET_CONFIG) {
1641                 iavf_send_vf_config_msg(adapter);
1642                 goto watchdog_done;
1643         }
1644
1645         if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_QUEUES) {
1646                 iavf_disable_queues(adapter);
1647                 goto watchdog_done;
1648         }
1649
1650         if (adapter->aq_required & IAVF_FLAG_AQ_MAP_VECTORS) {
1651                 iavf_map_queues(adapter);
1652                 goto watchdog_done;
1653         }
1654
1655         if (adapter->aq_required & IAVF_FLAG_AQ_ADD_MAC_FILTER) {
1656                 iavf_add_ether_addrs(adapter);
1657                 goto watchdog_done;
1658         }
1659
1660         if (adapter->aq_required & IAVF_FLAG_AQ_ADD_VLAN_FILTER) {
1661                 iavf_add_vlans(adapter);
1662                 goto watchdog_done;
1663         }
1664
1665         if (adapter->aq_required & IAVF_FLAG_AQ_DEL_MAC_FILTER) {
1666                 iavf_del_ether_addrs(adapter);
1667                 goto watchdog_done;
1668         }
1669
1670         if (adapter->aq_required & IAVF_FLAG_AQ_DEL_VLAN_FILTER) {
1671                 iavf_del_vlans(adapter);
1672                 goto watchdog_done;
1673         }
1674
1675         if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING) {
1676                 iavf_enable_vlan_stripping(adapter);
1677                 goto watchdog_done;
1678         }
1679
1680         if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING) {
1681                 iavf_disable_vlan_stripping(adapter);
1682                 goto watchdog_done;
1683         }
1684
1685         if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_QUEUES) {
1686                 iavf_configure_queues(adapter);
1687                 goto watchdog_done;
1688         }
1689
1690         if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_QUEUES) {
1691                 iavf_enable_queues(adapter);
1692                 goto watchdog_done;
1693         }
1694
1695         if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_RSS) {
1696                 /* This message goes straight to the firmware, not the
1697                  * PF, so we don't have to set current_op as we will
1698                  * not get a response through the ARQ.
1699                  */
1700                 iavf_init_rss(adapter);
1701                 adapter->aq_required &= ~IAVF_FLAG_AQ_CONFIGURE_RSS;
1702                 goto watchdog_done;
1703         }
1704         if (adapter->aq_required & IAVF_FLAG_AQ_GET_HENA) {
1705                 iavf_get_hena(adapter);
1706                 goto watchdog_done;
1707         }
1708         if (adapter->aq_required & IAVF_FLAG_AQ_SET_HENA) {
1709                 iavf_set_hena(adapter);
1710                 goto watchdog_done;
1711         }
1712         if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_KEY) {
1713                 iavf_set_rss_key(adapter);
1714                 goto watchdog_done;
1715         }
1716         if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_LUT) {
1717                 iavf_set_rss_lut(adapter);
1718                 goto watchdog_done;
1719         }
1720
1721         if (adapter->aq_required & IAVF_FLAG_AQ_REQUEST_PROMISC) {
1722                 iavf_set_promiscuous(adapter, FLAG_VF_UNICAST_PROMISC |
1723                                        FLAG_VF_MULTICAST_PROMISC);
1724                 goto watchdog_done;
1725         }
1726
1727         if (adapter->aq_required & IAVF_FLAG_AQ_REQUEST_ALLMULTI) {
1728                 iavf_set_promiscuous(adapter, FLAG_VF_MULTICAST_PROMISC);
1729                 goto watchdog_done;
1730         }
1731
1732         if ((adapter->aq_required & IAVF_FLAG_AQ_RELEASE_PROMISC) &&
1733             (adapter->aq_required & IAVF_FLAG_AQ_RELEASE_ALLMULTI)) {
1734                 iavf_set_promiscuous(adapter, 0);
1735                 goto watchdog_done;
1736         }
1737
1738         if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CHANNELS) {
1739                 iavf_enable_channels(adapter);
1740                 goto watchdog_done;
1741         }
1742
1743         if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CHANNELS) {
1744                 iavf_disable_channels(adapter);
1745                 goto watchdog_done;
1746         }
1747
1748         if (adapter->aq_required & IAVF_FLAG_AQ_ADD_CLOUD_FILTER) {
1749                 iavf_add_cloud_filter(adapter);
1750                 goto watchdog_done;
1751         }
1752
1753         if (adapter->aq_required & IAVF_FLAG_AQ_DEL_CLOUD_FILTER) {
1754                 iavf_del_cloud_filter(adapter);
1755                 goto watchdog_done;
1756         }
1757
1758         schedule_delayed_work(&adapter->client_task, msecs_to_jiffies(5));
1759
1760         if (adapter->state == __IAVF_RUNNING)
1761                 iavf_request_stats(adapter);
1762 watchdog_done:
1763         if (adapter->state == __IAVF_RUNNING)
1764                 iavf_detect_recover_hung(&adapter->vsi);
1765         clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section);
1766 restart_watchdog:
1767         if (adapter->state == __IAVF_REMOVE)
1768                 return;
1769         if (adapter->aq_required)
1770                 mod_timer(&adapter->watchdog_timer,
1771                           jiffies + msecs_to_jiffies(20));
1772         else
1773                 mod_timer(&adapter->watchdog_timer, jiffies + (HZ * 2));
1774         schedule_work(&adapter->adminq_task);
1775 }
1776
1777 static void iavf_disable_vf(struct iavf_adapter *adapter)
1778 {
1779         struct iavf_mac_filter *f, *ftmp;
1780         struct iavf_vlan_filter *fv, *fvtmp;
1781         struct iavf_cloud_filter *cf, *cftmp;
1782
1783         adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED;
1784
1785         /* We don't use netif_running() because it may be true prior to
1786          * ndo_open() returning, so we can't assume it means all our open
1787          * tasks have finished, since we're not holding the rtnl_lock here.
1788          */
1789         if (adapter->state == __IAVF_RUNNING) {
1790                 set_bit(__I40E_VSI_DOWN, adapter->vsi.state);
1791                 netif_carrier_off(adapter->netdev);
1792                 netif_tx_disable(adapter->netdev);
1793                 adapter->link_up = false;
1794                 iavf_napi_disable_all(adapter);
1795                 iavf_irq_disable(adapter);
1796                 iavf_free_traffic_irqs(adapter);
1797                 iavf_free_all_tx_resources(adapter);
1798                 iavf_free_all_rx_resources(adapter);
1799         }
1800
1801         spin_lock_bh(&adapter->mac_vlan_list_lock);
1802
1803         /* Delete all of the filters */
1804         list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
1805                 list_del(&f->list);
1806                 kfree(f);
1807         }
1808
1809         list_for_each_entry_safe(fv, fvtmp, &adapter->vlan_filter_list, list) {
1810                 list_del(&fv->list);
1811                 kfree(fv);
1812         }
1813
1814         spin_unlock_bh(&adapter->mac_vlan_list_lock);
1815
1816         spin_lock_bh(&adapter->cloud_filter_list_lock);
1817         list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) {
1818                 list_del(&cf->list);
1819                 kfree(cf);
1820                 adapter->num_cloud_filters--;
1821         }
1822         spin_unlock_bh(&adapter->cloud_filter_list_lock);
1823
1824         iavf_free_misc_irq(adapter);
1825         iavf_reset_interrupt_capability(adapter);
1826         iavf_free_queues(adapter);
1827         iavf_free_q_vectors(adapter);
1828         kfree(adapter->vf_res);
1829         iavf_shutdown_adminq(&adapter->hw);
1830         adapter->netdev->flags &= ~IFF_UP;
1831         clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section);
1832         adapter->flags &= ~IAVF_FLAG_RESET_PENDING;
1833         adapter->state = __IAVF_DOWN;
1834         wake_up(&adapter->down_waitqueue);
1835         dev_info(&adapter->pdev->dev, "Reset task did not complete, VF disabled\n");
1836 }
1837
1838 #define IAVF_RESET_WAIT_MS 10
1839 #define IAVF_RESET_WAIT_COUNT 500
1840 /**
1841  * iavf_reset_task - Call-back task to handle hardware reset
1842  * @work: pointer to work_struct
1843  *
1844  * During reset we need to shut down and reinitialize the admin queue
1845  * before we can use it to communicate with the PF again. We also clear
1846  * and reinit the rings because that context is lost as well.
1847  **/
1848 static void iavf_reset_task(struct work_struct *work)
1849 {
1850         struct iavf_adapter *adapter = container_of(work,
1851                                                       struct iavf_adapter,
1852                                                       reset_task);
1853         struct virtchnl_vf_resource *vfres = adapter->vf_res;
1854         struct net_device *netdev = adapter->netdev;
1855         struct i40e_hw *hw = &adapter->hw;
1856         struct iavf_vlan_filter *vlf;
1857         struct iavf_cloud_filter *cf;
1858         struct iavf_mac_filter *f;
1859         u32 reg_val;
1860         int i = 0, err;
1861         bool running;
1862
1863         /* When device is being removed it doesn't make sense to run the reset
1864          * task, just return in such a case.
1865          */
1866         if (test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section))
1867                 return;
1868
1869         while (test_and_set_bit(__IAVF_IN_CLIENT_TASK,
1870                                 &adapter->crit_section))
1871                 usleep_range(500, 1000);
1872         if (CLIENT_ENABLED(adapter)) {
1873                 adapter->flags &= ~(IAVF_FLAG_CLIENT_NEEDS_OPEN |
1874                                     IAVF_FLAG_CLIENT_NEEDS_CLOSE |
1875                                     IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS |
1876                                     IAVF_FLAG_SERVICE_CLIENT_REQUESTED);
1877                 cancel_delayed_work_sync(&adapter->client_task);
1878                 iavf_notify_client_close(&adapter->vsi, true);
1879         }
1880         iavf_misc_irq_disable(adapter);
1881         if (adapter->flags & IAVF_FLAG_RESET_NEEDED) {
1882                 adapter->flags &= ~IAVF_FLAG_RESET_NEEDED;
1883                 /* Restart the AQ here. If we have been reset but didn't
1884                  * detect it, or if the PF had to reinit, our AQ will be hosed.
1885                  */
1886                 iavf_shutdown_adminq(hw);
1887                 iavf_init_adminq(hw);
1888                 iavf_request_reset(adapter);
1889         }
1890         adapter->flags |= IAVF_FLAG_RESET_PENDING;
1891
1892         /* poll until we see the reset actually happen */
1893         for (i = 0; i < IAVF_RESET_WAIT_COUNT; i++) {
1894                 reg_val = rd32(hw, I40E_VF_ARQLEN1) &
1895                           I40E_VF_ARQLEN1_ARQENABLE_MASK;
1896                 if (!reg_val)
1897                         break;
1898                 usleep_range(5000, 10000);
1899         }
1900         if (i == IAVF_RESET_WAIT_COUNT) {
1901                 dev_info(&adapter->pdev->dev, "Never saw reset\n");
1902                 goto continue_reset; /* act like the reset happened */
1903         }
1904
1905         /* wait until the reset is complete and the PF is responding to us */
1906         for (i = 0; i < IAVF_RESET_WAIT_COUNT; i++) {
1907                 /* sleep first to make sure a minimum wait time is met */
1908                 msleep(IAVF_RESET_WAIT_MS);
1909
1910                 reg_val = rd32(hw, I40E_VFGEN_RSTAT) &
1911                           I40E_VFGEN_RSTAT_VFR_STATE_MASK;
1912                 if (reg_val == VIRTCHNL_VFR_VFACTIVE)
1913                         break;
1914         }
1915
1916         pci_set_master(adapter->pdev);
1917
1918         if (i == IAVF_RESET_WAIT_COUNT) {
1919                 dev_err(&adapter->pdev->dev, "Reset never finished (%x)\n",
1920                         reg_val);
1921                 iavf_disable_vf(adapter);
1922                 clear_bit(__IAVF_IN_CLIENT_TASK, &adapter->crit_section);
1923                 return; /* Do not attempt to reinit. It's dead, Jim. */
1924         }
1925
1926 continue_reset:
1927         /* We don't use netif_running() because it may be true prior to
1928          * ndo_open() returning, so we can't assume it means all our open
1929          * tasks have finished, since we're not holding the rtnl_lock here.
1930          */
1931         running = ((adapter->state == __IAVF_RUNNING) ||
1932                    (adapter->state == __IAVF_RESETTING));
1933
1934         if (running) {
1935                 netif_carrier_off(netdev);
1936                 netif_tx_stop_all_queues(netdev);
1937                 adapter->link_up = false;
1938                 iavf_napi_disable_all(adapter);
1939         }
1940         iavf_irq_disable(adapter);
1941
1942         adapter->state = __IAVF_RESETTING;
1943         adapter->flags &= ~IAVF_FLAG_RESET_PENDING;
1944
1945         /* free the Tx/Rx rings and descriptors, might be better to just
1946          * re-use them sometime in the future
1947          */
1948         iavf_free_all_rx_resources(adapter);
1949         iavf_free_all_tx_resources(adapter);
1950
1951         adapter->flags |= IAVF_FLAG_QUEUES_DISABLED;
1952         /* kill and reinit the admin queue */
1953         iavf_shutdown_adminq(hw);
1954         adapter->current_op = VIRTCHNL_OP_UNKNOWN;
1955         err = iavf_init_adminq(hw);
1956         if (err)
1957                 dev_info(&adapter->pdev->dev, "Failed to init adminq: %d\n",
1958                          err);
1959         adapter->aq_required = 0;
1960
1961         if (adapter->flags & IAVF_FLAG_REINIT_ITR_NEEDED) {
1962                 err = iavf_reinit_interrupt_scheme(adapter);
1963                 if (err)
1964                         goto reset_err;
1965         }
1966
1967         adapter->aq_required |= IAVF_FLAG_AQ_GET_CONFIG;
1968         adapter->aq_required |= IAVF_FLAG_AQ_MAP_VECTORS;
1969
1970         spin_lock_bh(&adapter->mac_vlan_list_lock);
1971
1972         /* re-add all MAC filters */
1973         list_for_each_entry(f, &adapter->mac_filter_list, list) {
1974                 f->add = true;
1975         }
1976         /* re-add all VLAN filters */
1977         list_for_each_entry(vlf, &adapter->vlan_filter_list, list) {
1978                 vlf->add = true;
1979         }
1980
1981         spin_unlock_bh(&adapter->mac_vlan_list_lock);
1982
1983         /* check if TCs are running and re-add all cloud filters */
1984         spin_lock_bh(&adapter->cloud_filter_list_lock);
1985         if ((vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&
1986             adapter->num_tc) {
1987                 list_for_each_entry(cf, &adapter->cloud_filter_list, list) {
1988                         cf->add = true;
1989                 }
1990         }
1991         spin_unlock_bh(&adapter->cloud_filter_list_lock);
1992
1993         adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER;
1994         adapter->aq_required |= IAVF_FLAG_AQ_ADD_VLAN_FILTER;
1995         adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER;
1996         iavf_misc_irq_enable(adapter);
1997
1998         mod_timer(&adapter->watchdog_timer, jiffies + 2);
1999
2000         /* We were running when the reset started, so we need to restore some
2001          * state here.
2002          */
2003         if (running) {
2004                 /* allocate transmit descriptors */
2005                 err = iavf_setup_all_tx_resources(adapter);
2006                 if (err)
2007                         goto reset_err;
2008
2009                 /* allocate receive descriptors */
2010                 err = iavf_setup_all_rx_resources(adapter);
2011                 if (err)
2012                         goto reset_err;
2013
2014                 if (adapter->flags & IAVF_FLAG_REINIT_ITR_NEEDED) {
2015                         err = iavf_request_traffic_irqs(adapter, netdev->name);
2016                         if (err)
2017                                 goto reset_err;
2018
2019                         adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED;
2020                 }
2021
2022                 iavf_configure(adapter);
2023
2024                 iavf_up_complete(adapter);
2025
2026                 iavf_irq_enable(adapter, true);
2027         } else {
2028                 adapter->state = __IAVF_DOWN;
2029                 wake_up(&adapter->down_waitqueue);
2030         }
2031         clear_bit(__IAVF_IN_CLIENT_TASK, &adapter->crit_section);
2032         clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section);
2033
2034         return;
2035 reset_err:
2036         clear_bit(__IAVF_IN_CLIENT_TASK, &adapter->crit_section);
2037         clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section);
2038         dev_err(&adapter->pdev->dev, "failed to allocate resources during reinit\n");
2039         iavf_close(netdev);
2040 }
2041
2042 /**
2043  * iavf_adminq_task - worker thread to clean the admin queue
2044  * @work: pointer to work_struct containing our data
2045  **/
2046 static void iavf_adminq_task(struct work_struct *work)
2047 {
2048         struct iavf_adapter *adapter =
2049                 container_of(work, struct iavf_adapter, adminq_task);
2050         struct i40e_hw *hw = &adapter->hw;
2051         struct i40e_arq_event_info event;
2052         enum virtchnl_ops v_op;
2053         iavf_status ret, v_ret;
2054         u32 val, oldval;
2055         u16 pending;
2056
2057         if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED)
2058                 goto out;
2059
2060         event.buf_len = IAVF_MAX_AQ_BUF_SIZE;
2061         event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
2062         if (!event.msg_buf)
2063                 goto out;
2064
2065         do {
2066                 ret = iavf_clean_arq_element(hw, &event, &pending);
2067                 v_op = (enum virtchnl_ops)le32_to_cpu(event.desc.cookie_high);
2068                 v_ret = (iavf_status)le32_to_cpu(event.desc.cookie_low);
2069
2070                 if (ret || !v_op)
2071                         break; /* No event to process or error cleaning ARQ */
2072
2073                 iavf_virtchnl_completion(adapter, v_op, v_ret, event.msg_buf,
2074                                          event.msg_len);
2075                 if (pending != 0)
2076                         memset(event.msg_buf, 0, IAVF_MAX_AQ_BUF_SIZE);
2077         } while (pending);
2078
2079         if ((adapter->flags &
2080              (IAVF_FLAG_RESET_PENDING | IAVF_FLAG_RESET_NEEDED)) ||
2081             adapter->state == __IAVF_RESETTING)
2082                 goto freedom;
2083
2084         /* check for error indications */
2085         val = rd32(hw, hw->aq.arq.len);
2086         if (val == 0xdeadbeef) /* indicates device in reset */
2087                 goto freedom;
2088         oldval = val;
2089         if (val & I40E_VF_ARQLEN1_ARQVFE_MASK) {
2090                 dev_info(&adapter->pdev->dev, "ARQ VF Error detected\n");
2091                 val &= ~I40E_VF_ARQLEN1_ARQVFE_MASK;
2092         }
2093         if (val & I40E_VF_ARQLEN1_ARQOVFL_MASK) {
2094                 dev_info(&adapter->pdev->dev, "ARQ Overflow Error detected\n");
2095                 val &= ~I40E_VF_ARQLEN1_ARQOVFL_MASK;
2096         }
2097         if (val & I40E_VF_ARQLEN1_ARQCRIT_MASK) {
2098                 dev_info(&adapter->pdev->dev, "ARQ Critical Error detected\n");
2099                 val &= ~I40E_VF_ARQLEN1_ARQCRIT_MASK;
2100         }
2101         if (oldval != val)
2102                 wr32(hw, hw->aq.arq.len, val);
2103
2104         val = rd32(hw, hw->aq.asq.len);
2105         oldval = val;
2106         if (val & I40E_VF_ATQLEN1_ATQVFE_MASK) {
2107                 dev_info(&adapter->pdev->dev, "ASQ VF Error detected\n");
2108                 val &= ~I40E_VF_ATQLEN1_ATQVFE_MASK;
2109         }
2110         if (val & I40E_VF_ATQLEN1_ATQOVFL_MASK) {
2111                 dev_info(&adapter->pdev->dev, "ASQ Overflow Error detected\n");
2112                 val &= ~I40E_VF_ATQLEN1_ATQOVFL_MASK;
2113         }
2114         if (val & I40E_VF_ATQLEN1_ATQCRIT_MASK) {
2115                 dev_info(&adapter->pdev->dev, "ASQ Critical Error detected\n");
2116                 val &= ~I40E_VF_ATQLEN1_ATQCRIT_MASK;
2117         }
2118         if (oldval != val)
2119                 wr32(hw, hw->aq.asq.len, val);
2120
2121 freedom:
2122         kfree(event.msg_buf);
2123 out:
2124         /* re-enable Admin queue interrupt cause */
2125         iavf_misc_irq_enable(adapter);
2126 }
2127
2128 /**
2129  * iavf_client_task - worker thread to perform client work
2130  * @work: pointer to work_struct containing our data
2131  *
2132  * This task handles client interactions. Because client calls can be
2133  * reentrant, we can't handle them in the watchdog.
2134  **/
2135 static void iavf_client_task(struct work_struct *work)
2136 {
2137         struct iavf_adapter *adapter =
2138                 container_of(work, struct iavf_adapter, client_task.work);
2139
2140         /* If we can't get the client bit, just give up. We'll be rescheduled
2141          * later.
2142          */
2143
2144         if (test_and_set_bit(__IAVF_IN_CLIENT_TASK, &adapter->crit_section))
2145                 return;
2146
2147         if (adapter->flags & IAVF_FLAG_SERVICE_CLIENT_REQUESTED) {
2148                 iavf_client_subtask(adapter);
2149                 adapter->flags &= ~IAVF_FLAG_SERVICE_CLIENT_REQUESTED;
2150                 goto out;
2151         }
2152         if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS) {
2153                 iavf_notify_client_l2_params(&adapter->vsi);
2154                 adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS;
2155                 goto out;
2156         }
2157         if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_CLOSE) {
2158                 iavf_notify_client_close(&adapter->vsi, false);
2159                 adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_CLOSE;
2160                 goto out;
2161         }
2162         if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_OPEN) {
2163                 iavf_notify_client_open(&adapter->vsi);
2164                 adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_OPEN;
2165         }
2166 out:
2167         clear_bit(__IAVF_IN_CLIENT_TASK, &adapter->crit_section);
2168 }
2169
2170 /**
2171  * iavf_free_all_tx_resources - Free Tx Resources for All Queues
2172  * @adapter: board private structure
2173  *
2174  * Free all transmit software resources
2175  **/
2176 void iavf_free_all_tx_resources(struct iavf_adapter *adapter)
2177 {
2178         int i;
2179
2180         if (!adapter->tx_rings)
2181                 return;
2182
2183         for (i = 0; i < adapter->num_active_queues; i++)
2184                 if (adapter->tx_rings[i].desc)
2185                         iavf_free_tx_resources(&adapter->tx_rings[i]);
2186 }
2187
2188 /**
2189  * iavf_setup_all_tx_resources - allocate all queues Tx resources
2190  * @adapter: board private structure
2191  *
2192  * If this function returns with an error, then it's possible one or
2193  * more of the rings is populated (while the rest are not).  It is the
2194  * callers duty to clean those orphaned rings.
2195  *
2196  * Return 0 on success, negative on failure
2197  **/
2198 static int iavf_setup_all_tx_resources(struct iavf_adapter *adapter)
2199 {
2200         int i, err = 0;
2201
2202         for (i = 0; i < adapter->num_active_queues; i++) {
2203                 adapter->tx_rings[i].count = adapter->tx_desc_count;
2204                 err = iavf_setup_tx_descriptors(&adapter->tx_rings[i]);
2205                 if (!err)
2206                         continue;
2207                 dev_err(&adapter->pdev->dev,
2208                         "Allocation for Tx Queue %u failed\n", i);
2209                 break;
2210         }
2211
2212         return err;
2213 }
2214
2215 /**
2216  * iavf_setup_all_rx_resources - allocate all queues Rx resources
2217  * @adapter: board private structure
2218  *
2219  * If this function returns with an error, then it's possible one or
2220  * more of the rings is populated (while the rest are not).  It is the
2221  * callers duty to clean those orphaned rings.
2222  *
2223  * Return 0 on success, negative on failure
2224  **/
2225 static int iavf_setup_all_rx_resources(struct iavf_adapter *adapter)
2226 {
2227         int i, err = 0;
2228
2229         for (i = 0; i < adapter->num_active_queues; i++) {
2230                 adapter->rx_rings[i].count = adapter->rx_desc_count;
2231                 err = iavf_setup_rx_descriptors(&adapter->rx_rings[i]);
2232                 if (!err)
2233                         continue;
2234                 dev_err(&adapter->pdev->dev,
2235                         "Allocation for Rx Queue %u failed\n", i);
2236                 break;
2237         }
2238         return err;
2239 }
2240
2241 /**
2242  * iavf_free_all_rx_resources - Free Rx Resources for All Queues
2243  * @adapter: board private structure
2244  *
2245  * Free all receive software resources
2246  **/
2247 void iavf_free_all_rx_resources(struct iavf_adapter *adapter)
2248 {
2249         int i;
2250
2251         if (!adapter->rx_rings)
2252                 return;
2253
2254         for (i = 0; i < adapter->num_active_queues; i++)
2255                 if (adapter->rx_rings[i].desc)
2256                         iavf_free_rx_resources(&adapter->rx_rings[i]);
2257 }
2258
2259 /**
2260  * iavf_validate_tx_bandwidth - validate the max Tx bandwidth
2261  * @adapter: board private structure
2262  * @max_tx_rate: max Tx bw for a tc
2263  **/
2264 static int iavf_validate_tx_bandwidth(struct iavf_adapter *adapter,
2265                                       u64 max_tx_rate)
2266 {
2267         int speed = 0, ret = 0;
2268
2269         switch (adapter->link_speed) {
2270         case I40E_LINK_SPEED_40GB:
2271                 speed = 40000;
2272                 break;
2273         case I40E_LINK_SPEED_25GB:
2274                 speed = 25000;
2275                 break;
2276         case I40E_LINK_SPEED_20GB:
2277                 speed = 20000;
2278                 break;
2279         case I40E_LINK_SPEED_10GB:
2280                 speed = 10000;
2281                 break;
2282         case I40E_LINK_SPEED_1GB:
2283                 speed = 1000;
2284                 break;
2285         case I40E_LINK_SPEED_100MB:
2286                 speed = 100;
2287                 break;
2288         default:
2289                 break;
2290         }
2291
2292         if (max_tx_rate > speed) {
2293                 dev_err(&adapter->pdev->dev,
2294                         "Invalid tx rate specified\n");
2295                 ret = -EINVAL;
2296         }
2297
2298         return ret;
2299 }
2300
2301 /**
2302  * iavf_validate_channel_config - validate queue mapping info
2303  * @adapter: board private structure
2304  * @mqprio_qopt: queue parameters
2305  *
2306  * This function validates if the config provided by the user to
2307  * configure queue channels is valid or not. Returns 0 on a valid
2308  * config.
2309  **/
2310 static int iavf_validate_ch_config(struct iavf_adapter *adapter,
2311                                    struct tc_mqprio_qopt_offload *mqprio_qopt)
2312 {
2313         u64 total_max_rate = 0;
2314         int i, num_qps = 0;
2315         u64 tx_rate = 0;
2316         int ret = 0;
2317
2318         if (mqprio_qopt->qopt.num_tc > IAVF_MAX_TRAFFIC_CLASS ||
2319             mqprio_qopt->qopt.num_tc < 1)
2320                 return -EINVAL;
2321
2322         for (i = 0; i <= mqprio_qopt->qopt.num_tc - 1; i++) {
2323                 if (!mqprio_qopt->qopt.count[i] ||
2324                     mqprio_qopt->qopt.offset[i] != num_qps)
2325                         return -EINVAL;
2326                 if (mqprio_qopt->min_rate[i]) {
2327                         dev_err(&adapter->pdev->dev,
2328                                 "Invalid min tx rate (greater than 0) specified\n");
2329                         return -EINVAL;
2330                 }
2331                 /*convert to Mbps */
2332                 tx_rate = div_u64(mqprio_qopt->max_rate[i],
2333                                   IAVF_MBPS_DIVISOR);
2334                 total_max_rate += tx_rate;
2335                 num_qps += mqprio_qopt->qopt.count[i];
2336         }
2337         if (num_qps > IAVF_MAX_REQ_QUEUES)
2338                 return -EINVAL;
2339
2340         ret = iavf_validate_tx_bandwidth(adapter, total_max_rate);
2341         return ret;
2342 }
2343
2344 /**
2345  * iavf_del_all_cloud_filters - delete all cloud filters
2346  * on the traffic classes
2347  **/
2348 static void iavf_del_all_cloud_filters(struct iavf_adapter *adapter)
2349 {
2350         struct iavf_cloud_filter *cf, *cftmp;
2351
2352         spin_lock_bh(&adapter->cloud_filter_list_lock);
2353         list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list,
2354                                  list) {
2355                 list_del(&cf->list);
2356                 kfree(cf);
2357                 adapter->num_cloud_filters--;
2358         }
2359         spin_unlock_bh(&adapter->cloud_filter_list_lock);
2360 }
2361
2362 /**
2363  * __iavf_setup_tc - configure multiple traffic classes
2364  * @netdev: network interface device structure
2365  * @type_date: tc offload data
2366  *
2367  * This function processes the config information provided by the
2368  * user to configure traffic classes/queue channels and packages the
2369  * information to request the PF to setup traffic classes.
2370  *
2371  * Returns 0 on success.
2372  **/
2373 static int __iavf_setup_tc(struct net_device *netdev, void *type_data)
2374 {
2375         struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
2376         struct iavf_adapter *adapter = netdev_priv(netdev);
2377         struct virtchnl_vf_resource *vfres = adapter->vf_res;
2378         u8 num_tc = 0, total_qps = 0;
2379         int ret = 0, netdev_tc = 0;
2380         u64 max_tx_rate;
2381         u16 mode;
2382         int i;
2383
2384         num_tc = mqprio_qopt->qopt.num_tc;
2385         mode = mqprio_qopt->mode;
2386
2387         /* delete queue_channel */
2388         if (!mqprio_qopt->qopt.hw) {
2389                 if (adapter->ch_config.state == __IAVF_TC_RUNNING) {
2390                         /* reset the tc configuration */
2391                         netdev_reset_tc(netdev);
2392                         adapter->num_tc = 0;
2393                         netif_tx_stop_all_queues(netdev);
2394                         netif_tx_disable(netdev);
2395                         iavf_del_all_cloud_filters(adapter);
2396                         adapter->aq_required = IAVF_FLAG_AQ_DISABLE_CHANNELS;
2397                         goto exit;
2398                 } else {
2399                         return -EINVAL;
2400                 }
2401         }
2402
2403         /* add queue channel */
2404         if (mode == TC_MQPRIO_MODE_CHANNEL) {
2405                 if (!(vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ)) {
2406                         dev_err(&adapter->pdev->dev, "ADq not supported\n");
2407                         return -EOPNOTSUPP;
2408                 }
2409                 if (adapter->ch_config.state != __IAVF_TC_INVALID) {
2410                         dev_err(&adapter->pdev->dev, "TC configuration already exists\n");
2411                         return -EINVAL;
2412                 }
2413
2414                 ret = iavf_validate_ch_config(adapter, mqprio_qopt);
2415                 if (ret)
2416                         return ret;
2417                 /* Return if same TC config is requested */
2418                 if (adapter->num_tc == num_tc)
2419                         return 0;
2420                 adapter->num_tc = num_tc;
2421
2422                 for (i = 0; i < IAVF_MAX_TRAFFIC_CLASS; i++) {
2423                         if (i < num_tc) {
2424                                 adapter->ch_config.ch_info[i].count =
2425                                         mqprio_qopt->qopt.count[i];
2426                                 adapter->ch_config.ch_info[i].offset =
2427                                         mqprio_qopt->qopt.offset[i];
2428                                 total_qps += mqprio_qopt->qopt.count[i];
2429                                 max_tx_rate = mqprio_qopt->max_rate[i];
2430                                 /* convert to Mbps */
2431                                 max_tx_rate = div_u64(max_tx_rate,
2432                                                       IAVF_MBPS_DIVISOR);
2433                                 adapter->ch_config.ch_info[i].max_tx_rate =
2434                                         max_tx_rate;
2435                         } else {
2436                                 adapter->ch_config.ch_info[i].count = 1;
2437                                 adapter->ch_config.ch_info[i].offset = 0;
2438                         }
2439                 }
2440                 adapter->ch_config.total_qps = total_qps;
2441                 netif_tx_stop_all_queues(netdev);
2442                 netif_tx_disable(netdev);
2443                 adapter->aq_required |= IAVF_FLAG_AQ_ENABLE_CHANNELS;
2444                 netdev_reset_tc(netdev);
2445                 /* Report the tc mapping up the stack */
2446                 netdev_set_num_tc(adapter->netdev, num_tc);
2447                 for (i = 0; i < IAVF_MAX_TRAFFIC_CLASS; i++) {
2448                         u16 qcount = mqprio_qopt->qopt.count[i];
2449                         u16 qoffset = mqprio_qopt->qopt.offset[i];
2450
2451                         if (i < num_tc)
2452                                 netdev_set_tc_queue(netdev, netdev_tc++, qcount,
2453                                                     qoffset);
2454                 }
2455         }
2456 exit:
2457         return ret;
2458 }
2459
2460 /**
2461  * iavf_parse_cls_flower - Parse tc flower filters provided by kernel
2462  * @adapter: board private structure
2463  * @cls_flower: pointer to struct tc_cls_flower_offload
2464  * @filter: pointer to cloud filter structure
2465  */
2466 static int iavf_parse_cls_flower(struct iavf_adapter *adapter,
2467                                  struct tc_cls_flower_offload *f,
2468                                  struct iavf_cloud_filter *filter)
2469 {
2470         u16 n_proto_mask = 0;
2471         u16 n_proto_key = 0;
2472         u8 field_flags = 0;
2473         u16 addr_type = 0;
2474         u16 n_proto = 0;
2475         int i = 0;
2476         struct virtchnl_filter *vf = &filter->f;
2477
2478         if (f->dissector->used_keys &
2479             ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) |
2480               BIT(FLOW_DISSECTOR_KEY_BASIC) |
2481               BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
2482               BIT(FLOW_DISSECTOR_KEY_VLAN) |
2483               BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
2484               BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
2485               BIT(FLOW_DISSECTOR_KEY_PORTS) |
2486               BIT(FLOW_DISSECTOR_KEY_ENC_KEYID))) {
2487                 dev_err(&adapter->pdev->dev, "Unsupported key used: 0x%x\n",
2488                         f->dissector->used_keys);
2489                 return -EOPNOTSUPP;
2490         }
2491
2492         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
2493                 struct flow_dissector_key_keyid *mask =
2494                         skb_flow_dissector_target(f->dissector,
2495                                                   FLOW_DISSECTOR_KEY_ENC_KEYID,
2496                                                   f->mask);
2497
2498                 if (mask->keyid != 0)
2499                         field_flags |= IAVF_CLOUD_FIELD_TEN_ID;
2500         }
2501
2502         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_BASIC)) {
2503                 struct flow_dissector_key_basic *key =
2504                         skb_flow_dissector_target(f->dissector,
2505                                                   FLOW_DISSECTOR_KEY_BASIC,
2506                                                   f->key);
2507
2508                 struct flow_dissector_key_basic *mask =
2509                         skb_flow_dissector_target(f->dissector,
2510                                                   FLOW_DISSECTOR_KEY_BASIC,
2511                                                   f->mask);
2512                 n_proto_key = ntohs(key->n_proto);
2513                 n_proto_mask = ntohs(mask->n_proto);
2514
2515                 if (n_proto_key == ETH_P_ALL) {
2516                         n_proto_key = 0;
2517                         n_proto_mask = 0;
2518                 }
2519                 n_proto = n_proto_key & n_proto_mask;
2520                 if (n_proto != ETH_P_IP && n_proto != ETH_P_IPV6)
2521                         return -EINVAL;
2522                 if (n_proto == ETH_P_IPV6) {
2523                         /* specify flow type as TCP IPv6 */
2524                         vf->flow_type = VIRTCHNL_TCP_V6_FLOW;
2525                 }
2526
2527                 if (key->ip_proto != IPPROTO_TCP) {
2528                         dev_info(&adapter->pdev->dev, "Only TCP transport is supported\n");
2529                         return -EINVAL;
2530                 }
2531         }
2532
2533         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
2534                 struct flow_dissector_key_eth_addrs *key =
2535                         skb_flow_dissector_target(f->dissector,
2536                                                   FLOW_DISSECTOR_KEY_ETH_ADDRS,
2537                                                   f->key);
2538
2539                 struct flow_dissector_key_eth_addrs *mask =
2540                         skb_flow_dissector_target(f->dissector,
2541                                                   FLOW_DISSECTOR_KEY_ETH_ADDRS,
2542                                                   f->mask);
2543                 /* use is_broadcast and is_zero to check for all 0xf or 0 */
2544                 if (!is_zero_ether_addr(mask->dst)) {
2545                         if (is_broadcast_ether_addr(mask->dst)) {
2546                                 field_flags |= IAVF_CLOUD_FIELD_OMAC;
2547                         } else {
2548                                 dev_err(&adapter->pdev->dev, "Bad ether dest mask %pM\n",
2549                                         mask->dst);
2550                                 return I40E_ERR_CONFIG;
2551                         }
2552                 }
2553
2554                 if (!is_zero_ether_addr(mask->src)) {
2555                         if (is_broadcast_ether_addr(mask->src)) {
2556                                 field_flags |= IAVF_CLOUD_FIELD_IMAC;
2557                         } else {
2558                                 dev_err(&adapter->pdev->dev, "Bad ether src mask %pM\n",
2559                                         mask->src);
2560                                 return I40E_ERR_CONFIG;
2561                         }
2562                 }
2563
2564                 if (!is_zero_ether_addr(key->dst))
2565                         if (is_valid_ether_addr(key->dst) ||
2566                             is_multicast_ether_addr(key->dst)) {
2567                                 /* set the mask if a valid dst_mac address */
2568                                 for (i = 0; i < ETH_ALEN; i++)
2569                                         vf->mask.tcp_spec.dst_mac[i] |= 0xff;
2570                                 ether_addr_copy(vf->data.tcp_spec.dst_mac,
2571                                                 key->dst);
2572                         }
2573
2574                 if (!is_zero_ether_addr(key->src))
2575                         if (is_valid_ether_addr(key->src) ||
2576                             is_multicast_ether_addr(key->src)) {
2577                                 /* set the mask if a valid dst_mac address */
2578                                 for (i = 0; i < ETH_ALEN; i++)
2579                                         vf->mask.tcp_spec.src_mac[i] |= 0xff;
2580                                 ether_addr_copy(vf->data.tcp_spec.src_mac,
2581                                                 key->src);
2582                 }
2583         }
2584
2585         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_VLAN)) {
2586                 struct flow_dissector_key_vlan *key =
2587                         skb_flow_dissector_target(f->dissector,
2588                                                   FLOW_DISSECTOR_KEY_VLAN,
2589                                                   f->key);
2590                 struct flow_dissector_key_vlan *mask =
2591                         skb_flow_dissector_target(f->dissector,
2592                                                   FLOW_DISSECTOR_KEY_VLAN,
2593                                                   f->mask);
2594
2595                 if (mask->vlan_id) {
2596                         if (mask->vlan_id == VLAN_VID_MASK) {
2597                                 field_flags |= IAVF_CLOUD_FIELD_IVLAN;
2598                         } else {
2599                                 dev_err(&adapter->pdev->dev, "Bad vlan mask %u\n",
2600                                         mask->vlan_id);
2601                                 return I40E_ERR_CONFIG;
2602                         }
2603                 }
2604                 vf->mask.tcp_spec.vlan_id |= cpu_to_be16(0xffff);
2605                 vf->data.tcp_spec.vlan_id = cpu_to_be16(key->vlan_id);
2606         }
2607
2608         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_CONTROL)) {
2609                 struct flow_dissector_key_control *key =
2610                         skb_flow_dissector_target(f->dissector,
2611                                                   FLOW_DISSECTOR_KEY_CONTROL,
2612                                                   f->key);
2613
2614                 addr_type = key->addr_type;
2615         }
2616
2617         if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
2618                 struct flow_dissector_key_ipv4_addrs *key =
2619                         skb_flow_dissector_target(f->dissector,
2620                                                   FLOW_DISSECTOR_KEY_IPV4_ADDRS,
2621                                                   f->key);
2622                 struct flow_dissector_key_ipv4_addrs *mask =
2623                         skb_flow_dissector_target(f->dissector,
2624                                                   FLOW_DISSECTOR_KEY_IPV4_ADDRS,
2625                                                   f->mask);
2626
2627                 if (mask->dst) {
2628                         if (mask->dst == cpu_to_be32(0xffffffff)) {
2629                                 field_flags |= IAVF_CLOUD_FIELD_IIP;
2630                         } else {
2631                                 dev_err(&adapter->pdev->dev, "Bad ip dst mask 0x%08x\n",
2632                                         be32_to_cpu(mask->dst));
2633                                 return I40E_ERR_CONFIG;
2634                         }
2635                 }
2636
2637                 if (mask->src) {
2638                         if (mask->src == cpu_to_be32(0xffffffff)) {
2639                                 field_flags |= IAVF_CLOUD_FIELD_IIP;
2640                         } else {
2641                                 dev_err(&adapter->pdev->dev, "Bad ip src mask 0x%08x\n",
2642                                         be32_to_cpu(mask->dst));
2643                                 return I40E_ERR_CONFIG;
2644                         }
2645                 }
2646
2647                 if (field_flags & IAVF_CLOUD_FIELD_TEN_ID) {
2648                         dev_info(&adapter->pdev->dev, "Tenant id not allowed for ip filter\n");
2649                         return I40E_ERR_CONFIG;
2650                 }
2651                 if (key->dst) {
2652                         vf->mask.tcp_spec.dst_ip[0] |= cpu_to_be32(0xffffffff);
2653                         vf->data.tcp_spec.dst_ip[0] = key->dst;
2654                 }
2655                 if (key->src) {
2656                         vf->mask.tcp_spec.src_ip[0] |= cpu_to_be32(0xffffffff);
2657                         vf->data.tcp_spec.src_ip[0] = key->src;
2658                 }
2659         }
2660
2661         if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
2662                 struct flow_dissector_key_ipv6_addrs *key =
2663                         skb_flow_dissector_target(f->dissector,
2664                                                   FLOW_DISSECTOR_KEY_IPV6_ADDRS,
2665                                                   f->key);
2666                 struct flow_dissector_key_ipv6_addrs *mask =
2667                         skb_flow_dissector_target(f->dissector,
2668                                                   FLOW_DISSECTOR_KEY_IPV6_ADDRS,
2669                                                   f->mask);
2670
2671                 /* validate mask, make sure it is not IPV6_ADDR_ANY */
2672                 if (ipv6_addr_any(&mask->dst)) {
2673                         dev_err(&adapter->pdev->dev, "Bad ipv6 dst mask 0x%02x\n",
2674                                 IPV6_ADDR_ANY);
2675                         return I40E_ERR_CONFIG;
2676                 }
2677
2678                 /* src and dest IPv6 address should not be LOOPBACK
2679                  * (0:0:0:0:0:0:0:1) which can be represented as ::1
2680                  */
2681                 if (ipv6_addr_loopback(&key->dst) ||
2682                     ipv6_addr_loopback(&key->src)) {
2683                         dev_err(&adapter->pdev->dev,
2684                                 "ipv6 addr should not be loopback\n");
2685                         return I40E_ERR_CONFIG;
2686                 }
2687                 if (!ipv6_addr_any(&mask->dst) || !ipv6_addr_any(&mask->src))
2688                         field_flags |= IAVF_CLOUD_FIELD_IIP;
2689
2690                 for (i = 0; i < 4; i++)
2691                         vf->mask.tcp_spec.dst_ip[i] |= cpu_to_be32(0xffffffff);
2692                 memcpy(&vf->data.tcp_spec.dst_ip, &key->dst.s6_addr32,
2693                        sizeof(vf->data.tcp_spec.dst_ip));
2694                 for (i = 0; i < 4; i++)
2695                         vf->mask.tcp_spec.src_ip[i] |= cpu_to_be32(0xffffffff);
2696                 memcpy(&vf->data.tcp_spec.src_ip, &key->src.s6_addr32,
2697                        sizeof(vf->data.tcp_spec.src_ip));
2698         }
2699         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_PORTS)) {
2700                 struct flow_dissector_key_ports *key =
2701                         skb_flow_dissector_target(f->dissector,
2702                                                   FLOW_DISSECTOR_KEY_PORTS,
2703                                                   f->key);
2704                 struct flow_dissector_key_ports *mask =
2705                         skb_flow_dissector_target(f->dissector,
2706                                                   FLOW_DISSECTOR_KEY_PORTS,
2707                                                   f->mask);
2708
2709                 if (mask->src) {
2710                         if (mask->src == cpu_to_be16(0xffff)) {
2711                                 field_flags |= IAVF_CLOUD_FIELD_IIP;
2712                         } else {
2713                                 dev_err(&adapter->pdev->dev, "Bad src port mask %u\n",
2714                                         be16_to_cpu(mask->src));
2715                                 return I40E_ERR_CONFIG;
2716                         }
2717                 }
2718
2719                 if (mask->dst) {
2720                         if (mask->dst == cpu_to_be16(0xffff)) {
2721                                 field_flags |= IAVF_CLOUD_FIELD_IIP;
2722                         } else {
2723                                 dev_err(&adapter->pdev->dev, "Bad dst port mask %u\n",
2724                                         be16_to_cpu(mask->dst));
2725                                 return I40E_ERR_CONFIG;
2726                         }
2727                 }
2728                 if (key->dst) {
2729                         vf->mask.tcp_spec.dst_port |= cpu_to_be16(0xffff);
2730                         vf->data.tcp_spec.dst_port = key->dst;
2731                 }
2732
2733                 if (key->src) {
2734                         vf->mask.tcp_spec.src_port |= cpu_to_be16(0xffff);
2735                         vf->data.tcp_spec.src_port = key->src;
2736                 }
2737         }
2738         vf->field_flags = field_flags;
2739
2740         return 0;
2741 }
2742
2743 /**
2744  * iavf_handle_tclass - Forward to a traffic class on the device
2745  * @adapter: board private structure
2746  * @tc: traffic class index on the device
2747  * @filter: pointer to cloud filter structure
2748  */
2749 static int iavf_handle_tclass(struct iavf_adapter *adapter, u32 tc,
2750                               struct iavf_cloud_filter *filter)
2751 {
2752         if (tc == 0)
2753                 return 0;
2754         if (tc < adapter->num_tc) {
2755                 if (!filter->f.data.tcp_spec.dst_port) {
2756                         dev_err(&adapter->pdev->dev,
2757                                 "Specify destination port to redirect to traffic class other than TC0\n");
2758                         return -EINVAL;
2759                 }
2760         }
2761         /* redirect to a traffic class on the same device */
2762         filter->f.action = VIRTCHNL_ACTION_TC_REDIRECT;
2763         filter->f.action_meta = tc;
2764         return 0;
2765 }
2766
2767 /**
2768  * iavf_configure_clsflower - Add tc flower filters
2769  * @adapter: board private structure
2770  * @cls_flower: Pointer to struct tc_cls_flower_offload
2771  */
2772 static int iavf_configure_clsflower(struct iavf_adapter *adapter,
2773                                     struct tc_cls_flower_offload *cls_flower)
2774 {
2775         int tc = tc_classid_to_hwtc(adapter->netdev, cls_flower->classid);
2776         struct iavf_cloud_filter *filter = NULL;
2777         int err = -EINVAL, count = 50;
2778
2779         if (tc < 0) {
2780                 dev_err(&adapter->pdev->dev, "Invalid traffic class\n");
2781                 return -EINVAL;
2782         }
2783
2784         filter = kzalloc(sizeof(*filter), GFP_KERNEL);
2785         if (!filter)
2786                 return -ENOMEM;
2787
2788         while (test_and_set_bit(__IAVF_IN_CRITICAL_TASK,
2789                                 &adapter->crit_section)) {
2790                 if (--count == 0)
2791                         goto err;
2792                 udelay(1);
2793         }
2794
2795         filter->cookie = cls_flower->cookie;
2796
2797         /* set the mask to all zeroes to begin with */
2798         memset(&filter->f.mask.tcp_spec, 0, sizeof(struct virtchnl_l4_spec));
2799         /* start out with flow type and eth type IPv4 to begin with */
2800         filter->f.flow_type = VIRTCHNL_TCP_V4_FLOW;
2801         err = iavf_parse_cls_flower(adapter, cls_flower, filter);
2802         if (err < 0)
2803                 goto err;
2804
2805         err = iavf_handle_tclass(adapter, tc, filter);
2806         if (err < 0)
2807                 goto err;
2808
2809         /* add filter to the list */
2810         spin_lock_bh(&adapter->cloud_filter_list_lock);
2811         list_add_tail(&filter->list, &adapter->cloud_filter_list);
2812         adapter->num_cloud_filters++;
2813         filter->add = true;
2814         adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER;
2815         spin_unlock_bh(&adapter->cloud_filter_list_lock);
2816 err:
2817         if (err)
2818                 kfree(filter);
2819
2820         clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section);
2821         return err;
2822 }
2823
2824 /* iavf_find_cf - Find the cloud filter in the list
2825  * @adapter: Board private structure
2826  * @cookie: filter specific cookie
2827  *
2828  * Returns ptr to the filter object or NULL. Must be called while holding the
2829  * cloud_filter_list_lock.
2830  */
2831 static struct iavf_cloud_filter *iavf_find_cf(struct iavf_adapter *adapter,
2832                                               unsigned long *cookie)
2833 {
2834         struct iavf_cloud_filter *filter = NULL;
2835
2836         if (!cookie)
2837                 return NULL;
2838
2839         list_for_each_entry(filter, &adapter->cloud_filter_list, list) {
2840                 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie)))
2841                         return filter;
2842         }
2843         return NULL;
2844 }
2845
2846 /**
2847  * iavf_delete_clsflower - Remove tc flower filters
2848  * @adapter: board private structure
2849  * @cls_flower: Pointer to struct tc_cls_flower_offload
2850  */
2851 static int iavf_delete_clsflower(struct iavf_adapter *adapter,
2852                                  struct tc_cls_flower_offload *cls_flower)
2853 {
2854         struct iavf_cloud_filter *filter = NULL;
2855         int err = 0;
2856
2857         spin_lock_bh(&adapter->cloud_filter_list_lock);
2858         filter = iavf_find_cf(adapter, &cls_flower->cookie);
2859         if (filter) {
2860                 filter->del = true;
2861                 adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER;
2862         } else {
2863                 err = -EINVAL;
2864         }
2865         spin_unlock_bh(&adapter->cloud_filter_list_lock);
2866
2867         return err;
2868 }
2869
2870 /**
2871  * iavf_setup_tc_cls_flower - flower classifier offloads
2872  * @netdev: net device to configure
2873  * @type_data: offload data
2874  */
2875 static int iavf_setup_tc_cls_flower(struct iavf_adapter *adapter,
2876                                     struct tc_cls_flower_offload *cls_flower)
2877 {
2878         if (cls_flower->common.chain_index)
2879                 return -EOPNOTSUPP;
2880
2881         switch (cls_flower->command) {
2882         case TC_CLSFLOWER_REPLACE:
2883                 return iavf_configure_clsflower(adapter, cls_flower);
2884         case TC_CLSFLOWER_DESTROY:
2885                 return iavf_delete_clsflower(adapter, cls_flower);
2886         case TC_CLSFLOWER_STATS:
2887                 return -EOPNOTSUPP;
2888         default:
2889                 return -EOPNOTSUPP;
2890         }
2891 }
2892
2893 /**
2894  * iavf_setup_tc_block_cb - block callback for tc
2895  * @type: type of offload
2896  * @type_data: offload data
2897  * @cb_priv:
2898  *
2899  * This function is the block callback for traffic classes
2900  **/
2901 static int iavf_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
2902                                   void *cb_priv)
2903 {
2904         switch (type) {
2905         case TC_SETUP_CLSFLOWER:
2906                 return iavf_setup_tc_cls_flower(cb_priv, type_data);
2907         default:
2908                 return -EOPNOTSUPP;
2909         }
2910 }
2911
2912 /**
2913  * iavf_setup_tc_block - register callbacks for tc
2914  * @netdev: network interface device structure
2915  * @f: tc offload data
2916  *
2917  * This function registers block callbacks for tc
2918  * offloads
2919  **/
2920 static int iavf_setup_tc_block(struct net_device *dev,
2921                                struct tc_block_offload *f)
2922 {
2923         struct iavf_adapter *adapter = netdev_priv(dev);
2924
2925         if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
2926                 return -EOPNOTSUPP;
2927
2928         switch (f->command) {
2929         case TC_BLOCK_BIND:
2930                 return tcf_block_cb_register(f->block, iavf_setup_tc_block_cb,
2931                                              adapter, adapter, f->extack);
2932         case TC_BLOCK_UNBIND:
2933                 tcf_block_cb_unregister(f->block, iavf_setup_tc_block_cb,
2934                                         adapter);
2935                 return 0;
2936         default:
2937                 return -EOPNOTSUPP;
2938         }
2939 }
2940
2941 /**
2942  * iavf_setup_tc - configure multiple traffic classes
2943  * @netdev: network interface device structure
2944  * @type: type of offload
2945  * @type_date: tc offload data
2946  *
2947  * This function is the callback to ndo_setup_tc in the
2948  * netdev_ops.
2949  *
2950  * Returns 0 on success
2951  **/
2952 static int iavf_setup_tc(struct net_device *netdev, enum tc_setup_type type,
2953                          void *type_data)
2954 {
2955         switch (type) {
2956         case TC_SETUP_QDISC_MQPRIO:
2957                 return __iavf_setup_tc(netdev, type_data);
2958         case TC_SETUP_BLOCK:
2959                 return iavf_setup_tc_block(netdev, type_data);
2960         default:
2961                 return -EOPNOTSUPP;
2962         }
2963 }
2964
2965 /**
2966  * iavf_open - Called when a network interface is made active
2967  * @netdev: network interface device structure
2968  *
2969  * Returns 0 on success, negative value on failure
2970  *
2971  * The open entry point is called when a network interface is made
2972  * active by the system (IFF_UP).  At this point all resources needed
2973  * for transmit and receive operations are allocated, the interrupt
2974  * handler is registered with the OS, the watchdog timer is started,
2975  * and the stack is notified that the interface is ready.
2976  **/
2977 static int iavf_open(struct net_device *netdev)
2978 {
2979         struct iavf_adapter *adapter = netdev_priv(netdev);
2980         int err;
2981
2982         if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) {
2983                 dev_err(&adapter->pdev->dev, "Unable to open device due to PF driver failure.\n");
2984                 return -EIO;
2985         }
2986
2987         while (test_and_set_bit(__IAVF_IN_CRITICAL_TASK,
2988                                 &adapter->crit_section))
2989                 usleep_range(500, 1000);
2990
2991         if (adapter->state != __IAVF_DOWN) {
2992                 err = -EBUSY;
2993                 goto err_unlock;
2994         }
2995
2996         /* allocate transmit descriptors */
2997         err = iavf_setup_all_tx_resources(adapter);
2998         if (err)
2999                 goto err_setup_tx;
3000
3001         /* allocate receive descriptors */
3002         err = iavf_setup_all_rx_resources(adapter);
3003         if (err)
3004                 goto err_setup_rx;
3005
3006         /* clear any pending interrupts, may auto mask */
3007         err = iavf_request_traffic_irqs(adapter, netdev->name);
3008         if (err)
3009                 goto err_req_irq;
3010
3011         spin_lock_bh(&adapter->mac_vlan_list_lock);
3012
3013         iavf_add_filter(adapter, adapter->hw.mac.addr);
3014
3015         spin_unlock_bh(&adapter->mac_vlan_list_lock);
3016
3017         iavf_configure(adapter);
3018
3019         iavf_up_complete(adapter);
3020
3021         iavf_irq_enable(adapter, true);
3022
3023         clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section);
3024
3025         return 0;
3026
3027 err_req_irq:
3028         iavf_down(adapter);
3029         iavf_free_traffic_irqs(adapter);
3030 err_setup_rx:
3031         iavf_free_all_rx_resources(adapter);
3032 err_setup_tx:
3033         iavf_free_all_tx_resources(adapter);
3034 err_unlock:
3035         clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section);
3036
3037         return err;
3038 }
3039
3040 /**
3041  * iavf_close - Disables a network interface
3042  * @netdev: network interface device structure
3043  *
3044  * Returns 0, this is not allowed to fail
3045  *
3046  * The close entry point is called when an interface is de-activated
3047  * by the OS.  The hardware is still under the drivers control, but
3048  * needs to be disabled. All IRQs except vector 0 (reserved for admin queue)
3049  * are freed, along with all transmit and receive resources.
3050  **/
3051 static int iavf_close(struct net_device *netdev)
3052 {
3053         struct iavf_adapter *adapter = netdev_priv(netdev);
3054         int status;
3055
3056         if (adapter->state <= __IAVF_DOWN_PENDING)
3057                 return 0;
3058
3059         while (test_and_set_bit(__IAVF_IN_CRITICAL_TASK,
3060                                 &adapter->crit_section))
3061                 usleep_range(500, 1000);
3062
3063         set_bit(__I40E_VSI_DOWN, adapter->vsi.state);
3064         if (CLIENT_ENABLED(adapter))
3065                 adapter->flags |= IAVF_FLAG_CLIENT_NEEDS_CLOSE;
3066
3067         iavf_down(adapter);
3068         adapter->state = __IAVF_DOWN_PENDING;
3069         iavf_free_traffic_irqs(adapter);
3070
3071         clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section);
3072
3073         /* We explicitly don't free resources here because the hardware is
3074          * still active and can DMA into memory. Resources are cleared in
3075          * iavf_virtchnl_completion() after we get confirmation from the PF
3076          * driver that the rings have been stopped.
3077          *
3078          * Also, we wait for state to transition to __IAVF_DOWN before
3079          * returning. State change occurs in iavf_virtchnl_completion() after
3080          * VF resources are released (which occurs after PF driver processes and
3081          * responds to admin queue commands).
3082          */
3083
3084         status = wait_event_timeout(adapter->down_waitqueue,
3085                                     adapter->state == __IAVF_DOWN,
3086                                     msecs_to_jiffies(200));
3087         if (!status)
3088                 netdev_warn(netdev, "Device resources not yet released\n");
3089         return 0;
3090 }
3091
3092 /**
3093  * iavf_change_mtu - Change the Maximum Transfer Unit
3094  * @netdev: network interface device structure
3095  * @new_mtu: new value for maximum frame size
3096  *
3097  * Returns 0 on success, negative on failure
3098  **/
3099 static int iavf_change_mtu(struct net_device *netdev, int new_mtu)
3100 {
3101         struct iavf_adapter *adapter = netdev_priv(netdev);
3102
3103         netdev->mtu = new_mtu;
3104         if (CLIENT_ENABLED(adapter)) {
3105                 iavf_notify_client_l2_params(&adapter->vsi);
3106                 adapter->flags |= IAVF_FLAG_SERVICE_CLIENT_REQUESTED;
3107         }
3108         adapter->flags |= IAVF_FLAG_RESET_NEEDED;
3109         schedule_work(&adapter->reset_task);
3110
3111         return 0;
3112 }
3113
3114 /**
3115  * i40e_set_features - set the netdev feature flags
3116  * @netdev: ptr to the netdev being adjusted
3117  * @features: the feature set that the stack is suggesting
3118  * Note: expects to be called while under rtnl_lock()
3119  **/
3120 static int iavf_set_features(struct net_device *netdev,
3121                              netdev_features_t features)
3122 {
3123         struct iavf_adapter *adapter = netdev_priv(netdev);
3124
3125         /* Don't allow changing VLAN_RX flag when adapter is not capable
3126          * of VLAN offload
3127          */
3128         if (!VLAN_ALLOWED(adapter)) {
3129                 if ((netdev->features ^ features) & NETIF_F_HW_VLAN_CTAG_RX)
3130                         return -EINVAL;
3131         } else if ((netdev->features ^ features) & NETIF_F_HW_VLAN_CTAG_RX) {
3132                 if (features & NETIF_F_HW_VLAN_CTAG_RX)
3133                         adapter->aq_required |=
3134                                 IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING;
3135                 else
3136                         adapter->aq_required |=
3137                                 IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING;
3138         }
3139
3140         return 0;
3141 }
3142
3143 /**
3144  * iavf_features_check - Validate encapsulated packet conforms to limits
3145  * @skb: skb buff
3146  * @dev: This physical port's netdev
3147  * @features: Offload features that the stack believes apply
3148  **/
3149 static netdev_features_t iavf_features_check(struct sk_buff *skb,
3150                                              struct net_device *dev,
3151                                              netdev_features_t features)
3152 {
3153         size_t len;
3154
3155         /* No point in doing any of this if neither checksum nor GSO are
3156          * being requested for this frame.  We can rule out both by just
3157          * checking for CHECKSUM_PARTIAL
3158          */
3159         if (skb->ip_summed != CHECKSUM_PARTIAL)
3160                 return features;
3161
3162         /* We cannot support GSO if the MSS is going to be less than
3163          * 64 bytes.  If it is then we need to drop support for GSO.
3164          */
3165         if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
3166                 features &= ~NETIF_F_GSO_MASK;
3167
3168         /* MACLEN can support at most 63 words */
3169         len = skb_network_header(skb) - skb->data;
3170         if (len & ~(63 * 2))
3171                 goto out_err;
3172
3173         /* IPLEN and EIPLEN can support at most 127 dwords */
3174         len = skb_transport_header(skb) - skb_network_header(skb);
3175         if (len & ~(127 * 4))
3176                 goto out_err;
3177
3178         if (skb->encapsulation) {
3179                 /* L4TUNLEN can support 127 words */
3180                 len = skb_inner_network_header(skb) - skb_transport_header(skb);
3181                 if (len & ~(127 * 2))
3182                         goto out_err;
3183
3184                 /* IPLEN can support at most 127 dwords */
3185                 len = skb_inner_transport_header(skb) -
3186                       skb_inner_network_header(skb);
3187                 if (len & ~(127 * 4))
3188                         goto out_err;
3189         }
3190
3191         /* No need to validate L4LEN as TCP is the only protocol with a
3192          * a flexible value and we support all possible values supported
3193          * by TCP, which is at most 15 dwords
3194          */
3195
3196         return features;
3197 out_err:
3198         return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
3199 }
3200
3201 /**
3202  * iavf_fix_features - fix up the netdev feature bits
3203  * @netdev: our net device
3204  * @features: desired feature bits
3205  *
3206  * Returns fixed-up features bits
3207  **/
3208 static netdev_features_t iavf_fix_features(struct net_device *netdev,
3209                                            netdev_features_t features)
3210 {
3211         struct iavf_adapter *adapter = netdev_priv(netdev);
3212
3213         if (!(adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN))
3214                 features &= ~(NETIF_F_HW_VLAN_CTAG_TX |
3215                               NETIF_F_HW_VLAN_CTAG_RX |
3216                               NETIF_F_HW_VLAN_CTAG_FILTER);
3217
3218         return features;
3219 }
3220
3221 static const struct net_device_ops iavf_netdev_ops = {
3222         .ndo_open               = iavf_open,
3223         .ndo_stop               = iavf_close,
3224         .ndo_start_xmit         = iavf_xmit_frame,
3225         .ndo_set_rx_mode        = iavf_set_rx_mode,
3226         .ndo_validate_addr      = eth_validate_addr,
3227         .ndo_set_mac_address    = iavf_set_mac,
3228         .ndo_change_mtu         = iavf_change_mtu,
3229         .ndo_tx_timeout         = iavf_tx_timeout,
3230         .ndo_vlan_rx_add_vid    = iavf_vlan_rx_add_vid,
3231         .ndo_vlan_rx_kill_vid   = iavf_vlan_rx_kill_vid,
3232         .ndo_features_check     = iavf_features_check,
3233         .ndo_fix_features       = iavf_fix_features,
3234         .ndo_set_features       = iavf_set_features,
3235 #ifdef CONFIG_NET_POLL_CONTROLLER
3236         .ndo_poll_controller    = iavf_netpoll,
3237 #endif
3238         .ndo_setup_tc           = iavf_setup_tc,
3239 };
3240
3241 /**
3242  * iavf_check_reset_complete - check that VF reset is complete
3243  * @hw: pointer to hw struct
3244  *
3245  * Returns 0 if device is ready to use, or -EBUSY if it's in reset.
3246  **/
3247 static int iavf_check_reset_complete(struct i40e_hw *hw)
3248 {
3249         u32 rstat;
3250         int i;
3251
3252         for (i = 0; i < 100; i++) {
3253                 rstat = rd32(hw, I40E_VFGEN_RSTAT) &
3254                             I40E_VFGEN_RSTAT_VFR_STATE_MASK;
3255                 if ((rstat == VIRTCHNL_VFR_VFACTIVE) ||
3256                     (rstat == VIRTCHNL_VFR_COMPLETED))
3257                         return 0;
3258                 usleep_range(10, 20);
3259         }
3260         return -EBUSY;
3261 }
3262
3263 /**
3264  * iavf_process_config - Process the config information we got from the PF
3265  * @adapter: board private structure
3266  *
3267  * Verify that we have a valid config struct, and set up our netdev features
3268  * and our VSI struct.
3269  **/
3270 int iavf_process_config(struct iavf_adapter *adapter)
3271 {
3272         struct virtchnl_vf_resource *vfres = adapter->vf_res;
3273         int i, num_req_queues = adapter->num_req_queues;
3274         struct net_device *netdev = adapter->netdev;
3275         struct i40e_vsi *vsi = &adapter->vsi;
3276         netdev_features_t hw_enc_features;
3277         netdev_features_t hw_features;
3278
3279         /* got VF config message back from PF, now we can parse it */
3280         for (i = 0; i < vfres->num_vsis; i++) {
3281                 if (vfres->vsi_res[i].vsi_type == VIRTCHNL_VSI_SRIOV)
3282                         adapter->vsi_res = &vfres->vsi_res[i];
3283         }
3284         if (!adapter->vsi_res) {
3285                 dev_err(&adapter->pdev->dev, "No LAN VSI found\n");
3286                 return -ENODEV;
3287         }
3288
3289         if (num_req_queues &&
3290             num_req_queues != adapter->vsi_res->num_queue_pairs) {
3291                 /* Problem.  The PF gave us fewer queues than what we had
3292                  * negotiated in our request.  Need a reset to see if we can't
3293                  * get back to a working state.
3294                  */
3295                 dev_err(&adapter->pdev->dev,
3296                         "Requested %d queues, but PF only gave us %d.\n",
3297                         num_req_queues,
3298                         adapter->vsi_res->num_queue_pairs);
3299                 adapter->flags |= IAVF_FLAG_REINIT_ITR_NEEDED;
3300                 adapter->num_req_queues = adapter->vsi_res->num_queue_pairs;
3301                 iavf_schedule_reset(adapter);
3302                 return -ENODEV;
3303         }
3304         adapter->num_req_queues = 0;
3305
3306         hw_enc_features = NETIF_F_SG                    |
3307                           NETIF_F_IP_CSUM               |
3308                           NETIF_F_IPV6_CSUM             |
3309                           NETIF_F_HIGHDMA               |
3310                           NETIF_F_SOFT_FEATURES |
3311                           NETIF_F_TSO                   |
3312                           NETIF_F_TSO_ECN               |
3313                           NETIF_F_TSO6                  |
3314                           NETIF_F_SCTP_CRC              |
3315                           NETIF_F_RXHASH                |
3316                           NETIF_F_RXCSUM                |
3317                           0;
3318
3319         /* advertise to stack only if offloads for encapsulated packets is
3320          * supported
3321          */
3322         if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ENCAP) {
3323                 hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL       |
3324                                    NETIF_F_GSO_GRE              |
3325                                    NETIF_F_GSO_GRE_CSUM         |
3326                                    NETIF_F_GSO_IPXIP4           |
3327                                    NETIF_F_GSO_IPXIP6           |
3328                                    NETIF_F_GSO_UDP_TUNNEL_CSUM  |
3329                                    NETIF_F_GSO_PARTIAL          |
3330                                    0;
3331
3332                 if (!(vfres->vf_cap_flags &
3333                       VIRTCHNL_VF_OFFLOAD_ENCAP_CSUM))
3334                         netdev->gso_partial_features |=
3335                                 NETIF_F_GSO_UDP_TUNNEL_CSUM;
3336
3337                 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
3338                 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
3339                 netdev->hw_enc_features |= hw_enc_features;
3340         }
3341         /* record features VLANs can make use of */
3342         netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;
3343
3344         /* Write features and hw_features separately to avoid polluting
3345          * with, or dropping, features that are set when we registered.
3346          */
3347         hw_features = hw_enc_features;
3348
3349         /* Enable VLAN features if supported */
3350         if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN)
3351                 hw_features |= (NETIF_F_HW_VLAN_CTAG_TX |
3352                                 NETIF_F_HW_VLAN_CTAG_RX);
3353         /* Enable cloud filter if ADQ is supported */
3354         if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ)
3355                 hw_features |= NETIF_F_HW_TC;
3356
3357         netdev->hw_features |= hw_features;
3358
3359         netdev->features |= hw_features;
3360
3361         if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN)
3362                 netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
3363
3364         netdev->priv_flags |= IFF_UNICAST_FLT;
3365
3366         /* Do not turn on offloads when they are requested to be turned off.
3367          * TSO needs minimum 576 bytes to work correctly.
3368          */
3369         if (netdev->wanted_features) {
3370                 if (!(netdev->wanted_features & NETIF_F_TSO) ||
3371                     netdev->mtu < 576)
3372                         netdev->features &= ~NETIF_F_TSO;
3373                 if (!(netdev->wanted_features & NETIF_F_TSO6) ||
3374                     netdev->mtu < 576)
3375                         netdev->features &= ~NETIF_F_TSO6;
3376                 if (!(netdev->wanted_features & NETIF_F_TSO_ECN))
3377                         netdev->features &= ~NETIF_F_TSO_ECN;
3378                 if (!(netdev->wanted_features & NETIF_F_GRO))
3379                         netdev->features &= ~NETIF_F_GRO;
3380                 if (!(netdev->wanted_features & NETIF_F_GSO))
3381                         netdev->features &= ~NETIF_F_GSO;
3382         }
3383
3384         adapter->vsi.id = adapter->vsi_res->vsi_id;
3385
3386         adapter->vsi.back = adapter;
3387         adapter->vsi.base_vector = 1;
3388         adapter->vsi.work_limit = I40E_DEFAULT_IRQ_WORK;
3389         vsi->netdev = adapter->netdev;
3390         vsi->qs_handle = adapter->vsi_res->qset_handle;
3391         if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) {
3392                 adapter->rss_key_size = vfres->rss_key_size;
3393                 adapter->rss_lut_size = vfres->rss_lut_size;
3394         } else {
3395                 adapter->rss_key_size = IAVF_HKEY_ARRAY_SIZE;
3396                 adapter->rss_lut_size = IAVF_HLUT_ARRAY_SIZE;
3397         }
3398
3399         return 0;
3400 }
3401
3402 /**
3403  * iavf_init_task - worker thread to perform delayed initialization
3404  * @work: pointer to work_struct containing our data
3405  *
3406  * This task completes the work that was begun in probe. Due to the nature
3407  * of VF-PF communications, we may need to wait tens of milliseconds to get
3408  * responses back from the PF. Rather than busy-wait in probe and bog down the
3409  * whole system, we'll do it in a task so we can sleep.
3410  * This task only runs during driver init. Once we've established
3411  * communications with the PF driver and set up our netdev, the watchdog
3412  * takes over.
3413  **/
3414 static void iavf_init_task(struct work_struct *work)
3415 {
3416         struct iavf_adapter *adapter = container_of(work,
3417                                                       struct iavf_adapter,
3418                                                       init_task.work);
3419         struct net_device *netdev = adapter->netdev;
3420         struct i40e_hw *hw = &adapter->hw;
3421         struct pci_dev *pdev = adapter->pdev;
3422         int err, bufsz;
3423
3424         switch (adapter->state) {
3425         case __IAVF_STARTUP:
3426                 /* driver loaded, probe complete */
3427                 adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED;
3428                 adapter->flags &= ~IAVF_FLAG_RESET_PENDING;
3429                 err = i40e_set_mac_type(hw);
3430                 if (err) {
3431                         dev_err(&pdev->dev, "Failed to set MAC type (%d)\n",
3432                                 err);
3433                         goto err;
3434                 }
3435                 err = iavf_check_reset_complete(hw);
3436                 if (err) {
3437                         dev_info(&pdev->dev, "Device is still in reset (%d), retrying\n",
3438                                  err);
3439                         goto err;
3440                 }
3441                 hw->aq.num_arq_entries = IAVF_AQ_LEN;
3442                 hw->aq.num_asq_entries = IAVF_AQ_LEN;
3443                 hw->aq.arq_buf_size = IAVF_MAX_AQ_BUF_SIZE;
3444                 hw->aq.asq_buf_size = IAVF_MAX_AQ_BUF_SIZE;
3445
3446                 err = iavf_init_adminq(hw);
3447                 if (err) {
3448                         dev_err(&pdev->dev, "Failed to init Admin Queue (%d)\n",
3449                                 err);
3450                         goto err;
3451                 }
3452                 err = iavf_send_api_ver(adapter);
3453                 if (err) {
3454                         dev_err(&pdev->dev, "Unable to send to PF (%d)\n", err);
3455                         iavf_shutdown_adminq(hw);
3456                         goto err;
3457                 }
3458                 adapter->state = __IAVF_INIT_VERSION_CHECK;
3459                 goto restart;
3460         case __IAVF_INIT_VERSION_CHECK:
3461                 if (!iavf_asq_done(hw)) {
3462                         dev_err(&pdev->dev, "Admin queue command never completed\n");
3463                         iavf_shutdown_adminq(hw);
3464                         adapter->state = __IAVF_STARTUP;
3465                         goto err;
3466                 }
3467
3468                 /* aq msg sent, awaiting reply */
3469                 err = iavf_verify_api_ver(adapter);
3470                 if (err) {
3471                         if (err == I40E_ERR_ADMIN_QUEUE_NO_WORK)
3472                                 err = iavf_send_api_ver(adapter);
3473                         else
3474                                 dev_err(&pdev->dev, "Unsupported PF API version %d.%d, expected %d.%d\n",
3475                                         adapter->pf_version.major,
3476                                         adapter->pf_version.minor,
3477                                         VIRTCHNL_VERSION_MAJOR,
3478                                         VIRTCHNL_VERSION_MINOR);
3479                         goto err;
3480                 }
3481                 err = iavf_send_vf_config_msg(adapter);
3482                 if (err) {
3483                         dev_err(&pdev->dev, "Unable to send config request (%d)\n",
3484                                 err);
3485                         goto err;
3486                 }
3487                 adapter->state = __IAVF_INIT_GET_RESOURCES;
3488                 goto restart;
3489         case __IAVF_INIT_GET_RESOURCES:
3490                 /* aq msg sent, awaiting reply */
3491                 if (!adapter->vf_res) {
3492                         bufsz = sizeof(struct virtchnl_vf_resource) +
3493                                 (I40E_MAX_VF_VSI *
3494                                  sizeof(struct virtchnl_vsi_resource));
3495                         adapter->vf_res = kzalloc(bufsz, GFP_KERNEL);
3496                         if (!adapter->vf_res)
3497                                 goto err;
3498                 }
3499                 err = iavf_get_vf_config(adapter);
3500                 if (err == I40E_ERR_ADMIN_QUEUE_NO_WORK) {
3501                         err = iavf_send_vf_config_msg(adapter);
3502                         goto err;
3503                 } else if (err == I40E_ERR_PARAM) {
3504                         /* We only get ERR_PARAM if the device is in a very bad
3505                          * state or if we've been disabled for previous bad
3506                          * behavior. Either way, we're done now.
3507                          */
3508                         iavf_shutdown_adminq(hw);
3509                         dev_err(&pdev->dev, "Unable to get VF config due to PF error condition, not retrying\n");
3510                         return;
3511                 }
3512                 if (err) {
3513                         dev_err(&pdev->dev, "Unable to get VF config (%d)\n",
3514                                 err);
3515                         goto err_alloc;
3516                 }
3517                 adapter->state = __IAVF_INIT_SW;
3518                 break;
3519         default:
3520                 goto err_alloc;
3521         }
3522
3523         if (iavf_process_config(adapter))
3524                 goto err_alloc;
3525         adapter->current_op = VIRTCHNL_OP_UNKNOWN;
3526
3527         adapter->flags |= IAVF_FLAG_RX_CSUM_ENABLED;
3528
3529         netdev->netdev_ops = &iavf_netdev_ops;
3530         iavf_set_ethtool_ops(netdev);
3531         netdev->watchdog_timeo = 5 * HZ;
3532
3533         /* MTU range: 68 - 9710 */
3534         netdev->min_mtu = ETH_MIN_MTU;
3535         netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD;
3536
3537         if (!is_valid_ether_addr(adapter->hw.mac.addr)) {
3538                 dev_info(&pdev->dev, "Invalid MAC address %pM, using random\n",
3539                          adapter->hw.mac.addr);
3540                 eth_hw_addr_random(netdev);
3541                 ether_addr_copy(adapter->hw.mac.addr, netdev->dev_addr);
3542         } else {
3543                 adapter->flags |= IAVF_FLAG_ADDR_SET_BY_PF;
3544                 ether_addr_copy(netdev->dev_addr, adapter->hw.mac.addr);
3545                 ether_addr_copy(netdev->perm_addr, adapter->hw.mac.addr);
3546         }
3547
3548         timer_setup(&adapter->watchdog_timer, iavf_watchdog_timer, 0);
3549         mod_timer(&adapter->watchdog_timer, jiffies + 1);
3550
3551         adapter->tx_desc_count = IAVF_DEFAULT_TXD;
3552         adapter->rx_desc_count = IAVF_DEFAULT_RXD;
3553         err = iavf_init_interrupt_scheme(adapter);
3554         if (err)
3555                 goto err_sw_init;
3556         iavf_map_rings_to_vectors(adapter);
3557         if (adapter->vf_res->vf_cap_flags &
3558             VIRTCHNL_VF_OFFLOAD_WB_ON_ITR)
3559                 adapter->flags |= IAVF_FLAG_WB_ON_ITR_CAPABLE;
3560
3561         err = iavf_request_misc_irq(adapter);
3562         if (err)
3563                 goto err_sw_init;
3564
3565         netif_carrier_off(netdev);
3566         adapter->link_up = false;
3567
3568         if (!adapter->netdev_registered) {
3569                 err = register_netdev(netdev);
3570                 if (err)
3571                         goto err_register;
3572         }
3573
3574         adapter->netdev_registered = true;
3575
3576         netif_tx_stop_all_queues(netdev);
3577         if (CLIENT_ALLOWED(adapter)) {
3578                 err = iavf_lan_add_device(adapter);
3579                 if (err)
3580                         dev_info(&pdev->dev, "Failed to add VF to client API service list: %d\n",
3581                                  err);
3582         }
3583
3584         dev_info(&pdev->dev, "MAC address: %pM\n", adapter->hw.mac.addr);
3585         if (netdev->features & NETIF_F_GRO)
3586                 dev_info(&pdev->dev, "GRO is enabled\n");
3587
3588         adapter->state = __IAVF_DOWN;
3589         set_bit(__I40E_VSI_DOWN, adapter->vsi.state);
3590         iavf_misc_irq_enable(adapter);
3591         wake_up(&adapter->down_waitqueue);
3592
3593         adapter->rss_key = kzalloc(adapter->rss_key_size, GFP_KERNEL);
3594         adapter->rss_lut = kzalloc(adapter->rss_lut_size, GFP_KERNEL);
3595         if (!adapter->rss_key || !adapter->rss_lut)
3596                 goto err_mem;
3597
3598         if (RSS_AQ(adapter)) {
3599                 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_RSS;
3600                 mod_timer_pending(&adapter->watchdog_timer, jiffies + 1);
3601         } else {
3602                 iavf_init_rss(adapter);
3603         }
3604         return;
3605 restart:
3606         schedule_delayed_work(&adapter->init_task, msecs_to_jiffies(30));
3607         return;
3608 err_mem:
3609         iavf_free_rss(adapter);
3610 err_register:
3611         iavf_free_misc_irq(adapter);
3612 err_sw_init:
3613         iavf_reset_interrupt_capability(adapter);
3614 err_alloc:
3615         kfree(adapter->vf_res);
3616         adapter->vf_res = NULL;
3617 err:
3618         /* Things went into the weeds, so try again later */
3619         if (++adapter->aq_wait_count > IAVF_AQ_MAX_ERR) {
3620                 dev_err(&pdev->dev, "Failed to communicate with PF; waiting before retry\n");
3621                 adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED;
3622                 iavf_shutdown_adminq(hw);
3623                 adapter->state = __IAVF_STARTUP;
3624                 schedule_delayed_work(&adapter->init_task, HZ * 5);
3625                 return;
3626         }
3627         schedule_delayed_work(&adapter->init_task, HZ);
3628 }
3629
3630 /**
3631  * iavf_shutdown - Shutdown the device in preparation for a reboot
3632  * @pdev: pci device structure
3633  **/
3634 static void iavf_shutdown(struct pci_dev *pdev)
3635 {
3636         struct net_device *netdev = pci_get_drvdata(pdev);
3637         struct iavf_adapter *adapter = netdev_priv(netdev);
3638
3639         netif_device_detach(netdev);
3640
3641         if (netif_running(netdev))
3642                 iavf_close(netdev);
3643
3644         /* Prevent the watchdog from running. */
3645         adapter->state = __IAVF_REMOVE;
3646         adapter->aq_required = 0;
3647
3648 #ifdef CONFIG_PM
3649         pci_save_state(pdev);
3650
3651 #endif
3652         pci_disable_device(pdev);
3653 }
3654
3655 /**
3656  * iavf_probe - Device Initialization Routine
3657  * @pdev: PCI device information struct
3658  * @ent: entry in iavf_pci_tbl
3659  *
3660  * Returns 0 on success, negative on failure
3661  *
3662  * iavf_probe initializes an adapter identified by a pci_dev structure.
3663  * The OS initialization, configuring of the adapter private structure,
3664  * and a hardware reset occur.
3665  **/
3666 static int iavf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
3667 {
3668         struct net_device *netdev;
3669         struct iavf_adapter *adapter = NULL;
3670         struct i40e_hw *hw = NULL;
3671         int err;
3672
3673         err = pci_enable_device(pdev);
3674         if (err)
3675                 return err;
3676
3677         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
3678         if (err) {
3679                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
3680                 if (err) {
3681                         dev_err(&pdev->dev,
3682                                 "DMA configuration failed: 0x%x\n", err);
3683                         goto err_dma;
3684                 }
3685         }
3686
3687         err = pci_request_regions(pdev, iavf_driver_name);
3688         if (err) {
3689                 dev_err(&pdev->dev,
3690                         "pci_request_regions failed 0x%x\n", err);
3691                 goto err_pci_reg;
3692         }
3693
3694         pci_enable_pcie_error_reporting(pdev);
3695
3696         pci_set_master(pdev);
3697
3698         netdev = alloc_etherdev_mq(sizeof(struct iavf_adapter),
3699                                    IAVF_MAX_REQ_QUEUES);
3700         if (!netdev) {
3701                 err = -ENOMEM;
3702                 goto err_alloc_etherdev;
3703         }
3704
3705         SET_NETDEV_DEV(netdev, &pdev->dev);
3706
3707         pci_set_drvdata(pdev, netdev);
3708         adapter = netdev_priv(netdev);
3709
3710         adapter->netdev = netdev;
3711         adapter->pdev = pdev;
3712
3713         hw = &adapter->hw;
3714         hw->back = adapter;
3715
3716         adapter->msg_enable = BIT(DEFAULT_DEBUG_LEVEL_SHIFT) - 1;
3717         adapter->state = __IAVF_STARTUP;
3718
3719         /* Call save state here because it relies on the adapter struct. */
3720         pci_save_state(pdev);
3721
3722         hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
3723                               pci_resource_len(pdev, 0));
3724         if (!hw->hw_addr) {
3725                 err = -EIO;
3726                 goto err_ioremap;
3727         }
3728         hw->vendor_id = pdev->vendor;
3729         hw->device_id = pdev->device;
3730         pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
3731         hw->subsystem_vendor_id = pdev->subsystem_vendor;
3732         hw->subsystem_device_id = pdev->subsystem_device;
3733         hw->bus.device = PCI_SLOT(pdev->devfn);
3734         hw->bus.func = PCI_FUNC(pdev->devfn);
3735         hw->bus.bus_id = pdev->bus->number;
3736
3737         /* set up the locks for the AQ, do this only once in probe
3738          * and destroy them only once in remove
3739          */
3740         mutex_init(&hw->aq.asq_mutex);
3741         mutex_init(&hw->aq.arq_mutex);
3742
3743         spin_lock_init(&adapter->mac_vlan_list_lock);
3744         spin_lock_init(&adapter->cloud_filter_list_lock);
3745
3746         INIT_LIST_HEAD(&adapter->mac_filter_list);
3747         INIT_LIST_HEAD(&adapter->vlan_filter_list);
3748         INIT_LIST_HEAD(&adapter->cloud_filter_list);
3749
3750         INIT_WORK(&adapter->reset_task, iavf_reset_task);
3751         INIT_WORK(&adapter->adminq_task, iavf_adminq_task);
3752         INIT_WORK(&adapter->watchdog_task, iavf_watchdog_task);
3753         INIT_DELAYED_WORK(&adapter->client_task, iavf_client_task);
3754         INIT_DELAYED_WORK(&adapter->init_task, iavf_init_task);
3755         schedule_delayed_work(&adapter->init_task,
3756                               msecs_to_jiffies(5 * (pdev->devfn & 0x07)));
3757
3758         /* Setup the wait queue for indicating transition to down status */
3759         init_waitqueue_head(&adapter->down_waitqueue);
3760
3761         return 0;
3762
3763 err_ioremap:
3764         free_netdev(netdev);
3765 err_alloc_etherdev:
3766         pci_release_regions(pdev);
3767 err_pci_reg:
3768 err_dma:
3769         pci_disable_device(pdev);
3770         return err;
3771 }
3772
3773 #ifdef CONFIG_PM
3774 /**
3775  * iavf_suspend - Power management suspend routine
3776  * @pdev: PCI device information struct
3777  * @state: unused
3778  *
3779  * Called when the system (VM) is entering sleep/suspend.
3780  **/
3781 static int iavf_suspend(struct pci_dev *pdev, pm_message_t state)
3782 {
3783         struct net_device *netdev = pci_get_drvdata(pdev);
3784         struct iavf_adapter *adapter = netdev_priv(netdev);
3785         int retval = 0;
3786
3787         netif_device_detach(netdev);
3788
3789         while (test_and_set_bit(__IAVF_IN_CRITICAL_TASK,
3790                                 &adapter->crit_section))
3791                 usleep_range(500, 1000);
3792
3793         if (netif_running(netdev)) {
3794                 rtnl_lock();
3795                 iavf_down(adapter);
3796                 rtnl_unlock();
3797         }
3798         iavf_free_misc_irq(adapter);
3799         iavf_reset_interrupt_capability(adapter);
3800
3801         clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section);
3802
3803         retval = pci_save_state(pdev);
3804         if (retval)
3805                 return retval;
3806
3807         pci_disable_device(pdev);
3808
3809         return 0;
3810 }
3811
3812 /**
3813  * iavf_resume - Power management resume routine
3814  * @pdev: PCI device information struct
3815  *
3816  * Called when the system (VM) is resumed from sleep/suspend.
3817  **/
3818 static int iavf_resume(struct pci_dev *pdev)
3819 {
3820         struct iavf_adapter *adapter = pci_get_drvdata(pdev);
3821         struct net_device *netdev = adapter->netdev;
3822         u32 err;
3823
3824         pci_set_power_state(pdev, PCI_D0);
3825         pci_restore_state(pdev);
3826         /* pci_restore_state clears dev->state_saved so call
3827          * pci_save_state to restore it.
3828          */
3829         pci_save_state(pdev);
3830
3831         err = pci_enable_device_mem(pdev);
3832         if (err) {
3833                 dev_err(&pdev->dev, "Cannot enable PCI device from suspend.\n");
3834                 return err;
3835         }
3836         pci_set_master(pdev);
3837
3838         rtnl_lock();
3839         err = iavf_set_interrupt_capability(adapter);
3840         if (err) {
3841                 rtnl_unlock();
3842                 dev_err(&pdev->dev, "Cannot enable MSI-X interrupts.\n");
3843                 return err;
3844         }
3845         err = iavf_request_misc_irq(adapter);
3846         rtnl_unlock();
3847         if (err) {
3848                 dev_err(&pdev->dev, "Cannot get interrupt vector.\n");
3849                 return err;
3850         }
3851
3852         schedule_work(&adapter->reset_task);
3853
3854         netif_device_attach(netdev);
3855
3856         return err;
3857 }
3858
3859 #endif /* CONFIG_PM */
3860 /**
3861  * iavf_remove - Device Removal Routine
3862  * @pdev: PCI device information struct
3863  *
3864  * iavf_remove is called by the PCI subsystem to alert the driver
3865  * that it should release a PCI device.  The could be caused by a
3866  * Hot-Plug event, or because the driver is going to be removed from
3867  * memory.
3868  **/
3869 static void iavf_remove(struct pci_dev *pdev)
3870 {
3871         struct net_device *netdev = pci_get_drvdata(pdev);
3872         struct iavf_adapter *adapter = netdev_priv(netdev);
3873         struct iavf_vlan_filter *vlf, *vlftmp;
3874         struct iavf_mac_filter *f, *ftmp;
3875         struct iavf_cloud_filter *cf, *cftmp;
3876         struct i40e_hw *hw = &adapter->hw;
3877         int err;
3878         /* Indicate we are in remove and not to run reset_task */
3879         set_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section);
3880         cancel_delayed_work_sync(&adapter->init_task);
3881         cancel_work_sync(&adapter->reset_task);
3882         cancel_delayed_work_sync(&adapter->client_task);
3883         if (adapter->netdev_registered) {
3884                 unregister_netdev(netdev);
3885                 adapter->netdev_registered = false;
3886         }
3887         if (CLIENT_ALLOWED(adapter)) {
3888                 err = iavf_lan_del_device(adapter);
3889                 if (err)
3890                         dev_warn(&pdev->dev, "Failed to delete client device: %d\n",
3891                                  err);
3892         }
3893
3894         /* Shut down all the garbage mashers on the detention level */
3895         adapter->state = __IAVF_REMOVE;
3896         adapter->aq_required = 0;
3897         adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED;
3898         iavf_request_reset(adapter);
3899         msleep(50);
3900         /* If the FW isn't responding, kick it once, but only once. */
3901         if (!iavf_asq_done(hw)) {
3902                 iavf_request_reset(adapter);
3903                 msleep(50);
3904         }
3905         iavf_free_all_tx_resources(adapter);
3906         iavf_free_all_rx_resources(adapter);
3907         iavf_misc_irq_disable(adapter);
3908         iavf_free_misc_irq(adapter);
3909         iavf_reset_interrupt_capability(adapter);
3910         iavf_free_q_vectors(adapter);
3911
3912         if (adapter->watchdog_timer.function)
3913                 del_timer_sync(&adapter->watchdog_timer);
3914
3915         cancel_work_sync(&adapter->adminq_task);
3916
3917         iavf_free_rss(adapter);
3918
3919         if (hw->aq.asq.count)
3920                 iavf_shutdown_adminq(hw);
3921
3922         /* destroy the locks only once, here */
3923         mutex_destroy(&hw->aq.arq_mutex);
3924         mutex_destroy(&hw->aq.asq_mutex);
3925
3926         iounmap(hw->hw_addr);
3927         pci_release_regions(pdev);
3928         iavf_free_all_tx_resources(adapter);
3929         iavf_free_all_rx_resources(adapter);
3930         iavf_free_queues(adapter);
3931         kfree(adapter->vf_res);
3932         spin_lock_bh(&adapter->mac_vlan_list_lock);
3933         /* If we got removed before an up/down sequence, we've got a filter
3934          * hanging out there that we need to get rid of.
3935          */
3936         list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
3937                 list_del(&f->list);
3938                 kfree(f);
3939         }
3940         list_for_each_entry_safe(vlf, vlftmp, &adapter->vlan_filter_list,
3941                                  list) {
3942                 list_del(&vlf->list);
3943                 kfree(vlf);
3944         }
3945
3946         spin_unlock_bh(&adapter->mac_vlan_list_lock);
3947
3948         spin_lock_bh(&adapter->cloud_filter_list_lock);
3949         list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) {
3950                 list_del(&cf->list);
3951                 kfree(cf);
3952         }
3953         spin_unlock_bh(&adapter->cloud_filter_list_lock);
3954
3955         free_netdev(netdev);
3956
3957         pci_disable_pcie_error_reporting(pdev);
3958
3959         pci_disable_device(pdev);
3960 }
3961
3962 static struct pci_driver iavf_driver = {
3963         .name     = iavf_driver_name,
3964         .id_table = iavf_pci_tbl,
3965         .probe    = iavf_probe,
3966         .remove   = iavf_remove,
3967 #ifdef CONFIG_PM
3968         .suspend  = iavf_suspend,
3969         .resume   = iavf_resume,
3970 #endif
3971         .shutdown = iavf_shutdown,
3972 };
3973
3974 /**
3975  * i40e_init_module - Driver Registration Routine
3976  *
3977  * i40e_init_module is the first routine called when the driver is
3978  * loaded. All it does is register with the PCI subsystem.
3979  **/
3980 static int __init iavf_init_module(void)
3981 {
3982         int ret;
3983
3984         pr_info("iavf: %s - version %s\n", iavf_driver_string,
3985                 iavf_driver_version);
3986
3987         pr_info("%s\n", iavf_copyright);
3988
3989         iavf_wq = alloc_workqueue("%s", WQ_UNBOUND | WQ_MEM_RECLAIM, 1,
3990                                   iavf_driver_name);
3991         if (!iavf_wq) {
3992                 pr_err("%s: Failed to create workqueue\n", iavf_driver_name);
3993                 return -ENOMEM;
3994         }
3995         ret = pci_register_driver(&iavf_driver);
3996         return ret;
3997 }
3998
3999 module_init(iavf_init_module);
4000
4001 /**
4002  * i40e_exit_module - Driver Exit Cleanup Routine
4003  *
4004  * i40e_exit_module is called just before the driver is removed
4005  * from memory.
4006  **/
4007 static void __exit iavf_exit_module(void)
4008 {
4009         pci_unregister_driver(&iavf_driver);
4010         destroy_workqueue(iavf_wq);
4011 }
4012
4013 module_exit(iavf_exit_module);
4014
4015 /* iavf_main.c */